From da9235f58e7f67eb9f2ab73007ae720976542031 Mon Sep 17 00:00:00 2001 From: ITotalJustice <47043333+ITotalJustice@users.noreply.github.com> Date: Mon, 19 May 2025 12:06:43 +0100 Subject: [PATCH] fix upload url path not being encoded, add seek api for uploads. --- sphaira/CMakeLists.txt | 1 + sphaira/include/download.hpp | 132 ++++++--------------- sphaira/source/download.cpp | 165 ++++++++++++++++++++------ sphaira/source/ui/menus/game_menu.cpp | 6 + 4 files changed, 169 insertions(+), 135 deletions(-) diff --git a/sphaira/CMakeLists.txt b/sphaira/CMakeLists.txt index da245c2..8a8701a 100644 --- a/sphaira/CMakeLists.txt +++ b/sphaira/CMakeLists.txt @@ -105,6 +105,7 @@ add_executable(sphaira target_compile_definitions(sphaira PRIVATE -DAPP_VERSION="${sphaira_VERSION}" -DAPP_VERSION_HASH="${sphaira_VERSION_HASH}" + -DCURL_NO_OLDIES=1 ) target_compile_options(sphaira PRIVATE diff --git a/sphaira/include/download.hpp b/sphaira/include/download.hpp index f970ede..97d2599 100644 --- a/sphaira/include/download.hpp +++ b/sphaira/include/download.hpp @@ -31,6 +31,7 @@ using Path = fs::FsPath; using OnComplete = std::function; using OnProgress = std::function; using OnUploadCallback = std::function; +using OnUploadSeek = std::function; using StopToken = std::stop_token; struct Url { @@ -271,105 +272,43 @@ struct Api { return curl::FromFileAsync(*this); } - void SetUpload(bool enable) { - m_is_upload = enable; - } + void SetUpload(bool enable) { m_is_upload = enable; } - auto IsUpload() const { - return m_is_upload; - } - auto& GetUrl() const { - return m_url.m_str; - } - auto& GetFields() const { - return m_fields.m_str; - } - auto& GetHeader() const { - return m_header; - } - auto& GetFlags() const { - return m_flags.m_flags; - } - auto& GetPath() const { - return m_path; - } - auto& GetPort() const { - return m_port; - } - auto& GetUserPass() const { - return m_userpass; - } - auto& GetBearer() const { - return m_bearer; - } - auto& GetPubKey() const { - return m_pub_key; - } - auto& GetPrivKey() const { - return m_priv_key; - } - auto& GetUploadInfo() const { - return m_info; - } - auto& GetOnComplete() const { - return m_on_complete; - } - auto& GetOnProgress() const { - return m_on_progress; - } - auto& GetPriority() const { - return m_prio; - } - auto& GetToken() const { - return m_stoken; - } + auto IsUpload() const { return m_is_upload; } + auto& GetUrl() const { return m_url.m_str; } + auto& GetFields() const { return m_fields.m_str; } + auto& GetHeader() const { return m_header; } + auto& GetFlags() const { return m_flags.m_flags; } + auto& GetPath() const { return m_path; } + auto& GetPort() const { return m_port.m_port; } + auto& GetUserPass() const { return m_userpass; } + auto& GetBearer() const { return m_bearer.m_str; } + auto& GetPubKey() const { return m_pub_key.m_str; } + auto& GetPrivKey() const { return m_priv_key.m_str; } + auto& GetUploadInfo() const { return m_info; } + auto& GetOnComplete() const { return m_on_complete; } + auto& GetOnProgress() const { return m_on_progress; } + auto& GetOnUploadSeek() const { return m_on_upload_seek; } + auto& GetPriority() const { return m_prio; } + auto& GetToken() const { return m_stoken; } private: - void SetOption(Url&& v) { - m_url = v; - } - void SetOption(Fields&& v) { - m_fields = v; - } - void SetOption(Header&& v) { - m_header = v; - } - void SetOption(Flags&& v) { - m_flags = v; - } - void SetOption(Path&& v) { - m_path = v; - } - void SetOption(Port&& v) { - m_port = v; - } - void SetOption(UserPass&& v) { - m_userpass = v; - } - void SetOption(Bearer&& v) { - m_bearer = v; - } - void SetOption(PubKey&& v) { - m_pub_key = v; - } - void SetOption(PrivKey&& v) { - m_priv_key = v; - } - void SetOption(UploadInfo&& v) { - m_info = v; - } - void SetOption(OnComplete&& v) { - m_on_complete = v; - } - void SetOption(OnProgress&& v) { - m_on_progress = v; - } - void SetOption(Priority&& v) { - m_prio = v; - } - void SetOption(StopToken&& v) { - m_stoken = v; - } + void SetOption(Url&& v) { m_url = v; } + void SetOption(Fields&& v) { m_fields = v; } + void SetOption(Header&& v) { m_header = v; } + void SetOption(Flags&& v) { m_flags = v; } + void SetOption(Path&& v) { m_path = v; } + void SetOption(Port&& v) { m_port = v; } + void SetOption(UserPass&& v) { m_userpass = v; } + void SetOption(Bearer&& v) { m_bearer = v; } + void SetOption(PubKey&& v) { m_pub_key = v; } + void SetOption(PrivKey&& v) { m_priv_key = v; } + void SetOption(UploadInfo&& v) { m_info = v; } + void SetOption(OnComplete&& v) { m_on_complete = v; } + void SetOption(OnProgress&& v) { m_on_progress = v; } + void SetOption(OnUploadSeek&& v) { m_on_upload_seek = v; } + void SetOption(Priority&& v) { m_prio = v; } + void SetOption(StopToken&& v) { m_stoken = v; } template void set_option(T&& t) { @@ -396,6 +335,7 @@ private: UploadInfo m_info{}; OnComplete m_on_complete{}; OnProgress m_on_progress{}; + OnUploadSeek m_on_upload_seek{}; Priority m_prio{Priority::High}; std::stop_source m_stop_source{}; StopToken m_stoken{m_stop_source.get_token()}; diff --git a/sphaira/source/download.cpp b/sphaira/source/download.cpp index f30f3f1..3e7e595 100644 --- a/sphaira/source/download.cpp +++ b/sphaira/source/download.cpp @@ -41,6 +41,7 @@ Mutex g_mutex_share[CURL_LOCK_DATA_LAST]{}; struct UploadStruct { std::span data; s64 offset{}; + s64 size{}; FsFile f{}; }; @@ -51,6 +52,11 @@ struct DataStruct { s64 file_offset{}; }; +struct SeekCustomData { + OnUploadSeek cb{}; + s64 size{}; +}; + auto generate_key_from_path(const fs::FsPath& path) -> std::string { const auto key = crc32Calculate(path.s, path.size()); return std::to_string(key); @@ -375,6 +381,46 @@ auto ProgressCallbackFunc2(void *clientp, curl_off_t dltotal, curl_off_t dlnow, return 0; } +auto SeekCallback(void *clientp, curl_off_t offset, int origin) -> int { + if (!g_running) { + return 0; + } + + auto data_struct = static_cast(clientp); + + if (origin == SEEK_SET) { + offset = offset; + } else if (origin == SEEK_CUR) { + offset = data_struct->offset + offset; + } else if (origin == SEEK_END) { + offset = data_struct->size; + } + + if (offset < 0 || offset > data_struct->size) { + return CURL_SEEKFUNC_CANTSEEK; + } + + data_struct->offset = offset; + return CURL_SEEKFUNC_OK; +} + +auto SeekCustomCallback(void *clientp, curl_off_t offset, int origin) -> int { + if (!g_running) { + return 0; + } + + auto data_struct = static_cast(clientp); + if (origin != SEEK_SET || offset < 0 || offset > data_struct->size) { + return CURL_SEEKFUNC_CANTSEEK; + } + + if (!data_struct->cb(offset)) { + return CURL_SEEKFUNC_CANTSEEK; + } + + return CURL_SEEKFUNC_OK; +} + auto ReadFileCallback(char *ptr, size_t size, size_t nmemb, void *userp) -> size_t { if (!g_running) { return 0; @@ -518,17 +564,20 @@ void SetCommonCurlOptions(CURL* curl, const Api& e) { // in most cases, this will use CURLAUTH_BASIC. CURL_EASY_SETOPT_LOG(curl, CURLOPT_HTTPAUTH, (long)CURLAUTH_ANY); + // enable TE is server supports it. + CURL_EASY_SETOPT_LOG(curl, CURLOPT_TRANSFER_ENCODING, 1L); + // set oath2 bearer. - if (!e.GetBearer().m_str.empty()) { - CURL_EASY_SETOPT_LOG(curl, CURLOPT_XOAUTH2_BEARER, e.GetBearer().m_str.c_str()); + if (!e.GetBearer().empty()) { + CURL_EASY_SETOPT_LOG(curl, CURLOPT_XOAUTH2_BEARER, e.GetBearer().c_str()); } // set ssh pub/priv key file. - if (!e.GetPubKey().m_str.empty()) { - CURL_EASY_SETOPT_LOG(curl, CURLOPT_SSH_PUBLIC_KEYFILE, e.GetPubKey().m_str.c_str()); + if (!e.GetPubKey().empty()) { + CURL_EASY_SETOPT_LOG(curl, CURLOPT_SSH_PUBLIC_KEYFILE, e.GetPubKey().c_str()); } - if (!e.GetPrivKey().m_str.empty()) { - CURL_EASY_SETOPT_LOG(curl, CURLOPT_SSH_PRIVATE_KEYFILE, e.GetPrivKey().m_str.c_str()); + if (!e.GetPrivKey().empty()) { + CURL_EASY_SETOPT_LOG(curl, CURLOPT_SSH_PRIVATE_KEYFILE, e.GetPrivKey().c_str()); } // set auth. @@ -540,13 +589,55 @@ void SetCommonCurlOptions(CURL* curl, const Api& e) { } // set port, if valid. - if (e.GetPort().m_port) { - CURL_EASY_SETOPT_LOG(curl, CURLOPT_PORT, (long)e.GetPort().m_port); + if (e.GetPort()) { + CURL_EASY_SETOPT_LOG(curl, CURLOPT_PORT, (long)e.GetPort()); } + // progress calls. + if (e.GetOnProgress()) { + CURL_EASY_SETOPT_LOG(curl, CURLOPT_XFERINFODATA, &e); + CURL_EASY_SETOPT_LOG(curl, CURLOPT_XFERINFOFUNCTION, ProgressCallbackFunc2); + } else { + CURL_EASY_SETOPT_LOG(curl, CURLOPT_XFERINFOFUNCTION, ProgressCallbackFunc1); + } CURL_EASY_SETOPT_LOG(curl, CURLOPT_NOPROGRESS, 0L); } +auto EscapeString(CURL* curl, const std::string& str) -> std::string { + char* s{}; + if (!curl) { + s = curl_escape(str.data(), str.length()); + } else { + s = curl_easy_escape(curl, str.data(), str.length()); + } + + if (!s) { + return str; + } + + const std::string result = s; + curl_free(s); + return result; +} + +auto EncodeUrl(const std::string& url) -> std::string { + auto clu = curl_url(); + R_UNLESS(clu, url); + ON_SCOPE_EXIT(curl_url_cleanup(clu)); + + CURLUcode clu_code; + clu_code = curl_url_set(clu, CURLUPART_URL, url.c_str(), CURLU_URLENCODE); + R_UNLESS(clu_code == CURLUE_OK, url); + + char* encoded_url; + clu_code = curl_url_get(clu, CURLUPART_URL, &encoded_url, 0); + R_UNLESS(clu_code == CURLUE_OK, url); + + const std::string out = encoded_url; + curl_free(encoded_url); + return out; +} + auto DownloadInternal(CURL* curl, const Api& e) -> ApiResult { // check if stop has been requested before starting download if (e.GetToken().stop_requested()) { @@ -622,15 +713,6 @@ auto DownloadInternal(CURL* curl, const Api& e) -> ApiResult { CURL_EASY_SETOPT_LOG(curl, CURLOPT_HTTPHEADER, list); } - // progress calls. - if (e.GetOnProgress()) { - CURL_EASY_SETOPT_LOG(curl, CURLOPT_XFERINFODATA, &e); - CURL_EASY_SETOPT_LOG(curl, CURLOPT_XFERINFOFUNCTION, ProgressCallbackFunc2); - } else { - CURL_EASY_SETOPT_LOG(curl, CURLOPT_XFERINFOFUNCTION, ProgressCallbackFunc1); - } - CURL_EASY_SETOPT_LOG(curl, CURLOPT_NOPROGRESS, 0L); - // write calls. CURL_EASY_SETOPT_LOG(curl, CURLOPT_WRITEFUNCTION, has_file ? WriteFileCallback : WriteMemoryCallback); CURL_EASY_SETOPT_LOG(curl, CURLOPT_WRITEDATA, &chunk); @@ -699,10 +781,10 @@ auto UploadInternal(CURL* curl, const Api& e) -> ApiResult { UploadStruct chunk{}; DataStruct chunk_out{}; + SeekCustomData seek_data{}; Header header_in = e.GetHeader(); Header header_out; fs::FsNativeSd fs{}; - s64 upload_size{}; if (has_file) { if (R_FAILED(fs.OpenFile(e.GetPath(), FsOpenMode_Read, &chunk.f))) { @@ -710,14 +792,14 @@ auto UploadInternal(CURL* curl, const Api& e) -> ApiResult { return {}; } - fsFileGetSize(&chunk.f, &upload_size); - log_write("got chunk size: %zd\n", upload_size); + fsFileGetSize(&chunk.f, &chunk.size); + log_write("got chunk size: %zd\n", chunk.size); } else { if (info.m_callback) { - upload_size = info.m_size; - log_write("setting upload size: %zu\n", upload_size); + chunk.size = info.m_size; + log_write("setting upload size: %zu\n", chunk.size); } else { - upload_size = info.m_data.size(); + chunk.size = info.m_data.size(); chunk.data = info.m_data; } } @@ -737,12 +819,20 @@ auto UploadInternal(CURL* curl, const Api& e) -> ApiResult { curl_easy_reset(curl); SetCommonCurlOptions(curl, e); - CURL_EASY_SETOPT_LOG(curl, CURLOPT_URL, url.c_str()); + // encode url + auto clu = curl_url(); + R_UNLESS(clu, {}); + ON_SCOPE_EXIT(curl_url_cleanup(clu)); + + const auto clu_code = curl_url_set(clu, CURLUPART_URL, url.c_str(), CURLU_URLENCODE); + R_UNLESS(clu_code == CURLUE_OK, {}); + + CURL_EASY_SETOPT_LOG(curl, CURLOPT_CURLU, clu); CURL_EASY_SETOPT_LOG(curl, CURLOPT_HEADERFUNCTION, header_callback); CURL_EASY_SETOPT_LOG(curl, CURLOPT_HEADERDATA, &header_out); CURL_EASY_SETOPT_LOG(curl, CURLOPT_UPLOAD, 1L); - CURL_EASY_SETOPT_LOG(curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)upload_size); + CURL_EASY_SETOPT_LOG(curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)chunk.size); // instruct libcurl to create ftp folders if they don't yet exist. CURL_EASY_SETOPT_LOG(curl, CURLOPT_FTP_CREATE_MISSING_DIRS, CURLFTP_CREATE_DIR_RETRY); @@ -776,17 +866,20 @@ auto UploadInternal(CURL* curl, const Api& e) -> ApiResult { if (info.m_callback) { CURL_EASY_SETOPT_LOG(curl, CURLOPT_READFUNCTION, ReadCustomCallback); CURL_EASY_SETOPT_LOG(curl, CURLOPT_READDATA, &info); + + if (e.GetOnUploadSeek()) { + seek_data.cb = e.GetOnUploadSeek(); + seek_data.size = chunk.size; + CURL_EASY_SETOPT_LOG(curl, CURLOPT_SEEKFUNCTION, SeekCustomCallback); + CURL_EASY_SETOPT_LOG(curl, CURLOPT_SEEKDATA, &seek_data); + } } else { CURL_EASY_SETOPT_LOG(curl, CURLOPT_READFUNCTION, has_file ? ReadFileCallback : ReadMemoryCallback); CURL_EASY_SETOPT_LOG(curl, CURLOPT_READDATA, &chunk); - } - // progress calls. - if (e.GetOnProgress()) { - CURL_EASY_SETOPT_LOG(curl, CURLOPT_XFERINFODATA, &e); - CURL_EASY_SETOPT_LOG(curl, CURLOPT_XFERINFOFUNCTION, ProgressCallbackFunc2); - } else { - CURL_EASY_SETOPT_LOG(curl, CURLOPT_XFERINFOFUNCTION, ProgressCallbackFunc1); + // allow for seeking upon uploads, may be used for ftp and http. + CURL_EASY_SETOPT_LOG(curl, CURLOPT_SEEKFUNCTION, SeekCallback); + CURL_EASY_SETOPT_LOG(curl, CURLOPT_SEEKDATA, &chunk); } // write calls. @@ -1013,13 +1106,7 @@ auto FromFileAsync(const Api& e) -> bool { } auto EscapeString(const std::string& str) -> std::string { - std::string result; - const auto s = curl_escape(str.data(), str.length()); - if (s) { - result = s; - curl_free(s); - } - return result; + return EscapeString(nullptr, str); } } // namespace sphaira::curl diff --git a/sphaira/source/ui/menus/game_menu.cpp b/sphaira/source/ui/menus/game_menu.cpp index 852c78d..3f3679c 100644 --- a/sphaira/source/ui/menus/game_menu.cpp +++ b/sphaira/source/ui/menus/game_menu.cpp @@ -426,6 +426,12 @@ Result DumpNspToNetwork(ProgressBox* pbox, const location::Entry& loc, std::span offset += bytes_read; return bytes_read; } + }, + curl::OnUploadSeek{ + [&e, &offset](s64 new_offset){ + offset = new_offset; + return true; + } } );