fix upload url path not being encoded, add seek api for uploads.

sphaira/CMakeLists.txt

@@ -105,6 +105,7 @@ add_executable(sphaira
 target_compile_definitions(sphaira PRIVATE
     -DAPP_VERSION="${sphaira_VERSION}"
     -DAPP_VERSION_HASH="${sphaira_VERSION_HASH}"
+    -DCURL_NO_OLDIES=1
 )
 
 target_compile_options(sphaira PRIVATE

sphaira/include/download.hpp

@@ -31,6 +31,7 @@ using Path = fs::FsPath;
 using OnComplete = std::function<void(ApiResult& result)>;
 using OnProgress = std::function<bool(s64 dltotal, s64 dlnow, s64 ultotal, s64 ulnow)>;
 using OnUploadCallback = std::function<size_t(void *ptr, size_t size)>;
+using OnUploadSeek = std::function<bool(s64 offset)>;
 using StopToken = std::stop_token;
 
 struct Url {
@@ -271,105 +272,43 @@ struct Api {
         return curl::FromFileAsync(*this);
     }
 
-    void SetUpload(bool enable) {
+    void SetUpload(bool enable) { m_is_upload = enable; }
-        m_is_upload = enable;
-    }
 
-    auto IsUpload() const {
+    auto IsUpload() const { return m_is_upload; }
-        return m_is_upload;
+    auto& GetUrl() const { return m_url.m_str; }
-    }
+    auto& GetFields() const { return m_fields.m_str; }
-    auto& GetUrl() const {
+    auto& GetHeader() const { return m_header; }
-        return m_url.m_str;
+    auto& GetFlags() const { return m_flags.m_flags; }
-    }
+    auto& GetPath() const { return m_path; }
-    auto& GetFields() const {
+    auto& GetPort() const { return m_port.m_port; }
-        return m_fields.m_str;
+    auto& GetUserPass() const { return m_userpass; }
-    }
+    auto& GetBearer() const { return m_bearer.m_str; }
-    auto& GetHeader() const {
+    auto& GetPubKey() const { return m_pub_key.m_str; }
-        return m_header;
+    auto& GetPrivKey() const { return m_priv_key.m_str; }
-    }
+    auto& GetUploadInfo() const { return m_info; }
-    auto& GetFlags() const {
+    auto& GetOnComplete() const { return m_on_complete; }
-        return m_flags.m_flags;
+    auto& GetOnProgress() const { return m_on_progress; }
-    }
+    auto& GetOnUploadSeek() const { return m_on_upload_seek; }
-    auto& GetPath() const {
+    auto& GetPriority() const { return m_prio; }
-        return m_path;
+    auto& GetToken() const { return m_stoken; }
-    }
-    auto& GetPort() const {
-        return m_port;
-    }
-    auto& GetUserPass() const {
-        return m_userpass;
-    }
-    auto& GetBearer() const {
-        return m_bearer;
-    }
-    auto& GetPubKey() const {
-        return m_pub_key;
-    }
-    auto& GetPrivKey() const {
-        return m_priv_key;
-    }
-    auto& GetUploadInfo() const {
-        return m_info;
-    }
-    auto& GetOnComplete() const {
-        return m_on_complete;
-    }
-    auto& GetOnProgress() const {
-        return m_on_progress;
-    }
-    auto& GetPriority() const {
-        return m_prio;
-    }
-    auto& GetToken() const {
-        return m_stoken;
-    }
 
 private:
-    void SetOption(Url&& v) {
+    void SetOption(Url&& v) { m_url = v; }
-        m_url = v;
+    void SetOption(Fields&& v) { m_fields = v; }
-    }
+    void SetOption(Header&& v) { m_header = v; }
-    void SetOption(Fields&& v) {
+    void SetOption(Flags&& v) { m_flags = v; }
-        m_fields = v;
+    void SetOption(Path&& v) { m_path = v; }
-    }
+    void SetOption(Port&& v) { m_port = v; }
-    void SetOption(Header&& v) {
+    void SetOption(UserPass&& v) { m_userpass = v; }
-        m_header = v;
+    void SetOption(Bearer&& v) { m_bearer = v; }
-    }
+    void SetOption(PubKey&& v) { m_pub_key = v; }
-    void SetOption(Flags&& v) {
+    void SetOption(PrivKey&& v) { m_priv_key = v; }
-        m_flags = v;
+    void SetOption(UploadInfo&& v) { m_info = v; }
-    }
+    void SetOption(OnComplete&& v) { m_on_complete = v; }
-    void SetOption(Path&& v) {
+    void SetOption(OnProgress&& v) { m_on_progress = v; }
-        m_path = v;
+    void SetOption(OnUploadSeek&& v) { m_on_upload_seek = v; }
-    }
+    void SetOption(Priority&& v) { m_prio = v; }
-    void SetOption(Port&& v) {
+    void SetOption(StopToken&& v) { m_stoken = v; }
-        m_port = v;
-    }
-    void SetOption(UserPass&& v) {
-        m_userpass = v;
-    }
-    void SetOption(Bearer&& v) {
-        m_bearer = v;
-    }
-    void SetOption(PubKey&& v) {
-        m_pub_key = v;
-    }
-    void SetOption(PrivKey&& v) {
-        m_priv_key = v;
-    }
-    void SetOption(UploadInfo&& v) {
-        m_info = v;
-    }
-    void SetOption(OnComplete&& v) {
-        m_on_complete = v;
-    }
-    void SetOption(OnProgress&& v) {
-        m_on_progress = v;
-    }
-    void SetOption(Priority&& v) {
-        m_prio = v;
-    }
-    void SetOption(StopToken&& v) {
-        m_stoken = v;
-    }
 
     template <typename T>
     void set_option(T&& t) {
@@ -396,6 +335,7 @@ private:
     UploadInfo m_info{};
     OnComplete m_on_complete{};
     OnProgress m_on_progress{};
+    OnUploadSeek m_on_upload_seek{};
     Priority m_prio{Priority::High};
     std::stop_source m_stop_source{};
     StopToken m_stoken{m_stop_source.get_token()};

sphaira/source/download.cpp

@@ -41,6 +41,7 @@ Mutex g_mutex_share[CURL_LOCK_DATA_LAST]{};
 struct UploadStruct {
     std::span<const u8> data;
     s64 offset{};
+    s64 size{};
     FsFile f{};
 };
 
@@ -51,6 +52,11 @@ struct DataStruct {
     s64 file_offset{};
 };
 
+struct SeekCustomData {
+    OnUploadSeek cb{};
+    s64 size{};
+};
+
 auto generate_key_from_path(const fs::FsPath& path) -> std::string {
     const auto key = crc32Calculate(path.s, path.size());
     return std::to_string(key);
@@ -375,6 +381,46 @@ auto ProgressCallbackFunc2(void *clientp, curl_off_t dltotal, curl_off_t dlnow,
     return 0;
 }
 
+auto SeekCallback(void *clientp, curl_off_t offset, int origin) -> int {
+    if (!g_running) {
+        return 0;
+    }
+
+    auto data_struct = static_cast<UploadStruct*>(clientp);
+
+    if (origin == SEEK_SET) {
+        offset = offset;
+    } else if (origin == SEEK_CUR) {
+        offset = data_struct->offset + offset;
+    } else if (origin == SEEK_END) {
+        offset = data_struct->size;
+    }
+
+    if (offset < 0 || offset > data_struct->size) {
+        return CURL_SEEKFUNC_CANTSEEK;
+    }
+
+    data_struct->offset = offset;
+    return CURL_SEEKFUNC_OK;
+}
+
+auto SeekCustomCallback(void *clientp, curl_off_t offset, int origin) -> int {
+    if (!g_running) {
+        return 0;
+    }
+
+    auto data_struct = static_cast<SeekCustomData*>(clientp);
+    if (origin != SEEK_SET || offset < 0 || offset > data_struct->size) {
+        return CURL_SEEKFUNC_CANTSEEK;
+    }
+
+    if (!data_struct->cb(offset)) {
+        return CURL_SEEKFUNC_CANTSEEK;
+    }
+
+    return CURL_SEEKFUNC_OK;
+}
+
 auto ReadFileCallback(char *ptr, size_t size, size_t nmemb, void *userp) -> size_t {
     if (!g_running) {
         return 0;
@@ -518,17 +564,20 @@ void SetCommonCurlOptions(CURL* curl, const Api& e) {
     // in most cases, this will use CURLAUTH_BASIC.
     CURL_EASY_SETOPT_LOG(curl, CURLOPT_HTTPAUTH, (long)CURLAUTH_ANY);
 
+    // enable TE is server supports it.
+    CURL_EASY_SETOPT_LOG(curl, CURLOPT_TRANSFER_ENCODING, 1L);
+
     // set oath2 bearer.
-    if (!e.GetBearer().m_str.empty()) {
+    if (!e.GetBearer().empty()) {
-        CURL_EASY_SETOPT_LOG(curl, CURLOPT_XOAUTH2_BEARER, e.GetBearer().m_str.c_str());
+        CURL_EASY_SETOPT_LOG(curl, CURLOPT_XOAUTH2_BEARER, e.GetBearer().c_str());
     }
 
     // set ssh pub/priv key file.
-    if (!e.GetPubKey().m_str.empty()) {
+    if (!e.GetPubKey().empty()) {
-        CURL_EASY_SETOPT_LOG(curl, CURLOPT_SSH_PUBLIC_KEYFILE, e.GetPubKey().m_str.c_str());
+        CURL_EASY_SETOPT_LOG(curl, CURLOPT_SSH_PUBLIC_KEYFILE, e.GetPubKey().c_str());
     }
-    if (!e.GetPrivKey().m_str.empty()) {
+    if (!e.GetPrivKey().empty()) {
-        CURL_EASY_SETOPT_LOG(curl, CURLOPT_SSH_PRIVATE_KEYFILE, e.GetPrivKey().m_str.c_str());
+        CURL_EASY_SETOPT_LOG(curl, CURLOPT_SSH_PRIVATE_KEYFILE, e.GetPrivKey().c_str());
     }
 
     // set auth.
@@ -540,13 +589,55 @@ void SetCommonCurlOptions(CURL* curl, const Api& e) {
     }
 
     // set port, if valid.
-    if (e.GetPort().m_port) {
+    if (e.GetPort()) {
-        CURL_EASY_SETOPT_LOG(curl, CURLOPT_PORT, (long)e.GetPort().m_port);
+        CURL_EASY_SETOPT_LOG(curl, CURLOPT_PORT, (long)e.GetPort());
     }
 
+    // progress calls.
+    if (e.GetOnProgress()) {
+        CURL_EASY_SETOPT_LOG(curl, CURLOPT_XFERINFODATA, &e);
+        CURL_EASY_SETOPT_LOG(curl, CURLOPT_XFERINFOFUNCTION, ProgressCallbackFunc2);
+    } else {
+        CURL_EASY_SETOPT_LOG(curl, CURLOPT_XFERINFOFUNCTION, ProgressCallbackFunc1);
+    }
     CURL_EASY_SETOPT_LOG(curl, CURLOPT_NOPROGRESS, 0L);
 }
 
+auto EscapeString(CURL* curl, const std::string& str) -> std::string {
+    char* s{};
+    if (!curl) {
+        s = curl_escape(str.data(), str.length());
+    } else {
+        s = curl_easy_escape(curl, str.data(), str.length());
+    }
+
+    if (!s) {
+        return str;
+    }
+
+    const std::string result = s;
+    curl_free(s);
+    return result;
+}
+
+auto EncodeUrl(const std::string& url) -> std::string {
+    auto clu = curl_url();
+    R_UNLESS(clu, url);
+    ON_SCOPE_EXIT(curl_url_cleanup(clu));
+
+    CURLUcode clu_code;
+    clu_code = curl_url_set(clu, CURLUPART_URL, url.c_str(), CURLU_URLENCODE);
+    R_UNLESS(clu_code == CURLUE_OK, url);
+
+    char* encoded_url;
+    clu_code = curl_url_get(clu, CURLUPART_URL, &encoded_url, 0);
+    R_UNLESS(clu_code == CURLUE_OK, url);
+
+    const std::string out = encoded_url;
+    curl_free(encoded_url);
+    return out;
+}
+
 auto DownloadInternal(CURL* curl, const Api& e) -> ApiResult {
     // check if stop has been requested before starting download
     if (e.GetToken().stop_requested()) {
@@ -622,15 +713,6 @@ auto DownloadInternal(CURL* curl, const Api& e) -> ApiResult {
         CURL_EASY_SETOPT_LOG(curl, CURLOPT_HTTPHEADER, list);
     }
 
-    // progress calls.
-    if (e.GetOnProgress()) {
-        CURL_EASY_SETOPT_LOG(curl, CURLOPT_XFERINFODATA, &e);
-        CURL_EASY_SETOPT_LOG(curl, CURLOPT_XFERINFOFUNCTION, ProgressCallbackFunc2);
-    } else {
-        CURL_EASY_SETOPT_LOG(curl, CURLOPT_XFERINFOFUNCTION, ProgressCallbackFunc1);
-    }
-    CURL_EASY_SETOPT_LOG(curl, CURLOPT_NOPROGRESS, 0L);
-
     // write calls.
     CURL_EASY_SETOPT_LOG(curl, CURLOPT_WRITEFUNCTION, has_file ? WriteFileCallback : WriteMemoryCallback);
     CURL_EASY_SETOPT_LOG(curl, CURLOPT_WRITEDATA, &chunk);
@@ -699,10 +781,10 @@ auto UploadInternal(CURL* curl, const Api& e) -> ApiResult {
 
     UploadStruct chunk{};
     DataStruct chunk_out{};
+    SeekCustomData seek_data{};
     Header header_in = e.GetHeader();
     Header header_out;
     fs::FsNativeSd fs{};
-    s64 upload_size{};
 
     if (has_file) {
         if (R_FAILED(fs.OpenFile(e.GetPath(), FsOpenMode_Read, &chunk.f))) {
@@ -710,14 +792,14 @@ auto UploadInternal(CURL* curl, const Api& e) -> ApiResult {
             return {};
         }
 
-        fsFileGetSize(&chunk.f, &upload_size);
+        fsFileGetSize(&chunk.f, &chunk.size);
-        log_write("got chunk size: %zd\n", upload_size);
+        log_write("got chunk size: %zd\n", chunk.size);
     } else {
         if (info.m_callback) {
-            upload_size = info.m_size;
+            chunk.size = info.m_size;
-            log_write("setting upload size: %zu\n", upload_size);
+            log_write("setting upload size: %zu\n", chunk.size);
         } else {
-            upload_size = info.m_data.size();
+            chunk.size = info.m_data.size();
             chunk.data = info.m_data;
         }
     }
@@ -737,12 +819,20 @@ auto UploadInternal(CURL* curl, const Api& e) -> ApiResult {
     curl_easy_reset(curl);
     SetCommonCurlOptions(curl, e);
 
-    CURL_EASY_SETOPT_LOG(curl, CURLOPT_URL, url.c_str());
+    // encode url
+    auto clu = curl_url();
+    R_UNLESS(clu, {});
+    ON_SCOPE_EXIT(curl_url_cleanup(clu));
+
+    const auto clu_code = curl_url_set(clu, CURLUPART_URL, url.c_str(), CURLU_URLENCODE);
+    R_UNLESS(clu_code == CURLUE_OK, {});
+
+    CURL_EASY_SETOPT_LOG(curl, CURLOPT_CURLU, clu);
     CURL_EASY_SETOPT_LOG(curl, CURLOPT_HEADERFUNCTION, header_callback);
     CURL_EASY_SETOPT_LOG(curl, CURLOPT_HEADERDATA, &header_out);
 
     CURL_EASY_SETOPT_LOG(curl, CURLOPT_UPLOAD, 1L);
-    CURL_EASY_SETOPT_LOG(curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)upload_size);
+    CURL_EASY_SETOPT_LOG(curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)chunk.size);
 
     // instruct libcurl to create ftp folders if they don't yet exist.
     CURL_EASY_SETOPT_LOG(curl, CURLOPT_FTP_CREATE_MISSING_DIRS, CURLFTP_CREATE_DIR_RETRY);
@@ -776,17 +866,20 @@ auto UploadInternal(CURL* curl, const Api& e) -> ApiResult {
     if (info.m_callback) {
         CURL_EASY_SETOPT_LOG(curl, CURLOPT_READFUNCTION, ReadCustomCallback);
         CURL_EASY_SETOPT_LOG(curl, CURLOPT_READDATA, &info);
+
+        if (e.GetOnUploadSeek()) {
+            seek_data.cb = e.GetOnUploadSeek();
+            seek_data.size = chunk.size;
+            CURL_EASY_SETOPT_LOG(curl, CURLOPT_SEEKFUNCTION, SeekCustomCallback);
+            CURL_EASY_SETOPT_LOG(curl, CURLOPT_SEEKDATA, &seek_data);
+        }
     } else {
         CURL_EASY_SETOPT_LOG(curl, CURLOPT_READFUNCTION, has_file ? ReadFileCallback : ReadMemoryCallback);
         CURL_EASY_SETOPT_LOG(curl, CURLOPT_READDATA, &chunk);
-    }
 
-    // progress calls.
+        // allow for seeking upon uploads, may be used for ftp and http.
-    if (e.GetOnProgress()) {
+        CURL_EASY_SETOPT_LOG(curl, CURLOPT_SEEKFUNCTION, SeekCallback);
-        CURL_EASY_SETOPT_LOG(curl, CURLOPT_XFERINFODATA, &e);
+        CURL_EASY_SETOPT_LOG(curl, CURLOPT_SEEKDATA, &chunk);
-        CURL_EASY_SETOPT_LOG(curl, CURLOPT_XFERINFOFUNCTION, ProgressCallbackFunc2);
-    } else {
-        CURL_EASY_SETOPT_LOG(curl, CURLOPT_XFERINFOFUNCTION, ProgressCallbackFunc1);
     }
 
     // write calls.
@@ -1013,13 +1106,7 @@ auto FromFileAsync(const Api& e) -> bool {
 }
 
 auto EscapeString(const std::string& str) -> std::string {
-    std::string result;
+    return EscapeString(nullptr, str);
-    const auto s = curl_escape(str.data(), str.length());
-    if (s) {
-        result = s;
-        curl_free(s);
-    }
-    return result;
 }
 
 } // namespace sphaira::curl

sphaira/source/ui/menus/game_menu.cpp

@@ -426,6 +426,12 @@ Result DumpNspToNetwork(ProgressBox* pbox, const location::Entry& loc, std::span
                     offset += bytes_read;
                     return bytes_read;
                 }
+            },
+            curl::OnUploadSeek{
+                [&e, &offset](s64 new_offset){
+                    offset = new_offset;
+                    return true;
+                }
             }
         );