From 5daca4354c3d4c19fd79dd4023b85b1feae9c685 Mon Sep 17 00:00:00 2001 From: ITotalJustice <47043333+ITotalJustice@users.noreply.github.com> Date: Mon, 19 May 2025 16:00:03 +0100 Subject: [PATCH] add support for webdav uploads by creating missing folders, game now dumps to title/title[id].nsp --- sphaira/include/download.hpp | 16 ++- sphaira/source/download.cpp | 199 +++++++++++++++++++------- sphaira/source/ui/menus/game_menu.cpp | 2 +- 3 files changed, 165 insertions(+), 52 deletions(-) diff --git a/sphaira/include/download.hpp b/sphaira/include/download.hpp index 97d2599..02ed1a7 100644 --- a/sphaira/include/download.hpp +++ b/sphaira/include/download.hpp @@ -13,10 +13,14 @@ namespace sphaira::curl { enum { Flag_None = 0, + // requests to download send etag in the header. // the received etag is then saved on success. // this api is only available on downloading to file. Flag_Cache = 1 << 0, + + // sets CURLOPT_NOBODY. + Flag_NoBody = 1 << 1, }; enum class Priority { @@ -70,6 +74,12 @@ struct Port { u16 m_port{}; }; +struct CustomRequest { + CustomRequest() = default; + CustomRequest(const std::string& str) : m_str{str} {} + std::string m_str; +}; + struct UserPass { UserPass() = default; UserPass(const std::string& user) : m_user{user} {} @@ -281,6 +291,7 @@ struct Api { auto& GetFlags() const { return m_flags.m_flags; } auto& GetPath() const { return m_path; } auto& GetPort() const { return m_port.m_port; } + auto& GetCustomRequest() const { return m_custom_request.m_str; } auto& GetUserPass() const { return m_userpass; } auto& GetBearer() const { return m_bearer.m_str; } auto& GetPubKey() const { return m_pub_key.m_str; } @@ -292,13 +303,13 @@ struct Api { auto& GetPriority() const { return m_prio; } auto& GetToken() const { return m_stoken; } -private: void SetOption(Url&& v) { m_url = v; } void SetOption(Fields&& v) { m_fields = v; } void SetOption(Header&& v) { m_header = v; } void SetOption(Flags&& v) { m_flags = v; } void SetOption(Path&& v) { m_path = v; } void SetOption(Port&& v) { m_port = v; } + void SetOption(CustomRequest&& v) { m_custom_request = v; } void SetOption(UserPass&& v) { m_userpass = v; } void SetOption(Bearer&& v) { m_bearer = v; } void SetOption(PubKey&& v) { m_pub_key = v; } @@ -322,12 +333,13 @@ private: } private: - Url m_url; + Url m_url{}; Fields m_fields{}; Header m_header{}; Flags m_flags{}; Path m_path{}; Port m_port{}; + CustomRequest m_custom_request{}; UserPass m_userpass{}; Bearer m_bearer{}; PubKey m_pub_key{}; diff --git a/sphaira/source/download.cpp b/sphaira/source/download.cpp index 3e7e595..d2b3b45 100644 --- a/sphaira/source/download.cpp +++ b/sphaira/source/download.cpp @@ -3,12 +3,15 @@ #include "defines.hpp" #include "evman.hpp" #include "fs.hpp" + #include #include #include #include #include #include +#include +#include #include #include @@ -57,6 +60,11 @@ struct SeekCustomData { s64 size{}; }; +// helper for creating webdav folders as libcurl does not have built-in +// support for it. +// only creates the folders if they don't exist. +auto WebdavCreateFolder(CURL* curl, const Api& e) -> bool; + auto generate_key_from_path(const fs::FsPath& path) -> std::string { const auto key = crc32Calculate(path.s, path.size()); return std::to_string(key); @@ -545,12 +553,59 @@ auto header_callback(char* b, size_t size, size_t nitems, void* userdata) -> siz return numbytes; } +auto EscapeString(CURL* curl, const std::string& str) -> std::string { + char* s{}; + if (!curl) { + s = curl_escape(str.data(), str.length()); + } else { + s = curl_easy_escape(curl, str.data(), str.length()); + } + + if (!s) { + return str; + } + + const std::string result = s; + curl_free(s); + return result; +} + +auto EncodeUrl(std::string url) -> std::string { + log_write("[CURL] encoding url\n"); + + if (url.starts_with("webdav://")) { + log_write("[CURL] updating host\n"); + url.replace(0, std::strlen("webdav"), "https"); + log_write("[CURL] updated host: %s\n", url.c_str()); + } + + auto clu = curl_url(); + R_UNLESS(clu, url); + ON_SCOPE_EXIT(curl_url_cleanup(clu)); + + log_write("[CURL] setting url\n"); + CURLUcode clu_code; + clu_code = curl_url_set(clu, CURLUPART_URL, url.c_str(), CURLU_URLENCODE); + R_UNLESS(clu_code == CURLUE_OK, url); + log_write("[CURL] set url success\n"); + + char* encoded_url; + clu_code = curl_url_get(clu, CURLUPART_URL, &encoded_url, 0); + R_UNLESS(clu_code == CURLUE_OK, url); + + log_write("[CURL] encoded url: %s [vs]: %s\n", encoded_url, url.c_str()); + const std::string out = encoded_url; + curl_free(encoded_url); + return out; +} + void SetCommonCurlOptions(CURL* curl, const Api& e) { CURL_EASY_SETOPT_LOG(curl, CURLOPT_USERAGENT, API_AGENT); CURL_EASY_SETOPT_LOG(curl, CURLOPT_FOLLOWLOCATION, 1L); CURL_EASY_SETOPT_LOG(curl, CURLOPT_SSL_VERIFYPEER, 0L); CURL_EASY_SETOPT_LOG(curl, CURLOPT_SSL_VERIFYHOST, 0L); CURL_EASY_SETOPT_LOG(curl, CURLOPT_FAILONERROR, 1L); + CURL_EASY_SETOPT_LOG(curl, CURLOPT_NOPROGRESS, 0L); CURL_EASY_SETOPT_LOG(curl, CURLOPT_SHARE, g_curl_share); CURL_EASY_SETOPT_LOG(curl, CURLOPT_BUFFERSIZE, 1024*512); CURL_EASY_SETOPT_LOG(curl, CURLOPT_UPLOAD_BUFFERSIZE, 1024*512); @@ -567,6 +622,17 @@ void SetCommonCurlOptions(CURL* curl, const Api& e) { // enable TE is server supports it. CURL_EASY_SETOPT_LOG(curl, CURLOPT_TRANSFER_ENCODING, 1L); + // set flags. + if (e.GetFlags() & Flag_NoBody) { + CURL_EASY_SETOPT_LOG(curl, CURLOPT_NOBODY, 1L); + } + + // set custom request. + if (!e.GetCustomRequest().empty()) { + log_write("[CURL] setting custom request: %s\n", e.GetCustomRequest().c_str()); + CURL_EASY_SETOPT_LOG(curl, CURLOPT_CUSTOMREQUEST, e.GetCustomRequest().c_str()); + } + // set oath2 bearer. if (!e.GetBearer().empty()) { CURL_EASY_SETOPT_LOG(curl, CURLOPT_XOAUTH2_BEARER, e.GetBearer().c_str()); @@ -600,44 +666,8 @@ void SetCommonCurlOptions(CURL* curl, const Api& e) { } else { CURL_EASY_SETOPT_LOG(curl, CURLOPT_XFERINFOFUNCTION, ProgressCallbackFunc1); } - CURL_EASY_SETOPT_LOG(curl, CURLOPT_NOPROGRESS, 0L); + } - -auto EscapeString(CURL* curl, const std::string& str) -> std::string { - char* s{}; - if (!curl) { - s = curl_escape(str.data(), str.length()); - } else { - s = curl_easy_escape(curl, str.data(), str.length()); - } - - if (!s) { - return str; - } - - const std::string result = s; - curl_free(s); - return result; -} - -auto EncodeUrl(const std::string& url) -> std::string { - auto clu = curl_url(); - R_UNLESS(clu, url); - ON_SCOPE_EXIT(curl_url_cleanup(clu)); - - CURLUcode clu_code; - clu_code = curl_url_set(clu, CURLUPART_URL, url.c_str(), CURLU_URLENCODE); - R_UNLESS(clu_code == CURLUE_OK, url); - - char* encoded_url; - clu_code = curl_url_get(clu, CURLUPART_URL, &encoded_url, 0); - R_UNLESS(clu_code == CURLUE_OK, url); - - const std::string out = encoded_url; - curl_free(encoded_url); - return out; -} - auto DownloadInternal(CURL* curl, const Api& e) -> ApiResult { // check if stop has been requested before starting download if (e.GetToken().stop_requested()) { @@ -647,6 +677,7 @@ auto DownloadInternal(CURL* curl, const Api& e) -> ApiResult { fs::FsPath tmp_buf; const bool has_file = !e.GetPath().empty() && e.GetPath() != ""; const bool has_post = !e.GetFields().empty() && e.GetFields() != ""; + const auto encoded_url = EncodeUrl(e.GetUrl()); DataStruct chunk; Header header_in = e.GetHeader(); @@ -679,7 +710,7 @@ auto DownloadInternal(CURL* curl, const Api& e) -> ApiResult { curl_easy_reset(curl); SetCommonCurlOptions(curl, e); - CURL_EASY_SETOPT_LOG(curl, CURLOPT_URL, e.GetUrl().c_str()); + CURL_EASY_SETOPT_LOG(curl, CURLOPT_URL, encoded_url.c_str()); CURL_EASY_SETOPT_LOG(curl, CURLOPT_HEADERFUNCTION, header_callback); CURL_EASY_SETOPT_LOG(curl, CURLOPT_HEADERDATA, &header_out); @@ -765,7 +796,7 @@ auto DownloadInternal(CURL* curl, const Api& e) -> ApiResult { } } - log_write("Downloaded %s %s\n", e.GetUrl().c_str(), curl_easy_strerror(res)); + log_write("Downloaded %s code: %ld %s\n", e.GetUrl().c_str(), http_code, curl_easy_strerror(res)); return {success, http_code, header_out, chunk.data, e.GetPath()}; } @@ -775,8 +806,16 @@ auto UploadInternal(CURL* curl, const Api& e) -> ApiResult { return {}; } + if (e.GetUrl().starts_with("webdav://")) { + if (!WebdavCreateFolder(curl, e)) { + log_write("[CURL] failed to create webdav folder, aborting\n"); + return {}; + } + } + const auto& info = e.GetUploadInfo(); const auto url = e.GetUrl() + "/" + info.m_name; + const auto encoded_url = EncodeUrl(url); const bool has_file = !e.GetPath().empty() && e.GetPath() != ""; UploadStruct chunk{}; @@ -819,15 +858,7 @@ auto UploadInternal(CURL* curl, const Api& e) -> ApiResult { curl_easy_reset(curl); SetCommonCurlOptions(curl, e); - // encode url - auto clu = curl_url(); - R_UNLESS(clu, {}); - ON_SCOPE_EXIT(curl_url_cleanup(clu)); - - const auto clu_code = curl_url_set(clu, CURLUPART_URL, url.c_str(), CURLU_URLENCODE); - R_UNLESS(clu_code == CURLUE_OK, {}); - - CURL_EASY_SETOPT_LOG(curl, CURLOPT_CURLU, clu); + CURL_EASY_SETOPT_LOG(curl, CURLOPT_URL, encoded_url.c_str()); CURL_EASY_SETOPT_LOG(curl, CURLOPT_HEADERFUNCTION, header_callback); CURL_EASY_SETOPT_LOG(curl, CURLOPT_HEADERDATA, &header_out); @@ -897,10 +928,80 @@ auto UploadInternal(CURL* curl, const Api& e) -> ApiResult { fsFileClose(&chunk.f); } - log_write("Uploaded %s %s\n", url.c_str(), curl_easy_strerror(res)); + log_write("Uploaded %s code: %ld %s\n", url.c_str(), http_code, curl_easy_strerror(res)); return {success, http_code, header_out, chunk_out.data}; } +auto WebdavCreateFolder(CURL* curl, const Api& e) -> bool { + // if using webdav, extract the file path and create the directories. + // https://github.com/WebDAVDevs/webdav-request-samples/blob/master/webdav_curl.md + if (e.GetUrl().starts_with("webdav://")) { + log_write("[CURL] found webdav url\n"); + + const auto info = e.GetUploadInfo(); + if (info.m_name.empty()) { + return true; + } + + const auto& file_path = info.m_name; + log_write("got file path: %s\n", file_path.c_str()); + + const auto file_loc = file_path.find_last_of('/'); + if (file_loc == file_path.npos) { + log_write("failed to find last slash\n"); + return true; + } + + const auto path_view = file_path.substr(0, file_loc); + log_write("got folder path: %s\n", path_view.c_str()); + + auto e2 = e; + e2.SetOption(Path{}); + e2.SetOption(Url{e.GetUrl() + "/" + path_view}); + e2.SetOption(Flags{e.GetFlags() | Flag_NoBody}); + e2.SetOption(CustomRequest{"PROPFIND"}); + e2.SetOption(Header{ + { "Depth", "0" }, + }); + + // test to see if the directory exists first. + const auto exist_result = DownloadInternal(curl, e2); + if (exist_result.success) { + log_write("[CURL] folder already exist: %s\n", path_view.c_str()); + return true; + } else { + log_write("[CURL] folder does NOT exist, manually creating: %s\n", path_view.c_str()); + } + + // make the request to create the folder. + std::string folder; + for (const auto dir : std::views::split(path_view, '/')) { + if (dir.empty()) { + continue; + } + + folder += "/" + std::string{dir.data(), dir.size()}; + e2.SetOption(Url{e.GetUrl() + folder}); + e2.SetOption(Header{}); + e2.SetOption(CustomRequest{"MKCOL"}); + + const auto result = DownloadInternal(curl, e2); + if (result.code == 201) { + log_write("[CURL] created webdav directory\n"); + } else if (result.code == 405) { + log_write("[CURL] webdav directory already exists: %ld\n", result.code); + } else { + log_write("[CURL] failed to create webdav directory: %ld\n", result.code); + return false; + } + } + } else { + log_write("[CURL] not a webdav url: %s\n", e.GetUrl().c_str()); + } + + return true; +} + void my_lock(CURL *handle, curl_lock_data data, curl_lock_access laccess, void *useptr) { mutexLock(&g_mutex_share[data]); } diff --git a/sphaira/source/ui/menus/game_menu.cpp b/sphaira/source/ui/menus/game_menu.cpp index 3f3679c..c4d6294 100644 --- a/sphaira/source/ui/menus/game_menu.cpp +++ b/sphaira/source/ui/menus/game_menu.cpp @@ -664,7 +664,7 @@ auto BuildNspPath(const Entry& e, const NsApplicationContentMetaStatus& status) } fs::FsPath path; - std::snprintf(path, sizeof(path), "%s %s[%016lX][v%u][%s].nsp", name_buf.s, version, status.application_id, status.version, ncm::GetMetaTypeShortStr(status.meta_type)); + std::snprintf(path, sizeof(path), "%s/%s %s[%016lX][v%u][%s].nsp", name_buf.s, name_buf.s, version, status.application_id, status.version, ncm::GetMetaTypeShortStr(status.meta_type)); return path; }