From 71df5317be23653f8d7fdbf863c1ba7c020b69c1 Mon Sep 17 00:00:00 2001 From: ITotalJustice <47043333+ITotalJustice@users.noreply.github.com> Date: Sun, 18 May 2025 20:30:04 +0100 Subject: [PATCH] add game dump uploading, fix download progress using u32 instead of s64, add progress and title for usb game dump. - added support for custom upload locations, set in /config/sphaira/locations.ini - add support for various auth options for download/upload (port, pub/priv key, user/pass, bearer). --- sphaira/CMakeLists.txt | 1 + sphaira/include/download.hpp | 161 +++++++++++++- sphaira/include/location.hpp | 24 +++ sphaira/include/ui/progress_box.hpp | 12 +- sphaira/source/download.cpp | 292 ++++++++++++++++++++++++-- sphaira/source/location.cpp | 75 +++++++ sphaira/source/ui/menus/game_menu.cpp | 110 +++++++++- 7 files changed, 641 insertions(+), 34 deletions(-) create mode 100644 sphaira/include/location.hpp create mode 100644 sphaira/source/location.cpp diff --git a/sphaira/CMakeLists.txt b/sphaira/CMakeLists.txt index de3319b..da245c2 100644 --- a/sphaira/CMakeLists.txt +++ b/sphaira/CMakeLists.txt @@ -70,6 +70,7 @@ add_executable(sphaira source/evman.cpp source/fs.cpp source/image.cpp + source/location.cpp source/log.cpp source/main.cpp source/nro.cpp diff --git a/sphaira/include/download.hpp b/sphaira/include/download.hpp index 4544ee3..2138940 100644 --- a/sphaira/include/download.hpp +++ b/sphaira/include/download.hpp @@ -29,7 +29,8 @@ struct ApiResult; using Path = fs::FsPath; using OnComplete = std::function; -using OnProgress = std::function; +using OnProgress = std::function; +using OnUploadCallback = std::function; using StopToken = std::stop_token; struct Url { @@ -62,6 +63,48 @@ struct Flags { u32 m_flags{Flag_None}; }; +struct Port { + Port() = default; + Port(u16 port) : m_port{port} {} + u16 m_port{}; +}; + +struct UserPass { + UserPass() = default; + UserPass(const std::string& user) : m_user{user} {} + UserPass(const std::string& user, const std::string& pass) : m_user{user}, m_pass{pass} {} + std::string m_user; + std::string m_pass; +}; + +struct UploadInfo { + UploadInfo() = default; + UploadInfo(const std::string& name, s64 size, OnUploadCallback cb) : m_name{name}, m_size{size}, m_callback{cb} {} + UploadInfo(const std::string& name, const std::vector& data) : m_name{name}, m_data{data} {} + std::string m_name{}; + std::vector m_data{}; + s64 m_size{}; + OnUploadCallback m_callback{}; +}; + +struct Bearer { + Bearer() = default; + Bearer(const std::string& str) : m_str{str} {} + std::string m_str; +}; + +struct PubKey { + PubKey() = default; + PubKey(const std::string& str) : m_str{str} {} + std::string m_str; +}; + +struct PrivKey { + PrivKey() = default; + PrivKey(const std::string& str) : m_str{str} {} + std::string m_str; +}; + struct ApiResult { bool success; long code; @@ -82,10 +125,14 @@ void Exit(); // sync functions auto ToMemory(const Api& e) -> ApiResult; auto ToFile(const Api& e) -> ApiResult; +auto FromMemory(const Api& e) -> ApiResult; +auto FromFile(const Api& e) -> ApiResult; // async functions auto ToMemoryAsync(const Api& e) -> bool; auto ToFileAsync(const Api& e) -> bool; +auto FromMemoryAsync(const Api& e) -> bool; +auto FromFileAsync(const Api& e) -> bool; // uses curl to convert string to their %XX auto EscapeString(const std::string& str) -> std::string; @@ -107,6 +154,15 @@ struct Api { } } + template + auto From(Ts&&... ts) { + if constexpr(std::disjunction_v...>) { + return FromFile(std::forward(ts)...); + } else { + return FromMemory(std::forward(ts)...); + } + } + template auto ToAsync(Ts&&... ts) { if constexpr(std::disjunction_v...>) { @@ -116,6 +172,15 @@ struct Api { } } + template + auto FromAsync(Ts&&... ts) { + if constexpr(std::disjunction_v...>) { + return FromFileAsync(std::forward(ts)...); + } else { + return FromMemoryAsync(std::forward(ts)...); + } + } + template auto ToMemory(Ts&&... ts) { static_assert(std::disjunction_v...>, "Url must be specified"); @@ -125,6 +190,16 @@ struct Api { return curl::ToMemory(*this); } + template + auto FromMemory(Ts&&... ts) { + static_assert(std::disjunction_v...>, "Url must be specified"); + static_assert(std::disjunction_v...>, "UploadInfo must be specified"); + static_assert(!std::disjunction_v...>, "Path must not valid for memory"); + static_assert(!std::disjunction_v...>, "OnComplete must not be specified"); + Api::set_option(std::forward(ts)...); + return curl::FromMemory(*this); + } + template auto ToFile(Ts&&... ts) { static_assert(std::disjunction_v...>, "Url must be specified"); @@ -134,6 +209,15 @@ struct Api { return curl::ToFile(*this); } + template + auto FromFile(Ts&&... ts) { + static_assert(std::disjunction_v...>, "Url must be specified"); + static_assert(std::disjunction_v...>, "Path must be specified"); + static_assert(!std::disjunction_v...>, "OnComplete must not be specified"); + Api::set_option(std::forward(ts)...); + return curl::FromFile(*this); + } + template auto ToMemoryAsync(Ts&&... ts) { static_assert(std::disjunction_v...>, "Url must be specified"); @@ -144,6 +228,17 @@ struct Api { return curl::ToMemoryAsync(*this); } + template + auto FromMemoryAsync(Ts&&... ts) { + static_assert(std::disjunction_v...>, "Url must be specified"); + static_assert(std::disjunction_v...>, "UploadInfo must be specified"); + static_assert(std::disjunction_v...>, "OnComplete must be specified"); + static_assert(!std::disjunction_v...>, "Path must not valid for memory"); + static_assert(std::disjunction_v...>, "StopToken must be specified"); + Api::set_option(std::forward(ts)...); + return curl::FromMemoryAsync(*this); + } + template auto ToFileAsync(Ts&&... ts) { static_assert(std::disjunction_v...>, "Url must be specified"); @@ -154,6 +249,23 @@ struct Api { return curl::ToFileAsync(*this); } + template + auto FromFileAsync(Ts&&... ts) { + static_assert(std::disjunction_v...>, "Url must be specified"); + static_assert(std::disjunction_v...>, "Path must be specified"); + static_assert(std::disjunction_v...>, "OnComplete must be specified"); + static_assert(std::disjunction_v...>, "StopToken must be specified"); + Api::set_option(std::forward(ts)...); + return curl::FromFileAsync(*this); + } + + void SetUpload(bool enable) { + m_is_upload = enable; + } + + auto IsUpload() const { + return m_is_upload; + } auto& GetUrl() const { return m_url.m_str; } @@ -169,6 +281,24 @@ struct Api { auto& GetPath() const { return m_path; } + auto& GetPort() const { + return m_port; + } + auto& GetUserPass() const { + return m_userpass; + } + auto& GetBearer() const { + return m_bearer; + } + auto& GetPubKey() const { + return m_pub_key; + } + auto& GetPrivKey() const { + return m_priv_key; + } + auto& GetUploadInfo() const { + return m_info; + } auto& GetOnComplete() const { return m_on_complete; } @@ -198,6 +328,24 @@ private: void SetOption(Path&& v) { m_path = v; } + void SetOption(Port&& v) { + m_port = v; + } + void SetOption(UserPass&& v) { + m_userpass = v; + } + void SetOption(Bearer&& v) { + m_bearer = v; + } + void SetOption(PubKey&& v) { + m_pub_key = v; + } + void SetOption(PrivKey&& v) { + m_priv_key = v; + } + void SetOption(UploadInfo&& v) { + m_info = v; + } void SetOption(OnComplete&& v) { m_on_complete = v; } @@ -228,11 +376,18 @@ private: Header m_header{}; Flags m_flags{}; Path m_path{}; - OnComplete m_on_complete{nullptr}; - OnProgress m_on_progress{nullptr}; + Port m_port{}; + UserPass m_userpass{}; + Bearer m_bearer{}; + PubKey m_pub_key{}; + PrivKey m_priv_key{}; + UploadInfo m_info{}; + OnComplete m_on_complete{}; + OnProgress m_on_progress{}; Priority m_prio{Priority::High}; std::stop_source m_stop_source{}; StopToken m_stoken{m_stop_source.get_token()}; + bool m_is_upload{}; }; } // namespace sphaira::curl diff --git a/sphaira/include/location.hpp b/sphaira/include/location.hpp new file mode 100644 index 0000000..5ccb28c --- /dev/null +++ b/sphaira/include/location.hpp @@ -0,0 +1,24 @@ +#pragma once + +#include +#include +#include + +namespace sphaira::location { + +struct Entry { + std::string name{}; + std::string url{}; + std::string user{}; + std::string pass{}; + std::string bearer{}; + std::string pub_key{}; + std::string priv_key{}; + u16 port{}; +}; +using Entries = std::vector; + +auto Load() -> Entries; +void Add(const Entry& e); + +} // namespace sphaira::location diff --git a/sphaira/include/ui/progress_box.hpp b/sphaira/include/ui/progress_box.hpp index bc8fc8a..b329cce 100644 --- a/sphaira/include/ui/progress_box.hpp +++ b/sphaira/include/ui/progress_box.hpp @@ -17,7 +17,7 @@ struct ProgressBox final : Widget { const std::string& action, const std::string& title, ProgressBoxCallback callback, ProgressBoxDoneCallback done = [](bool success){}, - int cpuid = 1, int prio = 0x2C, int stack_size = 1024*1024 + int cpuid = 1, int prio = 0x2C, int stack_size = 1024*128 ); ~ProgressBox(); @@ -38,11 +38,17 @@ struct ProgressBox final : Widget { void Yield(); auto OnDownloadProgressCallback() { - return [this](u32 dltotal, u32 dlnow, u32 ultotal, u32 ulnow){ + return [this](s64 dltotal, s64 dlnow, s64 ultotal, s64 ulnow){ if (this->ShouldExit()) { return false; } - this->UpdateTransfer(dlnow, dltotal); + + if (dltotal) { + this->UpdateTransfer(dlnow, dltotal); + } else { + this->UpdateTransfer(ulnow, ultotal); + } + return true; }; } diff --git a/sphaira/source/download.cpp b/sphaira/source/download.cpp index 61c15c1..56cb273 100644 --- a/sphaira/source/download.cpp +++ b/sphaira/source/download.cpp @@ -25,7 +25,7 @@ namespace { log_write("curl_share_setopt(%s, %s) msg: %s\n", #opt, #v, curl_share_strerror(r)); \ } \ -constexpr auto API_AGENT = "ITotalJustice"; +constexpr auto API_AGENT = "TotalJustice"; constexpr u64 CHUNK_SIZE = 1024*1024; constexpr auto MAX_THREADS = 4; constexpr int THREAD_PRIO = 0x2C; @@ -35,6 +35,12 @@ std::atomic_bool g_running{}; CURLSH* g_curl_share{}; Mutex g_mutex_share[CURL_LOCK_DATA_LAST]{}; +struct UploadStruct { + std::span data; + s64 offset{}; + FsFile f{}; +}; + struct DataStruct { std::vector data; s64 offset{}; @@ -302,7 +308,7 @@ struct ThreadQueue { threadClose(&m_thread); } - auto Add(const Api& api) -> bool { + auto Add(const Api& api, bool is_upload = false) -> bool { if (api.GetUrl().empty() || api.GetPath().empty() || !api.GetOnComplete()) { return false; } @@ -312,10 +318,10 @@ struct ThreadQueue { switch (api.GetPriority()) { case Priority::Normal: - m_entries.emplace_back(api); + m_entries.emplace_back(api).api.SetUpload(is_upload); break; case Priority::High: - m_entries.emplace_front(api); + m_entries.emplace_front(api).api.SetUpload(is_upload); break; } @@ -366,6 +372,54 @@ auto ProgressCallbackFunc2(void *clientp, curl_off_t dltotal, curl_off_t dlnow, return 0; } +auto ReadFileCallback(char *ptr, size_t size, size_t nmemb, void *userp) -> size_t { + if (!g_running) { + return 0; + } + + auto data_struct = static_cast(userp); + const auto realsize = size * nmemb; + + u64 bytes_read; + if (R_FAILED(fsFileRead(&data_struct->f, data_struct->offset, ptr, realsize, FsReadOption_None, &bytes_read))) { + log_write("reading file error\n"); + return 0; + } + + data_struct->offset += bytes_read; + svcSleepThread(YieldType_WithoutCoreMigration); + return bytes_read; +} + +auto ReadMemoryCallback(char *ptr, size_t size, size_t nmemb, void *userp) -> size_t { + if (!g_running) { + return 0; + } + + auto data_struct = static_cast(userp); + auto realsize = size * nmemb; + realsize = std::min(realsize, data_struct->data.size() - data_struct->offset); + + std::memcpy(ptr, data_struct->data.data(), realsize); + data_struct->offset += realsize; + + svcSleepThread(YieldType_WithoutCoreMigration); + return realsize; +} + +auto ReadCustomCallback(char *ptr, size_t size, size_t nmemb, void *userp) -> size_t { + if (!g_running) { + return 0; + } + + auto data_struct = static_cast(userp); + auto realsize = size * nmemb; + const auto result = data_struct->m_callback(ptr, realsize); + + svcSleepThread(YieldType_WithoutCoreMigration); + return result; +} + auto WriteMemoryCallback(void *contents, size_t size, size_t num_files, void *userp) -> size_t { if (!g_running) { return 0; @@ -381,11 +435,9 @@ auto WriteMemoryCallback(void *contents, size_t size, size_t num_files, void *us data_struct->data.resize(data_struct->offset + realsize); std::memcpy(data_struct->data.data() + data_struct->offset, contents, realsize); - data_struct->offset += realsize; svcSleepThread(YieldType_WithoutCoreMigration); - return realsize; } @@ -444,6 +496,54 @@ auto header_callback(char* b, size_t size, size_t nitems, void* userdata) -> siz return numbytes; } +void SetCommonCurlOptions(CURL* curl, const Api& e) { + CURL_EASY_SETOPT_LOG(curl, CURLOPT_USERAGENT, API_AGENT); + CURL_EASY_SETOPT_LOG(curl, CURLOPT_FOLLOWLOCATION, 1L); + CURL_EASY_SETOPT_LOG(curl, CURLOPT_SSL_VERIFYPEER, 0L); + CURL_EASY_SETOPT_LOG(curl, CURLOPT_SSL_VERIFYHOST, 0L); + CURL_EASY_SETOPT_LOG(curl, CURLOPT_FAILONERROR, 1L); + CURL_EASY_SETOPT_LOG(curl, CURLOPT_SHARE, g_curl_share); + CURL_EASY_SETOPT_LOG(curl, CURLOPT_BUFFERSIZE, 1024*512); + CURL_EASY_SETOPT_LOG(curl, CURLOPT_UPLOAD_BUFFERSIZE, 1024*512); + + // enable all forms of compression supported by libcurl. + CURL_EASY_SETOPT_LOG(curl, CURLOPT_ACCEPT_ENCODING, ""); + + // for smb / ftp, try and use ssl if possible. + CURL_EASY_SETOPT_LOG(curl, CURLOPT_USE_SSL, (long)CURLUSESSL_TRY); + + // in most cases, this will use CURLAUTH_BASIC. + CURL_EASY_SETOPT_LOG(curl, CURLOPT_HTTPAUTH, (long)CURLAUTH_ANY); + + // set oath2 bearer. + if (!e.GetBearer().m_str.empty()) { + CURL_EASY_SETOPT_LOG(curl, CURLOPT_XOAUTH2_BEARER, e.GetBearer().m_str.c_str()); + } + + // set ssh pub/priv key file. + if (!e.GetPubKey().m_str.empty()) { + CURL_EASY_SETOPT_LOG(curl, CURLOPT_SSH_PUBLIC_KEYFILE, e.GetPubKey().m_str.c_str()); + } + if (!e.GetPrivKey().m_str.empty()) { + CURL_EASY_SETOPT_LOG(curl, CURLOPT_SSH_PRIVATE_KEYFILE, e.GetPrivKey().m_str.c_str()); + } + + // set auth. + if (!e.GetUserPass().m_user.empty()) { + CURL_EASY_SETOPT_LOG(curl, CURLOPT_USERPWD, e.GetUserPass().m_user.c_str()); + } + if (!e.GetUserPass().m_pass.empty()) { + CURL_EASY_SETOPT_LOG(curl, CURLOPT_PASSWORD, e.GetUserPass().m_pass.c_str()); + } + + // set port, if valid. + if (e.GetPort().m_port) { + CURL_EASY_SETOPT_LOG(curl, CURLOPT_PORT, (long)e.GetPort().m_port); + } + + CURL_EASY_SETOPT_LOG(curl, CURLOPT_NOPROGRESS, 0L); +} + auto DownloadInternal(CURL* curl, const Api& e) -> ApiResult { // check if stop has been requested before starting download if (e.GetToken().stop_requested()) { @@ -483,18 +583,11 @@ auto DownloadInternal(CURL* curl, const Api& e) -> ApiResult { chunk.data.reserve(CHUNK_SIZE); curl_easy_reset(curl); + SetCommonCurlOptions(curl, e); + CURL_EASY_SETOPT_LOG(curl, CURLOPT_URL, e.GetUrl().c_str()); - CURL_EASY_SETOPT_LOG(curl, CURLOPT_USERAGENT, "TotalJustice"); - CURL_EASY_SETOPT_LOG(curl, CURLOPT_FOLLOWLOCATION, 1L); - CURL_EASY_SETOPT_LOG(curl, CURLOPT_SSL_VERIFYPEER, 0L); - CURL_EASY_SETOPT_LOG(curl, CURLOPT_SSL_VERIFYHOST, 0L); - CURL_EASY_SETOPT_LOG(curl, CURLOPT_FAILONERROR, 1L); - CURL_EASY_SETOPT_LOG(curl, CURLOPT_SHARE, g_curl_share); - CURL_EASY_SETOPT_LOG(curl, CURLOPT_BUFFERSIZE, 1024*512); CURL_EASY_SETOPT_LOG(curl, CURLOPT_HEADERFUNCTION, header_callback); CURL_EASY_SETOPT_LOG(curl, CURLOPT_HEADERDATA, &header_out); - // enable all forms of compression supported by libcurl. - CURL_EASY_SETOPT_LOG(curl, CURLOPT_ACCEPT_ENCODING, ""); if (has_post) { CURL_EASY_SETOPT_LOG(curl, CURLOPT_POSTFIELDS, e.GetFields().c_str()); @@ -591,6 +684,129 @@ auto DownloadInternal(CURL* curl, const Api& e) -> ApiResult { return {success, http_code, header_out, chunk.data, e.GetPath()}; } +auto UploadInternal(CURL* curl, const Api& e) -> ApiResult { + // check if stop has been requested before starting download + if (e.GetToken().stop_requested()) { + return {}; + } + + auto url = e.GetUrl(); + const auto& info = e.GetUploadInfo(); + const bool has_file = !e.GetPath().empty() && e.GetPath() != ""; + + UploadStruct chunk{}; + DataStruct chunk_out{}; + Header header_in = e.GetHeader(); + Header header_out; + fs::FsNativeSd fs{}; + s64 upload_size{}; + + if (has_file) { + if (R_FAILED(fs.OpenFile(e.GetPath(), FsOpenMode_Read, &chunk.f))) { + log_write("failed to open file: %s\n", e.GetPath().s); + return {}; + } + + fsFileGetSize(&chunk.f, &upload_size); + log_write("got chunk size: %zd\n", upload_size); + } else { + if (info.m_callback) { + upload_size = info.m_size; + log_write("setting upload size: %zu\n", upload_size); + } else { + upload_size = info.m_data.size(); + chunk.data = info.m_data; + } + + url += "/" + info.m_name; + } + + if (url.starts_with("file://")) { + const auto folder_path = fs::AppendPath("/", url.substr(std::strlen("file://"))); + log_write("creating local folder: %s\n", folder_path.s); + // create the folder as libcurl doesn't seem to manually create it. + fs.CreateDirectoryRecursivelyWithPath(folder_path); + // remove the path so that libcurl can upload over it. + fs.DeleteFile(folder_path); + } + + // reserve the first chunk + chunk_out.data.reserve(CHUNK_SIZE); + + curl_easy_reset(curl); + SetCommonCurlOptions(curl, e); + + CURL_EASY_SETOPT_LOG(curl, CURLOPT_URL, url.c_str()); + CURL_EASY_SETOPT_LOG(curl, CURLOPT_HEADERFUNCTION, header_callback); + CURL_EASY_SETOPT_LOG(curl, CURLOPT_HEADERDATA, &header_out); + + CURL_EASY_SETOPT_LOG(curl, CURLOPT_UPLOAD, 1L); + CURL_EASY_SETOPT_LOG(curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)upload_size); + + // instruct libcurl to create ftp folders if they don't yet exist. + CURL_EASY_SETOPT_LOG(curl, CURLOPT_FTP_CREATE_MISSING_DIRS, CURLFTP_CREATE_DIR_RETRY); + + struct curl_slist* list = NULL; + ON_SCOPE_EXIT(if (list) { curl_slist_free_all(list); } ); + + for (const auto& [key, value] : header_in.m_map) { + if (value.empty()) { + continue; + } + + // create header key value pair. + const auto header_str = key + ": " + value; + + // try to append header chunk. + auto temp = curl_slist_append(list, header_str.c_str()); + if (temp) { + log_write("adding header: %s\n", header_str.c_str()); + list = temp; + } else { + log_write("failed to append header\n"); + } + } + + if (list) { + CURL_EASY_SETOPT_LOG(curl, CURLOPT_HTTPHEADER, list); + } + + // set callback for reading more data. + if (info.m_callback) { + CURL_EASY_SETOPT_LOG(curl, CURLOPT_READFUNCTION, ReadCustomCallback); + CURL_EASY_SETOPT_LOG(curl, CURLOPT_READDATA, &info); + } else { + CURL_EASY_SETOPT_LOG(curl, CURLOPT_READFUNCTION, has_file ? ReadFileCallback : ReadMemoryCallback); + CURL_EASY_SETOPT_LOG(curl, CURLOPT_READDATA, &chunk); + } + + // progress calls. + if (e.GetOnProgress()) { + CURL_EASY_SETOPT_LOG(curl, CURLOPT_XFERINFODATA, &e); + CURL_EASY_SETOPT_LOG(curl, CURLOPT_XFERINFOFUNCTION, ProgressCallbackFunc2); + } else { + CURL_EASY_SETOPT_LOG(curl, CURLOPT_XFERINFOFUNCTION, ProgressCallbackFunc1); + } + + // write calls. + CURL_EASY_SETOPT_LOG(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback); + CURL_EASY_SETOPT_LOG(curl, CURLOPT_WRITEDATA, &chunk_out); + + // perform upload and cleanup after and report the result. + const auto res = curl_easy_perform(curl); + bool success = res == CURLE_OK; + + long http_code = 0; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); + + if (has_file) { + fsFileClose(&chunk.f); + } + + log_write("Uploaded %s %s\n", url.c_str(), curl_easy_strerror(res)); + return {success, http_code, header_out, chunk_out.data}; +} + auto DownloadInternal(const Api& e) -> ApiResult { auto curl = curl_easy_init(); if (!curl) { @@ -601,6 +817,16 @@ auto DownloadInternal(const Api& e) -> ApiResult { return DownloadInternal(curl, e); } +auto UploadInternal(const Api& e) -> ApiResult { + auto curl = curl_easy_init(); + if (!curl) { + log_write("curl init failed\n"); + return {}; + } + ON_SCOPE_EXIT(curl_easy_cleanup(curl)); + return UploadInternal(curl, e); +} + void my_lock(CURL *handle, curl_lock_data data, curl_lock_access laccess, void *useptr) { mutexLock(&g_mutex_share[data]); } @@ -622,10 +848,18 @@ void ThreadEntry::ThreadFunc(void* p) { continue; } - const auto result = DownloadInternal(data->m_curl, data->m_api); + // ApiResult result; + // if (data->m_api.IsUpload()) { + // result = UploadInternal(data->m_curl, data->m_api); + // } else { + // result = DownloadInternal(data->m_curl, data->m_api); + // } + const auto result = data->m_api.IsUpload() ? UploadInternal(data->m_curl, data->m_api) : DownloadInternal(data->m_curl, data->m_api); if (g_running && data->m_api.GetOnComplete() && !data->m_api.GetToken().stop_requested()) { - const DownloadEventData event_data{data->m_api.GetOnComplete(), result, data->m_api.GetToken()}; - evman::push(std::move(event_data), false); + evman::push( + DownloadEventData{data->m_api.GetOnComplete(), result, data->m_api.GetToken()}, + false + ); } data->m_in_progress = false; @@ -763,6 +997,20 @@ auto ToFile(const Api& e) -> ApiResult { return DownloadInternal(e); } +auto FromMemory(const Api& e) -> ApiResult { + if (!e.GetPath().empty()) { + return {}; + } + return UploadInternal(e); +} + +auto FromFile(const Api& e) -> ApiResult { + if (e.GetPath().empty()) { + return {}; + } + return UploadInternal(e); +} + auto ToMemoryAsync(const Api& api) -> bool { return g_thread_queue.Add(api); } @@ -771,6 +1019,14 @@ auto ToFileAsync(const Api& e) -> bool { return g_thread_queue.Add(e); } +auto FromMemoryAsync(const Api& api) -> bool { + return g_thread_queue.Add(api, true); +} + +auto FromFileAsync(const Api& e) -> bool { + return g_thread_queue.Add(e, true); +} + auto EscapeString(const std::string& str) -> std::string { std::string result; const auto s = curl_escape(str.data(), str.length()); diff --git a/sphaira/source/location.cpp b/sphaira/source/location.cpp new file mode 100644 index 0000000..9fcb8e8 --- /dev/null +++ b/sphaira/source/location.cpp @@ -0,0 +1,75 @@ +#include "location.hpp" +#include "fs.hpp" + +#include +#include + +namespace sphaira::location { +namespace { + +constexpr fs::FsPath location_path{"/config/sphaira/locations.ini"}; + +} // namespace + +void Add(const Entry& e) { + if (e.name.empty() || e.url.empty()) { + return; + } + + ini_puts(e.name.c_str(), "url", e.url.c_str(), location_path); + if (!e.user.empty()) { + ini_puts(e.name.c_str(), "user", e.user.c_str(), location_path); + } + if (!e.pass.empty()) { + ini_puts(e.name.c_str(), "pass", e.pass.c_str(), location_path); + } + if (!e.bearer.empty()) { + ini_puts(e.name.c_str(), "bearer", e.bearer.c_str(), location_path); + } + if (!e.pub_key.empty()) { + ini_puts(e.name.c_str(), "pub_key", e.pub_key.c_str(), location_path); + } + if (!e.priv_key.empty()) { + ini_puts(e.name.c_str(), "priv_key", e.priv_key.c_str(), location_path); + } + if (e.port) { + ini_putl(e.name.c_str(), "port", e.port, location_path); + } +} + +auto Load() -> Entries { + Entries out{}; + + auto cb = [](const mTCHAR *Section, const mTCHAR *Key, const mTCHAR *Value, void *UserData) -> int { + auto e = static_cast(UserData); + + // add new entry if use section changed. + if (e->empty() || std::strcmp(Section, e->back().name.c_str())) { + e->emplace_back(Section); + } + + if (!std::strcmp(Key, "url")) { + e->back().url = Value; + } else if (!std::strcmp(Key, "user")) { + e->back().user = Value; + } else if (!std::strcmp(Key, "pass")) { + e->back().pass = Value; + } else if (!std::strcmp(Key, "bearer")) { + e->back().bearer = Value; + } else if (!std::strcmp(Key, "pub_key")) { + e->back().pub_key = Value; + } else if (!std::strcmp(Key, "priv_key")) { + e->back().priv_key = Value; + } else if (!std::strcmp(Key, "port")) { + e->back().port = std::atoi(Value); + } + + return 1; + }; + + ini_browse(cb, &out, location_path); + + return out; +} + +} // namespace sphaira::location diff --git a/sphaira/source/ui/menus/game_menu.cpp b/sphaira/source/ui/menus/game_menu.cpp index 9753ede..f6bcd0d 100644 --- a/sphaira/source/ui/menus/game_menu.cpp +++ b/sphaira/source/ui/menus/game_menu.cpp @@ -1,6 +1,11 @@ #include "app.hpp" #include "log.hpp" #include "fs.hpp" +#include "download.hpp" +#include "defines.hpp" +#include "i18n.hpp" +#include "location.hpp" + #include "ui/menus/game_menu.hpp" #include "ui/sidebar.hpp" #include "ui/error_box.hpp" @@ -8,8 +13,6 @@ #include "ui/progress_box.hpp" #include "ui/popup_list.hpp" #include "ui/nvg_util.hpp" -#include "defines.hpp" -#include "i18n.hpp" #include "yati/nx/ncm.hpp" #include "yati/nx/nca.hpp" @@ -22,6 +25,7 @@ #include #include #include +#include namespace sphaira::ui::menu::game { namespace { @@ -157,7 +161,7 @@ struct NspEntry { // adjust offset. off -= nsp_data.size(); - for (auto& collection : collections) { + for (const auto& collection : collections) { if (InRange(off, collection.offset, collection.size)) { // adjust offset relative to the collection. off -= collection.offset; @@ -217,6 +221,30 @@ struct NspSource final : BaseSource { return m_entries; } + auto GetName(const std::string& path) const -> std::string { + const auto it = std::ranges::find_if(m_entries, [&path](auto& e){ + return path == e.path; + }); + + if (it != m_entries.end()) { + return it->application_name; + } + + return {}; + } + + auto GetSize(const std::string& path) const -> s64 { + const auto it = std::ranges::find_if(m_entries, [&path](auto& e){ + return path == e.path; + }); + + if (it != m_entries.end()) { + return it->nsp_size; + } + + return 0; + } + private: std::span m_entries{}; }; @@ -230,11 +258,17 @@ struct UsbTest final : usb::upload::Usb { Result Read(const std::string& path, void* buf, s64 off, s64 size, u64* bytes_read) override { if (m_path != path) { m_path = path; + m_progress = 0; + m_size = m_source->GetSize(path); + m_pbox->SetTitle(m_source->GetName(path)); m_pbox->NewTransfer(m_path); } R_TRY(m_source->Read(path, buf, off, size, bytes_read)); + m_offset += *bytes_read; + m_progress += *bytes_read; + m_pbox->UpdateTransfer(m_progress, m_size); R_SUCCEED(); } @@ -244,6 +278,8 @@ private: ProgressBox* m_pbox{}; std::string m_path{}; s64 m_offset{}; + s64 m_size{}; + s64 m_progress{}; }; Result DumpNspToFile(ProgressBox* pbox, std::span entries) { @@ -360,6 +396,50 @@ Result DumpNspToDevNull(ProgressBox* pbox, std::span entries) { R_SUCCEED(); } +Result DumpNspToNetwork(ProgressBox* pbox, const location::Entry& loc, std::span entries) { + auto source = std::make_unique(entries); + for (const auto& e : entries) { + if (pbox->ShouldExit()) { + R_THROW(0xFFFF); + } + + pbox->SetTitle(e.application_name); + pbox->NewTransfer(e.path); + + s64 offset{}; + const auto result = curl::Api().FromMemory( + curl::Url{loc.url}, + curl::UserPass{loc.user, loc.pass}, + curl::Bearer{loc.bearer}, + curl::PubKey{loc.pub_key}, + curl::PrivKey{loc.priv_key}, + curl::Port(loc.port), + curl::OnProgress{pbox->OnDownloadProgressCallback()}, + curl::UploadInfo{ + e.path, e.nsp_size, + [&pbox, &e, &source, &offset](void *ptr, size_t size) -> size_t { + u64 bytes_read{}; + if (R_FAILED(source->Read(e.path, ptr, offset, size, &bytes_read))) { + // curl will request past the size of the file, causing an error. + // only log the error if it failed in the middle of a transfer. + if (offset != e.nsp_size) { + log_write("failed to read in custom callback: %zd size: %zd\n", offset, e.nsp_size); + } + return 0; + } + + offset += bytes_read; + return bytes_read; + } + } + ); + + R_UNLESS(result.success, 0x1); + } + + R_SUCCEED(); +} + Result Notify(Result rc, const std::string& error_message) { if (R_FAILED(rc)) { App::Push(std::make_shared(rc, @@ -1215,7 +1295,7 @@ void Menu::OnLayoutChange() { } void Menu::DeleteGames() { - App::Push(std::make_shared(0, "Deleting Games"_i18n, "", [this](auto pbox) -> bool { + App::Push(std::make_shared(0, "Deleting"_i18n, "", [this](auto pbox) -> bool { auto targets = GetSelectedEntries(); for (s64 i = 0; i < std::size(targets); i++) { @@ -1242,18 +1322,24 @@ void Menu::DeleteGames() { void Menu::DumpGames(u32 flags) { PopupList::Items items; + const auto network_locations = location::Load(); + + for (const auto&p : network_locations) { + items.emplace_back(p.name); + } + for (const auto&p : DUMP_LOCATIONS) { - items.emplace_back(p.display_name); + items.emplace_back(i18n::get(p.display_name)); } App::Push(std::make_shared( - "Select dump location"_i18n, items, [this, flags](auto op_index){ + "Select dump location"_i18n, items, [this, network_locations, flags](auto op_index){ if (!op_index) { return; } const auto index = *op_index; - App::Push(std::make_shared(0, "Dumping Games"_i18n, "", [this, index, flags](auto pbox) -> bool { + App::Push(std::make_shared(0, "Dumping"_i18n, "", [this, network_locations, index, flags](auto pbox) -> bool { auto targets = GetSelectedEntries(); std::vector nsp_entries; @@ -1261,11 +1347,15 @@ void Menu::DumpGames(u32 flags) { BuildNspEntries(e, flags, nsp_entries); } - if (index == DumpLocationType_SdCard) { + const auto index2 = index - network_locations.size(); + + if (!network_locations.empty() && index < network_locations.size()) { + return R_SUCCEEDED(DumpNspToNetwork(pbox, network_locations[index], nsp_entries)); + } else if (index2 == DumpLocationType_SdCard) { return R_SUCCEEDED(DumpNspToFile(pbox, nsp_entries)); - } else if (index == DumpLocationType_UsbS2S) { + } else if (index2 == DumpLocationType_UsbS2S) { return R_SUCCEEDED(DumpNspToUsbS2S(pbox, nsp_entries)); - } else if (index == DumpLocationType_DevNull) { + } else if (index2 == DumpLocationType_DevNull) { return R_SUCCEEDED(DumpNspToDevNull(pbox, nsp_entries)); }