add support for file uploads in the file browser, optimise curl single thread download.

- curl now keeps the handle alive for single threaded downloads, rather than creating it each time.

sphaira/include/download.hpp

@@ -79,6 +79,7 @@ struct UserPass {
 
 struct UploadInfo {
     UploadInfo() = default;
+    UploadInfo(const std::string& name) : m_name{name} {}
     UploadInfo(const std::string& name, s64 size, OnUploadCallback cb) : m_name{name}, m_size{size}, m_callback{cb} {}
     UploadInfo(const std::string& name, const std::vector<u8>& data) : m_name{name}, m_data{data} {}
     std::string m_name{};
@@ -119,6 +120,15 @@ struct DownloadEventData {
     StopToken stoken;
 };
 
+// helper that generates the api using an location.
+#define CURL_LOCATION_TO_API(loc) \
+    curl::Url{loc.url}, \
+    curl::UserPass{loc.user, loc.pass}, \
+    curl::Bearer{loc.bearer}, \
+    curl::PubKey{loc.pub_key}, \
+    curl::PrivKey{loc.priv_key}, \
+    curl::Port(loc.port)
+
 auto Init() -> bool;
 void Exit();
 
@@ -213,6 +223,7 @@ struct Api {
     auto FromFile(Ts&&... ts) {
         static_assert(std::disjunction_v<std::is_same<Url, Ts>...>, "Url must be specified");
         static_assert(std::disjunction_v<std::is_same<Path, Ts>...>, "Path must be specified");
+        static_assert(std::disjunction_v<std::is_same<UploadInfo, Ts>...>, "UploadInfo must be specified");
         static_assert(!std::disjunction_v<std::is_same<OnComplete, Ts>...>, "OnComplete must not be specified");
         Api::set_option(std::forward<Ts>(ts)...);
         return curl::FromFile(*this);
@@ -253,6 +264,7 @@ struct Api {
     auto FromFileAsync(Ts&&... ts) {
         static_assert(std::disjunction_v<std::is_same<Url, Ts>...>, "Url must be specified");
         static_assert(std::disjunction_v<std::is_same<Path, Ts>...>, "Path must be specified");
+        static_assert(std::disjunction_v<std::is_same<UploadInfo, Ts>...>, "UploadInfo must be specified");
         static_assert(std::disjunction_v<std::is_same<OnComplete, Ts>...>, "OnComplete must be specified");
         static_assert(std::disjunction_v<std::is_same<StopToken, Ts>...>, "StopToken must be specified");
         Api::set_option(std::forward<Ts>(ts)...);

sphaira/include/ui/menus/filebrowser.hpp

@@ -136,14 +136,11 @@ struct Menu final : MenuBase {
 private:
     void SetIndex(s64 index);
     void InstallForwarder();
-    void InstallFile(const FileEntry& target);
-    void InstallFiles(const std::vector<FileEntry>& targets);
 
-    void UnzipFile(const fs::FsPath& folder, const FileEntry& target);
+    void InstallFiles();
-    void UnzipFiles(fs::FsPath folder, const std::vector<FileEntry>& targets);
+    void UnzipFiles(fs::FsPath folder);
-
+    void ZipFiles(fs::FsPath zip_path);
-    void ZipFile(const fs::FsPath& zip_path, const FileEntry& target);
+    void UploadFiles();
-    void ZipFiles(fs::FsPath zip_path, const std::vector<FileEntry>& targets);
 
     auto Scan(const fs::FsPath& new_path, bool is_walk_up = false) -> Result;
 
@@ -164,15 +161,15 @@ private:
     }
 
     auto GetSelectedEntries() const -> std::vector<FileEntry> {
-        if (!m_selected_count) {
-            return {};
-        }
-
         std::vector<FileEntry> out;
 
-        for (auto&e : m_entries) {
+        if (!m_selected_count) {
-            if (e.IsSelected()) {
+            out.emplace_back(GetEntry());
-                out.emplace_back(e);
+        } else {
+            for (auto&e : m_entries) {
+                if (e.IsSelected()) {
+                    out.emplace_back(e);
+                }
             }
         }
 
@@ -191,13 +188,6 @@ private:
         m_selected_path = m_path;
     }
 
-    void AddCurrentFileToSelection(SelectedType type) {
-        m_selected_files.emplace_back(GetEntry());
-        m_selected_count++;
-        m_selected_type = type;
-        m_selected_path = m_path;
-    }
-
     void ResetSelection() {
         m_selected_files.clear();
         m_selected_count = 0;

sphaira/source/download.cpp

@@ -33,6 +33,9 @@ constexpr int THREAD_CORE = 1;
 
 std::atomic_bool g_running{};
 CURLSH* g_curl_share{};
+// this is used for single threaded blocking installs.
+// avoids the needed for re-creating the handle each time.
+CURL* g_curl_single{};
 Mutex g_mutex_share[CURL_LOCK_DATA_LAST]{};
 
 struct UploadStruct {
@@ -690,8 +693,8 @@ auto UploadInternal(CURL* curl, const Api& e) -> ApiResult {
         return {};
     }
 
-    auto url = e.GetUrl();
     const auto& info = e.GetUploadInfo();
+    const auto url = e.GetUrl() + "/" + info.m_name;
     const bool has_file = !e.GetPath().empty() && e.GetPath() != "";
 
     UploadStruct chunk{};
@@ -717,8 +720,6 @@ auto UploadInternal(CURL* curl, const Api& e) -> ApiResult {
             upload_size = info.m_data.size();
             chunk.data = info.m_data;
         }
-
-        url += "/" + info.m_name;
     }
 
     if (url.starts_with("file://")) {
@@ -807,26 +808,6 @@ auto UploadInternal(CURL* curl, const Api& e) -> ApiResult {
     return {success, http_code, header_out, chunk_out.data};
 }
 
-auto DownloadInternal(const Api& e) -> ApiResult {
-    auto curl = curl_easy_init();
-    if (!curl) {
-        log_write("curl init failed\n");
-        return {};
-    }
-    ON_SCOPE_EXIT(curl_easy_cleanup(curl));
-    return DownloadInternal(curl, e);
-}
-
-auto UploadInternal(const Api& e) -> ApiResult {
-    auto curl = curl_easy_init();
-    if (!curl) {
-        log_write("curl init failed\n");
-        return {};
-    }
-    ON_SCOPE_EXIT(curl_easy_cleanup(curl));
-    return UploadInternal(curl, e);
-}
-
 void my_lock(CURL *handle, curl_lock_data data, curl_lock_access laccess, void *useptr) {
     mutexLock(&g_mutex_share[data]);
 }
@@ -848,12 +829,6 @@ void ThreadEntry::ThreadFunc(void* p) {
             continue;
         }
 
-        // ApiResult result;
-        // if (data->m_api.IsUpload()) {
-        //     result = UploadInternal(data->m_curl, data->m_api);
-        // } else {
-        //     result = DownloadInternal(data->m_curl, data->m_api);
-        // }
         const auto result = data->m_api.IsUpload() ? UploadInternal(data->m_curl, data->m_api) : DownloadInternal(data->m_curl, data->m_api);
         if (g_running && data->m_api.GetOnComplete() && !data->m_api.GetToken().stop_requested()) {
             evman::push(
@@ -956,6 +931,11 @@ auto Init() -> bool {
         }
     }
 
+    g_curl_single = curl_easy_init();
+    if (!g_curl_single) {
+        log_write("failed to create g_curl_single\n");
+    }
+
     log_write("finished creating threads\n");
 
     if (!g_cache.init()) {
@@ -970,6 +950,11 @@ void Exit() {
 
     g_thread_queue.Close();
 
+    if (g_curl_single) {
+        curl_easy_cleanup(g_curl_single);
+        g_curl_single = nullptr;
+    }
+
     for (auto& entry : g_threads) {
         entry.Close();
     }
@@ -987,28 +972,28 @@ auto ToMemory(const Api& e) -> ApiResult {
     if (!e.GetPath().empty()) {
         return {};
     }
-    return DownloadInternal(e);
+    return DownloadInternal(g_curl_single, e);
 }
 
 auto ToFile(const Api& e) -> ApiResult {
     if (e.GetPath().empty()) {
         return {};
     }
-    return DownloadInternal(e);
+    return DownloadInternal(g_curl_single, e);
 }
 
 auto FromMemory(const Api& e) -> ApiResult {
     if (!e.GetPath().empty()) {
         return {};
     }
-    return UploadInternal(e);
+    return UploadInternal(g_curl_single, e);
 }
 
 auto FromFile(const Api& e) -> ApiResult {
     if (e.GetPath().empty()) {
         return {};
     }
-    return UploadInternal(e);
+    return UploadInternal(g_curl_single, e);
 }
 
 auto ToMemoryAsync(const Api& api) -> bool {

sphaira/source/ui/menus/filebrowser.cpp

@@ -18,6 +18,8 @@
 #include "owo.hpp"
 #include "swkbd.hpp"
 #include "i18n.hpp"
+#include "location.hpp"
+
 #include "yati/yati.hpp"
 #include "yati/source/file.hpp"
 
@@ -337,7 +339,7 @@ Menu::Menu(const std::vector<NroEntry>& nro_entries) : MenuBase{"FileBrowser"_i1
                             }
                         }));
                 } else if (App::GetInstallEnable() && IsExtension(entry.GetExtension(), INSTALL_EXTENSIONS)) {
-                    InstallFile(GetEntry());
+                    InstallFiles();
                 } else {
                     const auto assoc_list = FindFileAssocFor();
                     if (!assoc_list.empty()) {
@@ -427,27 +429,16 @@ Menu::Menu(const std::vector<NroEntry>& nro_entries) : MenuBase{"FileBrowser"_i1
 
             if (m_entries_current.size()) {
                 options->Add(std::make_shared<SidebarEntryCallback>("Cut"_i18n, [this](){
-                    if (!m_selected_count) {
+                    AddSelectedEntries(SelectedType::Cut);
-                        AddCurrentFileToSelection(SelectedType::Cut);
-                    } else {
-                        AddSelectedEntries(SelectedType::Cut);
-                    }
                 }, true));
 
                 options->Add(std::make_shared<SidebarEntryCallback>("Copy"_i18n, [this](){
-                    if (!m_selected_count) {
+                    AddSelectedEntries(SelectedType::Copy);
-                        AddCurrentFileToSelection(SelectedType::Copy);
-                    } else {
-                        AddSelectedEntries(SelectedType::Copy);
-                    }
                 }, true));
 
                 options->Add(std::make_shared<SidebarEntryCallback>("Delete"_i18n, [this](){
-                    if (!m_selected_count) {
+                    AddSelectedEntries(SelectedType::Delete);
-                        AddCurrentFileToSelection(SelectedType::Delete);
+
-                    } else {
-                        AddSelectedEntries(SelectedType::Delete);
-                    }
                     log_write("clicked on delete\n");
                     App::Push(std::make_shared<OptionBox>(
                         "Delete Selected files?"_i18n, "No"_i18n, "Yes"_i18n, 0, [this](auto op_index){
@@ -522,11 +513,7 @@ Menu::Menu(const std::vector<NroEntry>& nro_entries) : MenuBase{"FileBrowser"_i1
             if (m_entries_current.size() && App::GetInstallEnable()) {
                 if (check_all_ext(INSTALL_EXTENSIONS)) {
                     options->Add(std::make_shared<SidebarEntryCallback>("Install"_i18n, [this](){
-                        if (!m_selected_count) {
+                        InstallFiles();
-                            InstallFile(GetEntry());
-                        } else {
-                            InstallFiles(GetSelectedEntries());
-                        }
                     }));
                 }
             }
@@ -557,22 +544,14 @@ Menu::Menu(const std::vector<NroEntry>& nro_entries) : MenuBase{"FileBrowser"_i1
                         ON_SCOPE_EXIT(App::Push(options));
 
                         options->Add(std::make_shared<SidebarEntryCallback>("Extract here"_i18n, [this](){
-                            if (!m_selected_count) {
+                            UnzipFiles("");
-                                UnzipFile("", GetEntry());
-                            } else {
-                                UnzipFiles("", GetSelectedEntries());
-                            }
                         }));
 
                         options->Add(std::make_shared<SidebarEntryCallback>("Extract to root"_i18n, [this](){
                             App::Push(std::make_shared<OptionBox>("Are you sure you want to extract to root?"_i18n,
                                 "No"_i18n, "Yes"_i18n, 0, [this](auto op_index){
                                 if (op_index && *op_index) {
-                                    if (!m_selected_count) {
+                                    UnzipFiles("/");
-                                        UnzipFile("/", GetEntry());
-                                    } else {
-                                        UnzipFiles("/", GetSelectedEntries());
-                                    }
                                 }
                             }));
                         }));
@@ -580,11 +559,7 @@ Menu::Menu(const std::vector<NroEntry>& nro_entries) : MenuBase{"FileBrowser"_i1
                         options->Add(std::make_shared<SidebarEntryCallback>("Extract to..."_i18n, [this](){
                             std::string out;
                             if (R_SUCCEEDED(swkbd::ShowText(out, "Enter the path to the folder to extract into", fs::AppendPath(m_path, ""))) && !out.empty()) {
-                                if (!m_selected_count) {
+                                UnzipFiles(out);
-                                    UnzipFile(out, GetEntry());
-                                } else {
-                                    UnzipFiles(out, GetSelectedEntries());
-                                }
                             }
                         }));
                     }));
@@ -596,21 +571,13 @@ Menu::Menu(const std::vector<NroEntry>& nro_entries) : MenuBase{"FileBrowser"_i1
                         ON_SCOPE_EXIT(App::Push(options));
 
                         options->Add(std::make_shared<SidebarEntryCallback>("Compress"_i18n, [this](){
-                            if (!m_selected_count) {
+                            ZipFiles("");
-                                ZipFile("", GetEntry());
-                            } else {
-                                ZipFiles("", GetSelectedEntries());
-                            }
                         }));
 
                         options->Add(std::make_shared<SidebarEntryCallback>("Compress to..."_i18n, [this](){
                             std::string out;
                             if (R_SUCCEEDED(swkbd::ShowText(out, "Enter the path to the folder to extract into", m_path)) && !out.empty()) {
-                                if (!m_selected_count) {
+                                ZipFiles(out);
-                                    ZipFile(out, GetEntry());
-                                } else {
-                                    ZipFiles(out, GetSelectedEntries());
-                                }
                             }
                         }));
                     }));
@@ -670,6 +637,12 @@ Menu::Menu(const std::vector<NroEntry>& nro_entries) : MenuBase{"FileBrowser"_i1
                     }));
                 }
 
+                if (m_fs_type == FsType::Sd && m_entries_current.size()) {
+                    options->Add(std::make_shared<SidebarEntryCallback>("Upload"_i18n, [this](){
+                        UploadFiles();
+                    }));
+                }
+
                 options->Add(std::make_shared<SidebarEntryBool>("Ignore read only"_i18n, m_ignore_read_only.Get(), [this](bool& v_out){
                     m_ignore_read_only.Set(v_out);
                     m_fs->SetIgnoreReadOnly(v_out);
@@ -918,12 +891,9 @@ void Menu::InstallForwarder() {
     ));
 }
 
-void Menu::InstallFile(const FileEntry& target) {
+void Menu::InstallFiles() {
-    std::vector<FileEntry> targets{target};
+    const auto targets = GetSelectedEntries();
-    InstallFiles(targets);
-}
 
-void Menu::InstallFiles(const std::vector<FileEntry>& targets) {
     App::Push(std::make_shared<OptionBox>("Install Selected files?"_i18n, "No"_i18n, "Yes"_i18n, 0, [this, targets](auto op_index){
         if (op_index && *op_index) {
             App::PopToMenu();
@@ -946,12 +916,9 @@ void Menu::InstallFiles(const std::vector<FileEntry>& targets) {
     }));
 }
 
-void Menu::UnzipFile(const fs::FsPath& dir_path, const FileEntry& target) {
+void Menu::UnzipFiles(fs::FsPath dir_path) {
-    std::vector<FileEntry> targets{target};
+    const auto targets = GetSelectedEntries();
-    UnzipFiles(dir_path, targets);
-}
 
-void Menu::UnzipFiles(fs::FsPath dir_path, const std::vector<FileEntry>& targets) {
     // set to current path.
     if (dir_path.empty()) {
         dir_path = m_path;
@@ -1058,12 +1025,9 @@ void Menu::UnzipFiles(fs::FsPath dir_path, const std::vector<FileEntry>& targets
     }));
 }
 
-void Menu::ZipFile(const fs::FsPath& zip_path, const FileEntry& target) {
+void Menu::ZipFiles(fs::FsPath zip_out) {
-    std::vector<FileEntry> targets{target};
+    const auto targets = GetSelectedEntries();
-    ZipFiles(zip_path, targets);
-}
 
-void Menu::ZipFiles(fs::FsPath zip_out, const std::vector<FileEntry>& targets) {
     // set to current path.
     if (zip_out.empty()) {
         if (std::size(targets) == 1) {
@@ -1212,6 +1176,83 @@ void Menu::ZipFiles(fs::FsPath zip_out, const std::vector<FileEntry>& targets) {
     }));
 }
 
+void Menu::UploadFiles() {
+    const auto targets = GetSelectedEntries();
+
+    const auto network_locations = location::Load();
+    if (network_locations.empty()) {
+        App::Notify("No upload locations set!");
+        return;
+    }
+
+    PopupList::Items items;
+    for (const auto&p : network_locations) {
+        items.emplace_back(p.name);
+    }
+
+    App::Push(std::make_shared<PopupList>(
+        "Select upload location"_i18n, items, [this, network_locations](auto op_index){
+            if (!op_index) {
+                return;
+            }
+
+            const auto loc = network_locations[*op_index];
+            App::Push(std::make_shared<ProgressBox>(0, "Uploading"_i18n, "", [this, loc](auto pbox) -> bool {
+                auto targets = GetSelectedEntries();
+
+                const auto file_add = [&](const fs::FsPath& file_path, const char* name){
+                    // the file name needs to be relative to the current directory.
+                    const auto relative_file_name = file_path.s + std::strlen(m_path);
+                    pbox->SetTitle(name);
+                    pbox->NewTransfer(relative_file_name);
+
+                    const auto result = curl::Api().FromFile(
+                        CURL_LOCATION_TO_API(loc),
+                        curl::Path{file_path},
+                        curl::OnProgress{pbox->OnDownloadProgressCallback()},
+                        curl::UploadInfo{relative_file_name}
+                    );
+
+                    return result.success;
+                };
+
+                for (auto& e : targets) {
+                    if (e.IsFile()) {
+                        const auto file_path = GetNewPath(e);
+                        if (!file_add(file_path, e.GetName().c_str())) {
+                            return false;
+                        }
+                    } else {
+                        FsDirCollections collections;
+                        get_collections(GetNewPath(e), e.name, collections);
+
+                        for (const auto& collection : collections) {
+                            for (const auto& file : collection.files) {
+                                const auto file_path = fs::AppendPath(collection.path, file.name);
+                                if (!file_add(file_path, file.name)) {
+                                    return false;
+                                }
+                            }
+                        }
+                    }
+                }
+
+                return true;
+            }, [this](bool success){
+                ResetSelection();
+
+                if (success) {
+                    App::Notify("Upload successfull!");
+                    log_write("Upload successfull!!!\n");
+                } else {
+                    App::Notify("Upload failed!");
+                    log_write("Upload failed!!!\n");
+                }
+            }));
+        }
+    ));
+}
+
 auto Menu::Scan(const fs::FsPath& new_path, bool is_walk_up) -> Result {
     log_write("new scan path: %s\n", new_path.s);
     if (!is_walk_up && !m_path.empty() && !m_entries_current.empty()) {

sphaira/source/ui/menus/game_menu.cpp

@@ -408,12 +408,7 @@ Result DumpNspToNetwork(ProgressBox* pbox, const location::Entry& loc, std::span
 
         s64 offset{};
         const auto result = curl::Api().FromMemory(
-            curl::Url{loc.url},
+            CURL_LOCATION_TO_API(loc),
-            curl::UserPass{loc.user, loc.pass},
-            curl::Bearer{loc.bearer},
-            curl::PubKey{loc.pub_key},
-            curl::PrivKey{loc.priv_key},
-            curl::Port(loc.port),
             curl::OnProgress{pbox->OnDownloadProgressCallback()},
             curl::UploadInfo{
                 e.path, e.nsp_size,