OmniNX/sphaira
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add support for file uploads in the file browser, optimise curl single thread download.

Browse Source 
- curl now keeps the handle alive for single threaded downloads, rather than creating it each time.
This commit is contained in:
ITotalJustice
2025-05-18 23:00:51 +01:00
parent 71df5317be
commit eadc46b0e4
5 changed files with 144 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
sphaira/include/download.hpp
View File

@@ -79,6 +79,7 @@ struct UserPass {
struct UploadInfo {
UploadInfo() = default;
UploadInfo(const std::string& name) : m_name{name} {}
UploadInfo(const std::string& name, s64 size, OnUploadCallback cb) : m_name{name}, m_size{size}, m_callback{cb} {}
UploadInfo(const std::string& name, const std::vector<u8>& data) : m_name{name}, m_data{data} {}
std::string m_name{};
@@ -119,6 +120,15 @@ struct DownloadEventData {
StopToken stoken;
};
// helper that generates the api using an location.
#define CURL_LOCATION_TO_API(loc) \
curl::Url{loc.url}, \
curl::UserPass{loc.user, loc.pass}, \
curl::Bearer{loc.bearer}, \
curl::PubKey{loc.pub_key}, \
curl::PrivKey{loc.priv_key}, \
curl::Port(loc.port)
auto Init() -> bool;
void Exit();
@@ -213,6 +223,7 @@ struct Api {
auto FromFile(Ts&&... ts) {
static_assert(std::disjunction_v<std::is_same<Url, Ts>...>, "Url must be specified");
static_assert(std::disjunction_v<std::is_same<Path, Ts>...>, "Path must be specified");
static_assert(std::disjunction_v<std::is_same<UploadInfo, Ts>...>, "UploadInfo must be specified");
static_assert(!std::disjunction_v<std::is_same<OnComplete, Ts>...>, "OnComplete must not be specified");
Api::set_option(std::forward<Ts>(ts)...);
return curl::FromFile(*this);
@@ -253,6 +264,7 @@ struct Api {
auto FromFileAsync(Ts&&... ts) {
static_assert(std::disjunction_v<std::is_same<Url, Ts>...>, "Url must be specified");
static_assert(std::disjunction_v<std::is_same<Path, Ts>...>, "Path must be specified");
static_assert(std::disjunction_v<std::is_same<UploadInfo, Ts>...>, "UploadInfo must be specified");
static_assert(std::disjunction_v<std::is_same<OnComplete, Ts>...>, "OnComplete must be specified");
static_assert(std::disjunction_v<std::is_same<StopToken, Ts>...>, "StopToken must be specified");
Api::set_option(std::forward<Ts>(ts)...);

32
sphaira/include/ui/menus/filebrowser.hpp
View File

@@ -136,14 +136,11 @@ struct Menu final : MenuBase {
private:
void SetIndex(s64 index);
void InstallForwarder();
void InstallFile(const FileEntry& target);
void InstallFiles(const std::vector<FileEntry>& targets);
void UnzipFile(const fs::FsPath& folder, const FileEntry& target);
void UnzipFiles(fs::FsPath folder, const std::vector<FileEntry>& targets);
void ZipFile(const fs::FsPath& zip_path, const FileEntry& target);
void ZipFiles(fs::FsPath zip_path, const std::vector<FileEntry>& targets);
void InstallFiles();
void UnzipFiles(fs::FsPath folder);
void ZipFiles(fs::FsPath zip_path);
void UploadFiles();
auto Scan(const fs::FsPath& new_path, bool is_walk_up = false) -> Result;
@@ -164,15 +161,15 @@ private:
}
auto GetSelectedEntries() const -> std::vector<FileEntry> {
if (!m_selected_count) {
return {};
}
std::vector<FileEntry> out;
for (auto&e : m_entries) {
if (e.IsSelected()) {
out.emplace_back(e);
if (!m_selected_count) {
out.emplace_back(GetEntry());
} else {
for (auto&e : m_entries) {
if (e.IsSelected()) {
out.emplace_back(e);
}
}
}
@@ -191,13 +188,6 @@ private:
m_selected_path = m_path;
}
void AddCurrentFileToSelection(SelectedType type) {
m_selected_files.emplace_back(GetEntry());
m_selected_count++;
m_selected_type = type;
m_selected_path = m_path;
}
void ResetSelection() {
m_selected_files.clear();
m_selected_count = 0;

51
sphaira/source/download.cpp
View File

@@ -33,6 +33,9 @@ constexpr int THREAD_CORE = 1;
std::atomic_bool g_running{};
CURLSH* g_curl_share{};
// this is used for single threaded blocking installs.
// avoids the needed for re-creating the handle each time.
CURL* g_curl_single{};
Mutex g_mutex_share[CURL_LOCK_DATA_LAST]{};
struct UploadStruct {
@@ -690,8 +693,8 @@ auto UploadInternal(CURL* curl, const Api& e) -> ApiResult {
return {};
}
auto url = e.GetUrl();
const auto& info = e.GetUploadInfo();
const auto url = e.GetUrl() + "/" + info.m_name;
const bool has_file = !e.GetPath().empty() && e.GetPath() != "";
UploadStruct chunk{};
@@ -717,8 +720,6 @@ auto UploadInternal(CURL* curl, const Api& e) -> ApiResult {
upload_size = info.m_data.size();
chunk.data = info.m_data;
}
url += "/" + info.m_name;
}
if (url.starts_with("file://")) {
@@ -807,26 +808,6 @@ auto UploadInternal(CURL* curl, const Api& e) -> ApiResult {
return {success, http_code, header_out, chunk_out.data};
}
auto DownloadInternal(const Api& e) -> ApiResult {
auto curl = curl_easy_init();
if (!curl) {
log_write("curl init failed\n");
return {};
}
ON_SCOPE_EXIT(curl_easy_cleanup(curl));
return DownloadInternal(curl, e);
}
auto UploadInternal(const Api& e) -> ApiResult {
auto curl = curl_easy_init();
if (!curl) {
log_write("curl init failed\n");
return {};
}
ON_SCOPE_EXIT(curl_easy_cleanup(curl));
return UploadInternal(curl, e);
}
void my_lock(CURL *handle, curl_lock_data data, curl_lock_access laccess, void *useptr) {
mutexLock(&g_mutex_share[data]);
}
@@ -848,12 +829,6 @@ void ThreadEntry::ThreadFunc(void* p) {
continue;
}
// ApiResult result;
// if (data->m_api.IsUpload()) {
// result = UploadInternal(data->m_curl, data->m_api);
// } else {
// result = DownloadInternal(data->m_curl, data->m_api);
// }
const auto result = data->m_api.IsUpload() ? UploadInternal(data->m_curl, data->m_api) : DownloadInternal(data->m_curl, data->m_api);
if (g_running && data->m_api.GetOnComplete() && !data->m_api.GetToken().stop_requested()) {
evman::push(
@@ -956,6 +931,11 @@ auto Init() -> bool {
}
}
g_curl_single = curl_easy_init();
if (!g_curl_single) {
log_write("failed to create g_curl_single\n");
}
log_write("finished creating threads\n");
if (!g_cache.init()) {
@@ -970,6 +950,11 @@ void Exit() {
g_thread_queue.Close();
if (g_curl_single) {
curl_easy_cleanup(g_curl_single);
g_curl_single = nullptr;
}
for (auto& entry : g_threads) {
entry.Close();
}
@@ -987,28 +972,28 @@ auto ToMemory(const Api& e) -> ApiResult {
if (!e.GetPath().empty()) {
return {};
}
return DownloadInternal(e);
return DownloadInternal(g_curl_single, e);
}
auto ToFile(const Api& e) -> ApiResult {
if (e.GetPath().empty()) {
return {};
}
return DownloadInternal(e);
return DownloadInternal(g_curl_single, e);
}
auto FromMemory(const Api& e) -> ApiResult {
if (!e.GetPath().empty()) {
return {};
}
return UploadInternal(e);
return UploadInternal(g_curl_single, e);
}
auto FromFile(const Api& e) -> ApiResult {
if (e.GetPath().empty()) {
return {};
}
return UploadInternal(e);
return UploadInternal(g_curl_single, e);
}
auto ToMemoryAsync(const Api& api) -> bool {

163
sphaira/source/ui/menus/filebrowser.cpp
View File

@@ -18,6 +18,8 @@
#include "owo.hpp"
#include "swkbd.hpp"
#include "i18n.hpp"
#include "location.hpp"
#include "yati/yati.hpp"
#include "yati/source/file.hpp"
@@ -337,7 +339,7 @@ Menu::Menu(const std::vector<NroEntry>& nro_entries) : MenuBase{"FileBrowser"_i1
}
}));
} else if (App::GetInstallEnable() && IsExtension(entry.GetExtension(), INSTALL_EXTENSIONS)) {
InstallFile(GetEntry());
InstallFiles();
} else {
const auto assoc_list = FindFileAssocFor();
if (!assoc_list.empty()) {
@@ -427,27 +429,16 @@ Menu::Menu(const std::vector<NroEntry>& nro_entries) : MenuBase{"FileBrowser"_i1
if (m_entries_current.size()) {
options->Add(std::make_shared<SidebarEntryCallback>("Cut"_i18n, [this](){
if (!m_selected_count) {
AddCurrentFileToSelection(SelectedType::Cut);
} else {
AddSelectedEntries(SelectedType::Cut);
}
AddSelectedEntries(SelectedType::Cut);
}, true));
options->Add(std::make_shared<SidebarEntryCallback>("Copy"_i18n, [this](){
if (!m_selected_count) {
AddCurrentFileToSelection(SelectedType::Copy);
} else {
AddSelectedEntries(SelectedType::Copy);
}
AddSelectedEntries(SelectedType::Copy);
}, true));
options->Add(std::make_shared<SidebarEntryCallback>("Delete"_i18n, [this](){
if (!m_selected_count) {
AddCurrentFileToSelection(SelectedType::Delete);
} else {
AddSelectedEntries(SelectedType::Delete);
}
AddSelectedEntries(SelectedType::Delete);
log_write("clicked on delete\n");
App::Push(std::make_shared<OptionBox>(
"Delete Selected files?"_i18n, "No"_i18n, "Yes"_i18n, 0, [this](auto op_index){
@@ -522,11 +513,7 @@ Menu::Menu(const std::vector<NroEntry>& nro_entries) : MenuBase{"FileBrowser"_i1
if (m_entries_current.size() && App::GetInstallEnable()) {
if (check_all_ext(INSTALL_EXTENSIONS)) {
options->Add(std::make_shared<SidebarEntryCallback>("Install"_i18n, [this](){
if (!m_selected_count) {
InstallFile(GetEntry());
} else {
InstallFiles(GetSelectedEntries());
}
InstallFiles();
}));
}
}
@@ -557,22 +544,14 @@ Menu::Menu(const std::vector<NroEntry>& nro_entries) : MenuBase{"FileBrowser"_i1
ON_SCOPE_EXIT(App::Push(options));
options->Add(std::make_shared<SidebarEntryCallback>("Extract here"_i18n, [this](){
if (!m_selected_count) {
UnzipFile("", GetEntry());
} else {
UnzipFiles("", GetSelectedEntries());
}
UnzipFiles("");
}));
options->Add(std::make_shared<SidebarEntryCallback>("Extract to root"_i18n, [this](){
App::Push(std::make_shared<OptionBox>("Are you sure you want to extract to root?"_i18n,
"No"_i18n, "Yes"_i18n, 0, [this](auto op_index){
if (op_index && *op_index) {
if (!m_selected_count) {
UnzipFile("/", GetEntry());
} else {
UnzipFiles("/", GetSelectedEntries());
}
UnzipFiles("/");
}
}));
}));
@@ -580,11 +559,7 @@ Menu::Menu(const std::vector<NroEntry>& nro_entries) : MenuBase{"FileBrowser"_i1
options->Add(std::make_shared<SidebarEntryCallback>("Extract to..."_i18n, [this](){
std::string out;
if (R_SUCCEEDED(swkbd::ShowText(out, "Enter the path to the folder to extract into", fs::AppendPath(m_path, ""))) && !out.empty()) {
if (!m_selected_count) {
UnzipFile(out, GetEntry());
} else {
UnzipFiles(out, GetSelectedEntries());
}
UnzipFiles(out);
}
}));
}));
@@ -596,21 +571,13 @@ Menu::Menu(const std::vector<NroEntry>& nro_entries) : MenuBase{"FileBrowser"_i1
ON_SCOPE_EXIT(App::Push(options));
options->Add(std::make_shared<SidebarEntryCallback>("Compress"_i18n, [this](){
if (!m_selected_count) {
ZipFile("", GetEntry());
} else {
ZipFiles("", GetSelectedEntries());
}
ZipFiles("");
}));
options->Add(std::make_shared<SidebarEntryCallback>("Compress to..."_i18n, [this](){
std::string out;
if (R_SUCCEEDED(swkbd::ShowText(out, "Enter the path to the folder to extract into", m_path)) && !out.empty()) {
if (!m_selected_count) {
ZipFile(out, GetEntry());
} else {
ZipFiles(out, GetSelectedEntries());
}
ZipFiles(out);
}
}));
}));
@@ -670,6 +637,12 @@ Menu::Menu(const std::vector<NroEntry>& nro_entries) : MenuBase{"FileBrowser"_i1
}));
}
if (m_fs_type == FsType::Sd && m_entries_current.size()) {
options->Add(std::make_shared<SidebarEntryCallback>("Upload"_i18n, [this](){
UploadFiles();
}));
}
options->Add(std::make_shared<SidebarEntryBool>("Ignore read only"_i18n, m_ignore_read_only.Get(), [this](bool& v_out){
m_ignore_read_only.Set(v_out);
m_fs->SetIgnoreReadOnly(v_out);
@@ -918,12 +891,9 @@ void Menu::InstallForwarder() {
));
}
void Menu::InstallFile(const FileEntry& target) {
std::vector<FileEntry> targets{target};
InstallFiles(targets);
}
void Menu::InstallFiles() {
const auto targets = GetSelectedEntries();
void Menu::InstallFiles(const std::vector<FileEntry>& targets) {
App::Push(std::make_shared<OptionBox>("Install Selected files?"_i18n, "No"_i18n, "Yes"_i18n, 0, [this, targets](auto op_index){
if (op_index && *op_index) {
App::PopToMenu();
@@ -946,12 +916,9 @@ void Menu::InstallFiles(const std::vector<FileEntry>& targets) {
}));
}
void Menu::UnzipFile(const fs::FsPath& dir_path, const FileEntry& target) {
std::vector<FileEntry> targets{target};
UnzipFiles(dir_path, targets);
}
void Menu::UnzipFiles(fs::FsPath dir_path) {
const auto targets = GetSelectedEntries();
void Menu::UnzipFiles(fs::FsPath dir_path, const std::vector<FileEntry>& targets) {
// set to current path.
if (dir_path.empty()) {
dir_path = m_path;
@@ -1058,12 +1025,9 @@ void Menu::UnzipFiles(fs::FsPath dir_path, const std::vector<FileEntry>& targets
}));
}
void Menu::ZipFile(const fs::FsPath& zip_path, const FileEntry& target) {
std::vector<FileEntry> targets{target};
ZipFiles(zip_path, targets);
}
void Menu::ZipFiles(fs::FsPath zip_out) {
const auto targets = GetSelectedEntries();
void Menu::ZipFiles(fs::FsPath zip_out, const std::vector<FileEntry>& targets) {
// set to current path.
if (zip_out.empty()) {
if (std::size(targets) == 1) {
@@ -1212,6 +1176,83 @@ void Menu::ZipFiles(fs::FsPath zip_out, const std::vector<FileEntry>& targets) {
}));
}
void Menu::UploadFiles() {
const auto targets = GetSelectedEntries();
const auto network_locations = location::Load();
if (network_locations.empty()) {
App::Notify("No upload locations set!");
return;
}
PopupList::Items items;
for (const auto&p : network_locations) {
items.emplace_back(p.name);
}
App::Push(std::make_shared<PopupList>(
"Select upload location"_i18n, items, [this, network_locations](auto op_index){
if (!op_index) {
return;
}
const auto loc = network_locations[*op_index];
App::Push(std::make_shared<ProgressBox>(0, "Uploading"_i18n, "", [this, loc](auto pbox) -> bool {
auto targets = GetSelectedEntries();
const auto file_add = [&](const fs::FsPath& file_path, const char* name){
// the file name needs to be relative to the current directory.
const auto relative_file_name = file_path.s + std::strlen(m_path);
pbox->SetTitle(name);
pbox->NewTransfer(relative_file_name);
const auto result = curl::Api().FromFile(
CURL_LOCATION_TO_API(loc),
curl::Path{file_path},
curl::OnProgress{pbox->OnDownloadProgressCallback()},
curl::UploadInfo{relative_file_name}
);
return result.success;
};
for (auto& e : targets) {
if (e.IsFile()) {
const auto file_path = GetNewPath(e);
if (!file_add(file_path, e.GetName().c_str())) {
return false;
}
} else {
FsDirCollections collections;
get_collections(GetNewPath(e), e.name, collections);
for (const auto& collection : collections) {
for (const auto& file : collection.files) {
const auto file_path = fs::AppendPath(collection.path, file.name);
if (!file_add(file_path, file.name)) {
return false;
}
}
}
}
}
return true;
}, [this](bool success){
ResetSelection();
if (success) {
App::Notify("Upload successfull!");
log_write("Upload successfull!!!\n");
} else {
App::Notify("Upload failed!");
log_write("Upload failed!!!\n");
}
}));
}
));
}
auto Menu::Scan(const fs::FsPath& new_path, bool is_walk_up) -> Result {
log_write("new scan path: %s\n", new_path.s);
if (!is_walk_up && !m_path.empty() && !m_entries_current.empty()) {

7
sphaira/source/ui/menus/game_menu.cpp
View File

@@ -408,12 +408,7 @@ Result DumpNspToNetwork(ProgressBox* pbox, const location::Entry& loc, std::span
s64 offset{};
const auto result = curl::Api().FromMemory(
curl::Url{loc.url},
curl::UserPass{loc.user, loc.pass},
curl::Bearer{loc.bearer},
curl::PubKey{loc.pub_key},
curl::PrivKey{loc.priv_key},
curl::Port(loc.port),
CURL_LOCATION_TO_API(loc),
curl::OnProgress{pbox->OnDownloadProgressCallback()},
curl::UploadInfo{
e.path, e.nsp_size,