OmniNX/sphaira
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix upload url path not being encoded, add seek api for uploads.

Browse Source
This commit is contained in:
ITotalJustice
2025-05-19 12:06:43 +01:00
parent bd6566524c
commit da9235f58e
4 changed files with 169 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
sphaira/CMakeLists.txt
View File

@@ -105,6 +105,7 @@ add_executable(sphaira
target_compile_definitions(sphaira PRIVATE
-DAPP_VERSION="${sphaira_VERSION}"
-DAPP_VERSION_HASH="${sphaira_VERSION_HASH}"
-DCURL_NO_OLDIES=1
)
target_compile_options(sphaira PRIVATE

132
sphaira/include/download.hpp
View File

@@ -31,6 +31,7 @@ using Path = fs::FsPath;
using OnComplete = std::function<void(ApiResult& result)>;
using OnProgress = std::function<bool(s64 dltotal, s64 dlnow, s64 ultotal, s64 ulnow)>;
using OnUploadCallback = std::function<size_t(void *ptr, size_t size)>;
using OnUploadSeek = std::function<bool(s64 offset)>;
using StopToken = std::stop_token;
struct Url {
@@ -271,105 +272,43 @@ struct Api {
return curl::FromFileAsync(*this);
}
void SetUpload(bool enable) {
m_is_upload = enable;
}
void SetUpload(bool enable) { m_is_upload = enable; }
auto IsUpload() const {
return m_is_upload;
}
auto& GetUrl() const {
return m_url.m_str;
}
auto& GetFields() const {
return m_fields.m_str;
}
auto& GetHeader() const {
return m_header;
}
auto& GetFlags() const {
return m_flags.m_flags;
}
auto& GetPath() const {
return m_path;
}
auto& GetPort() const {
return m_port;
}
auto& GetUserPass() const {
return m_userpass;
}
auto& GetBearer() const {
return m_bearer;
}
auto& GetPubKey() const {
return m_pub_key;
}
auto& GetPrivKey() const {
return m_priv_key;
}
auto& GetUploadInfo() const {
return m_info;
}
auto& GetOnComplete() const {
return m_on_complete;
}
auto& GetOnProgress() const {
return m_on_progress;
}
auto& GetPriority() const {
return m_prio;
}
auto& GetToken() const {
return m_stoken;
}
auto IsUpload() const { return m_is_upload; }
auto& GetUrl() const { return m_url.m_str; }
auto& GetFields() const { return m_fields.m_str; }
auto& GetHeader() const { return m_header; }
auto& GetFlags() const { return m_flags.m_flags; }
auto& GetPath() const { return m_path; }
auto& GetPort() const { return m_port.m_port; }
auto& GetUserPass() const { return m_userpass; }
auto& GetBearer() const { return m_bearer.m_str; }
auto& GetPubKey() const { return m_pub_key.m_str; }
auto& GetPrivKey() const { return m_priv_key.m_str; }
auto& GetUploadInfo() const { return m_info; }
auto& GetOnComplete() const { return m_on_complete; }
auto& GetOnProgress() const { return m_on_progress; }
auto& GetOnUploadSeek() const { return m_on_upload_seek; }
auto& GetPriority() const { return m_prio; }
auto& GetToken() const { return m_stoken; }
private:
void SetOption(Url&& v) {
m_url = v;
}
void SetOption(Fields&& v) {
m_fields = v;
}
void SetOption(Header&& v) {
m_header = v;
}
void SetOption(Flags&& v) {
m_flags = v;
}
void SetOption(Path&& v) {
m_path = v;
}
void SetOption(Port&& v) {
m_port = v;
}
void SetOption(UserPass&& v) {
m_userpass = v;
}
void SetOption(Bearer&& v) {
m_bearer = v;
}
void SetOption(PubKey&& v) {
m_pub_key = v;
}
void SetOption(PrivKey&& v) {
m_priv_key = v;
}
void SetOption(UploadInfo&& v) {
m_info = v;
}
void SetOption(OnComplete&& v) {
m_on_complete = v;
}
void SetOption(OnProgress&& v) {
m_on_progress = v;
}
void SetOption(Priority&& v) {
m_prio = v;
}
void SetOption(StopToken&& v) {
m_stoken = v;
}
void SetOption(Url&& v) { m_url = v; }
void SetOption(Fields&& v) { m_fields = v; }
void SetOption(Header&& v) { m_header = v; }
void SetOption(Flags&& v) { m_flags = v; }
void SetOption(Path&& v) { m_path = v; }
void SetOption(Port&& v) { m_port = v; }
void SetOption(UserPass&& v) { m_userpass = v; }
void SetOption(Bearer&& v) { m_bearer = v; }
void SetOption(PubKey&& v) { m_pub_key = v; }
void SetOption(PrivKey&& v) { m_priv_key = v; }
void SetOption(UploadInfo&& v) { m_info = v; }
void SetOption(OnComplete&& v) { m_on_complete = v; }
void SetOption(OnProgress&& v) { m_on_progress = v; }
void SetOption(OnUploadSeek&& v) { m_on_upload_seek = v; }
void SetOption(Priority&& v) { m_prio = v; }
void SetOption(StopToken&& v) { m_stoken = v; }
template <typename T>
void set_option(T&& t) {
@@ -396,6 +335,7 @@ private:
UploadInfo m_info{};
OnComplete m_on_complete{};
OnProgress m_on_progress{};
OnUploadSeek m_on_upload_seek{};
Priority m_prio{Priority::High};
std::stop_source m_stop_source{};
StopToken m_stoken{m_stop_source.get_token()};

165
sphaira/source/download.cpp
View File

@@ -41,6 +41,7 @@ Mutex g_mutex_share[CURL_LOCK_DATA_LAST]{};
struct UploadStruct {
std::span<const u8> data;
s64 offset{};
s64 size{};
FsFile f{};
};
@@ -51,6 +52,11 @@ struct DataStruct {
s64 file_offset{};
};
struct SeekCustomData {
OnUploadSeek cb{};
s64 size{};
};
auto generate_key_from_path(const fs::FsPath& path) -> std::string {
const auto key = crc32Calculate(path.s, path.size());
return std::to_string(key);
@@ -375,6 +381,46 @@ auto ProgressCallbackFunc2(void *clientp, curl_off_t dltotal, curl_off_t dlnow,
return 0;
}
auto SeekCallback(void *clientp, curl_off_t offset, int origin) -> int {
if (!g_running) {
return 0;
}
auto data_struct = static_cast<UploadStruct*>(clientp);
if (origin == SEEK_SET) {
offset = offset;
} else if (origin == SEEK_CUR) {
offset = data_struct->offset + offset;
} else if (origin == SEEK_END) {
offset = data_struct->size;
}
if (offset < 0 || offset > data_struct->size) {
return CURL_SEEKFUNC_CANTSEEK;
}
data_struct->offset = offset;
return CURL_SEEKFUNC_OK;
}
auto SeekCustomCallback(void *clientp, curl_off_t offset, int origin) -> int {
if (!g_running) {
return 0;
}
auto data_struct = static_cast<SeekCustomData*>(clientp);
if (origin != SEEK_SET || offset < 0 || offset > data_struct->size) {
return CURL_SEEKFUNC_CANTSEEK;
}
if (!data_struct->cb(offset)) {
return CURL_SEEKFUNC_CANTSEEK;
}
return CURL_SEEKFUNC_OK;
}
auto ReadFileCallback(char *ptr, size_t size, size_t nmemb, void *userp) -> size_t {
if (!g_running) {
return 0;
@@ -518,17 +564,20 @@ void SetCommonCurlOptions(CURL* curl, const Api& e) {
// in most cases, this will use CURLAUTH_BASIC.
CURL_EASY_SETOPT_LOG(curl, CURLOPT_HTTPAUTH, (long)CURLAUTH_ANY);
// enable TE is server supports it.
CURL_EASY_SETOPT_LOG(curl, CURLOPT_TRANSFER_ENCODING, 1L);
// set oath2 bearer.
if (!e.GetBearer().m_str.empty()) {
CURL_EASY_SETOPT_LOG(curl, CURLOPT_XOAUTH2_BEARER, e.GetBearer().m_str.c_str());
if (!e.GetBearer().empty()) {
CURL_EASY_SETOPT_LOG(curl, CURLOPT_XOAUTH2_BEARER, e.GetBearer().c_str());
}
// set ssh pub/priv key file.
if (!e.GetPubKey().m_str.empty()) {
CURL_EASY_SETOPT_LOG(curl, CURLOPT_SSH_PUBLIC_KEYFILE, e.GetPubKey().m_str.c_str());
if (!e.GetPubKey().empty()) {
CURL_EASY_SETOPT_LOG(curl, CURLOPT_SSH_PUBLIC_KEYFILE, e.GetPubKey().c_str());
}
if (!e.GetPrivKey().m_str.empty()) {
CURL_EASY_SETOPT_LOG(curl, CURLOPT_SSH_PRIVATE_KEYFILE, e.GetPrivKey().m_str.c_str());
if (!e.GetPrivKey().empty()) {
CURL_EASY_SETOPT_LOG(curl, CURLOPT_SSH_PRIVATE_KEYFILE, e.GetPrivKey().c_str());
}
// set auth.
@@ -540,13 +589,55 @@ void SetCommonCurlOptions(CURL* curl, const Api& e) {
}
// set port, if valid.
if (e.GetPort().m_port) {
CURL_EASY_SETOPT_LOG(curl, CURLOPT_PORT, (long)e.GetPort().m_port);
if (e.GetPort()) {
CURL_EASY_SETOPT_LOG(curl, CURLOPT_PORT, (long)e.GetPort());
}
// progress calls.
if (e.GetOnProgress()) {
CURL_EASY_SETOPT_LOG(curl, CURLOPT_XFERINFODATA, &e);
CURL_EASY_SETOPT_LOG(curl, CURLOPT_XFERINFOFUNCTION, ProgressCallbackFunc2);
} else {
CURL_EASY_SETOPT_LOG(curl, CURLOPT_XFERINFOFUNCTION, ProgressCallbackFunc1);
}
CURL_EASY_SETOPT_LOG(curl, CURLOPT_NOPROGRESS, 0L);
}
auto EscapeString(CURL* curl, const std::string& str) -> std::string {
char* s{};
if (!curl) {
s = curl_escape(str.data(), str.length());
} else {
s = curl_easy_escape(curl, str.data(), str.length());
}
if (!s) {
return str;
}
const std::string result = s;
curl_free(s);
return result;
}
auto EncodeUrl(const std::string& url) -> std::string {
auto clu = curl_url();
R_UNLESS(clu, url);
ON_SCOPE_EXIT(curl_url_cleanup(clu));
CURLUcode clu_code;
clu_code = curl_url_set(clu, CURLUPART_URL, url.c_str(), CURLU_URLENCODE);
R_UNLESS(clu_code == CURLUE_OK, url);
char* encoded_url;
clu_code = curl_url_get(clu, CURLUPART_URL, &encoded_url, 0);
R_UNLESS(clu_code == CURLUE_OK, url);
const std::string out = encoded_url;
curl_free(encoded_url);
return out;
}
auto DownloadInternal(CURL* curl, const Api& e) -> ApiResult {
// check if stop has been requested before starting download
if (e.GetToken().stop_requested()) {
@@ -622,15 +713,6 @@ auto DownloadInternal(CURL* curl, const Api& e) -> ApiResult {
CURL_EASY_SETOPT_LOG(curl, CURLOPT_HTTPHEADER, list);
}
// progress calls.
if (e.GetOnProgress()) {
CURL_EASY_SETOPT_LOG(curl, CURLOPT_XFERINFODATA, &e);
CURL_EASY_SETOPT_LOG(curl, CURLOPT_XFERINFOFUNCTION, ProgressCallbackFunc2);
} else {
CURL_EASY_SETOPT_LOG(curl, CURLOPT_XFERINFOFUNCTION, ProgressCallbackFunc1);
}
CURL_EASY_SETOPT_LOG(curl, CURLOPT_NOPROGRESS, 0L);
// write calls.
CURL_EASY_SETOPT_LOG(curl, CURLOPT_WRITEFUNCTION, has_file ? WriteFileCallback : WriteMemoryCallback);
CURL_EASY_SETOPT_LOG(curl, CURLOPT_WRITEDATA, &chunk);
@@ -699,10 +781,10 @@ auto UploadInternal(CURL* curl, const Api& e) -> ApiResult {
UploadStruct chunk{};
DataStruct chunk_out{};
SeekCustomData seek_data{};
Header header_in = e.GetHeader();
Header header_out;
fs::FsNativeSd fs{};
s64 upload_size{};
if (has_file) {
if (R_FAILED(fs.OpenFile(e.GetPath(), FsOpenMode_Read, &chunk.f))) {
@@ -710,14 +792,14 @@ auto UploadInternal(CURL* curl, const Api& e) -> ApiResult {
return {};
}
fsFileGetSize(&chunk.f, &upload_size);
log_write("got chunk size: %zd\n", upload_size);
fsFileGetSize(&chunk.f, &chunk.size);
log_write("got chunk size: %zd\n", chunk.size);
} else {
if (info.m_callback) {
upload_size = info.m_size;
log_write("setting upload size: %zu\n", upload_size);
chunk.size = info.m_size;
log_write("setting upload size: %zu\n", chunk.size);
} else {
upload_size = info.m_data.size();
chunk.size = info.m_data.size();
chunk.data = info.m_data;
}
}
@@ -737,12 +819,20 @@ auto UploadInternal(CURL* curl, const Api& e) -> ApiResult {
curl_easy_reset(curl);
SetCommonCurlOptions(curl, e);
CURL_EASY_SETOPT_LOG(curl, CURLOPT_URL, url.c_str());
// encode url
auto clu = curl_url();
R_UNLESS(clu, {});
ON_SCOPE_EXIT(curl_url_cleanup(clu));
const auto clu_code = curl_url_set(clu, CURLUPART_URL, url.c_str(), CURLU_URLENCODE);
R_UNLESS(clu_code == CURLUE_OK, {});
CURL_EASY_SETOPT_LOG(curl, CURLOPT_CURLU, clu);
CURL_EASY_SETOPT_LOG(curl, CURLOPT_HEADERFUNCTION, header_callback);
CURL_EASY_SETOPT_LOG(curl, CURLOPT_HEADERDATA, &header_out);
CURL_EASY_SETOPT_LOG(curl, CURLOPT_UPLOAD, 1L);
CURL_EASY_SETOPT_LOG(curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)upload_size);
CURL_EASY_SETOPT_LOG(curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)chunk.size);
// instruct libcurl to create ftp folders if they don't yet exist.
CURL_EASY_SETOPT_LOG(curl, CURLOPT_FTP_CREATE_MISSING_DIRS, CURLFTP_CREATE_DIR_RETRY);
@@ -776,17 +866,20 @@ auto UploadInternal(CURL* curl, const Api& e) -> ApiResult {
if (info.m_callback) {
CURL_EASY_SETOPT_LOG(curl, CURLOPT_READFUNCTION, ReadCustomCallback);
CURL_EASY_SETOPT_LOG(curl, CURLOPT_READDATA, &info);
if (e.GetOnUploadSeek()) {
seek_data.cb = e.GetOnUploadSeek();
seek_data.size = chunk.size;
CURL_EASY_SETOPT_LOG(curl, CURLOPT_SEEKFUNCTION, SeekCustomCallback);
CURL_EASY_SETOPT_LOG(curl, CURLOPT_SEEKDATA, &seek_data);
}
} else {
CURL_EASY_SETOPT_LOG(curl, CURLOPT_READFUNCTION, has_file ? ReadFileCallback : ReadMemoryCallback);
CURL_EASY_SETOPT_LOG(curl, CURLOPT_READDATA, &chunk);
}
// progress calls.
if (e.GetOnProgress()) {
CURL_EASY_SETOPT_LOG(curl, CURLOPT_XFERINFODATA, &e);
CURL_EASY_SETOPT_LOG(curl, CURLOPT_XFERINFOFUNCTION, ProgressCallbackFunc2);
} else {
CURL_EASY_SETOPT_LOG(curl, CURLOPT_XFERINFOFUNCTION, ProgressCallbackFunc1);
// allow for seeking upon uploads, may be used for ftp and http.
CURL_EASY_SETOPT_LOG(curl, CURLOPT_SEEKFUNCTION, SeekCallback);
CURL_EASY_SETOPT_LOG(curl, CURLOPT_SEEKDATA, &chunk);
}
// write calls.
@@ -1013,13 +1106,7 @@ auto FromFileAsync(const Api& e) -> bool {
}
auto EscapeString(const std::string& str) -> std::string {
std::string result;
const auto s = curl_escape(str.data(), str.length());
if (s) {
result = s;
curl_free(s);
}
return result;
return EscapeString(nullptr, str);
}
} // namespace sphaira::curl

6
sphaira/source/ui/menus/game_menu.cpp
View File

@@ -426,6 +426,12 @@ Result DumpNspToNetwork(ProgressBox* pbox, const location::Entry& loc, std::span
offset += bytes_read;
return bytes_read;
}
},
curl::OnUploadSeek{
[&e, &offset](s64 new_offset){
offset = new_offset;
return true;
}
}
);