From 390c1e870d6aa9c3321e314878f9fe61a6c3246f Mon Sep 17 00:00:00 2001 From: ITotalJustice <47043333+ITotalJustice@users.noreply.github.com> Date: Fri, 30 May 2025 12:34:29 +0100 Subject: [PATCH 1/3] multi-thread zip and unzip code. option to download appstore zip to mem. hasher mem support. --- sphaira/include/hasher.hpp | 2 + sphaira/include/threaded_file_transfer.hpp | 30 +- sphaira/source/hasher.cpp | 24 ++ sphaira/source/threaded_file_transfer.cpp | 339 ++++++++++++++++----- sphaira/source/ui/menus/appstore.cpp | 229 +++++++------- sphaira/source/ui/menus/filebrowser.cpp | 105 +------ sphaira/source/ui/menus/ghdl.cpp | 75 +---- sphaira/source/ui/menus/main_menu.cpp | 110 ++----- sphaira/source/ui/menus/themezer.cpp | 64 +--- 9 files changed, 468 insertions(+), 510 deletions(-) diff --git a/sphaira/include/hasher.hpp b/sphaira/include/hasher.hpp index caec499..d10505f 100644 --- a/sphaira/include/hasher.hpp +++ b/sphaira/include/hasher.hpp @@ -4,6 +4,7 @@ #include "ui/progress_box.hpp" #include #include +#include #include namespace sphaira::hash { @@ -26,5 +27,6 @@ auto GetTypeStr(Type type) -> const char*; // returns the hash string. Result Hash(ui::ProgressBox* pbox, Type type, std::shared_ptr source, std::string& out); Result Hash(ui::ProgressBox* pbox, Type type, fs::Fs* fs, const fs::FsPath& path, std::string& out); +Result Hash(ui::ProgressBox* pbox, Type type, std::span data, std::string& out); } // namespace sphaira::hash diff --git a/sphaira/include/threaded_file_transfer.hpp b/sphaira/include/threaded_file_transfer.hpp index 4a6c13a..acd4e49 100644 --- a/sphaira/include/threaded_file_transfer.hpp +++ b/sphaira/include/threaded_file_transfer.hpp @@ -6,6 +6,15 @@ namespace sphaira::thread { +enum class Mode { + // default, always multi-thread. + MultiThreaded, + // always single-thread. + SingleThreaded, + // check buffer size, if smaller, single thread. + SingleThreadedIfSmaller, +}; + using ReadCallback = std::function; using WriteCallback = std::function; @@ -23,10 +32,25 @@ using StartCallback = std::function; using StartCallback2 = std::function; // reads data from rfunc into wfunc. -Result Transfer(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, WriteCallback wfunc); +Result Transfer(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, WriteCallback wfunc, Mode mode = Mode::MultiThreaded); // reads data from rfunc, pull data from provided pull() callback. -Result TransferPull(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, StartCallback sfunc); -Result TransferPull(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, StartCallback2 sfunc); +Result TransferPull(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, StartCallback sfunc, Mode mode = Mode::MultiThreaded); +Result TransferPull(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, StartCallback2 sfunc, Mode mode = Mode::MultiThreaded); + +// helper for extract zips. +// this will multi-thread unzip if size >= 512KiB, otherwise it'll single pass. +Result TransferUnzip(ui::ProgressBox* pbox, void* zfile, fs::Fs* fs, const fs::FsPath& path, s64 size, u32 crc32 = 0); + +// same as above but for zipping files. +Result TransferZip(ui::ProgressBox* pbox, void* zfile, fs::Fs* fs, const fs::FsPath& path); + +// passes the name inside the zip an final output path. +using UnzipAllFilter = std::function; + +// helper all-in-one unzip function that unzips a zip (either open or path provided). +// the filter function can be used to modify the path and filter out unwanted files. +Result TransferUnzipAll(ui::ProgressBox* pbox, void* zfile, fs::Fs* fs, const fs::FsPath& base_path, UnzipAllFilter filter = nullptr); +Result TransferUnzipAll(ui::ProgressBox* pbox, const fs::FsPath& zip_out, fs::Fs* fs, const fs::FsPath& base_path, UnzipAllFilter filter = nullptr); } // namespace sphaira::thread diff --git a/sphaira/source/hasher.cpp b/sphaira/source/hasher.cpp index da48249..3fda5b3 100644 --- a/sphaira/source/hasher.cpp +++ b/sphaira/source/hasher.cpp @@ -35,6 +35,25 @@ private: bool m_is_file_based_emummc{}; }; +struct MemSource final : BaseSource { + MemSource(std::span data) : m_data{data} { } + + Result Size(s64* out) { + *out = m_data.size(); + R_SUCCEED(); + } + + Result Read(void* buf, s64 off, s64 size, u64* bytes_read) { + size = std::min(size, m_data.size() - off); + std::memcpy(buf, m_data.data() + off, size); + *bytes_read = size; + R_SUCCEED(); + } + +private: + const std::span m_data; +}; + struct HashSource { virtual ~HashSource() = default; virtual void Update(const void* buf, s64 size) = 0; @@ -181,4 +200,9 @@ Result Hash(ui::ProgressBox* pbox, Type type, fs::Fs* fs, const fs::FsPath& path return Hash(pbox, type, source, out); } +Result Hash(ui::ProgressBox* pbox, Type type, std::span data, std::string& out) { + auto source = std::make_shared(data); + return Hash(pbox, type, source, out); +} + } // namespace sphaira::has diff --git a/sphaira/source/threaded_file_transfer.cpp b/sphaira/source/threaded_file_transfer.cpp index 9385cbf..5daf596 100644 --- a/sphaira/source/threaded_file_transfer.cpp +++ b/sphaira/source/threaded_file_transfer.cpp @@ -6,15 +6,20 @@ #include #include #include +#include +#include namespace sphaira::thread { namespace { -constexpr u64 READ_BUFFER_MAX = 1024*1024*4; +// used for file based emummc and zip/unzip. +constexpr u64 SMALL_BUFFER_SIZE = 1024 * 512; +// used for everything else. +constexpr u64 NORMAL_BUFFER_SIZE = 1024*1024*4; struct ThreadBuffer { ThreadBuffer() { - buf.reserve(READ_BUFFER_MAX); + buf.reserve(NORMAL_BUFFER_SIZE); } std::vector buf; @@ -65,7 +70,7 @@ public: }; struct ThreadData { - ThreadData(ui::ProgressBox* _pbox, s64 size, ReadCallback _rfunc, WriteCallback _wfunc); + ThreadData(ui::ProgressBox* _pbox, s64 size, ReadCallback _rfunc, WriteCallback _wfunc, u64 buffer_size); auto GetResults() -> Result; void WakeAllThreads(); @@ -104,9 +109,9 @@ private: private: // these need to be copied - ui::ProgressBox* pbox{}; - ReadCallback rfunc{}; - WriteCallback wfunc{}; + ui::ProgressBox* const pbox; + const ReadCallback rfunc; + const WriteCallback wfunc; // these need to be created Mutex mutex{}; @@ -121,21 +126,24 @@ private: std::vector pull_buffer{}; s64 pull_buffer_offset{}; - u64 read_buffer_size{}; - u64 max_buffer_size{}; + const u64 read_buffer_size; + const s64 write_size; // these are shared between threads volatile s64 read_offset{}; volatile s64 write_offset{}; - volatile s64 write_size{}; volatile Result read_result{}; volatile Result write_result{}; volatile Result pull_result{}; }; -ThreadData::ThreadData(ui::ProgressBox* _pbox, s64 size, ReadCallback _rfunc, WriteCallback _wfunc) -: pbox{_pbox}, rfunc{_rfunc}, wfunc{_wfunc} { +ThreadData::ThreadData(ui::ProgressBox* _pbox, s64 size, ReadCallback _rfunc, WriteCallback _wfunc, u64 buffer_size) +: pbox{_pbox} +, rfunc{_rfunc} +, wfunc{_wfunc} +, read_buffer_size{buffer_size} +, write_size{size} { mutexInit(std::addressof(mutex)); mutexInit(std::addressof(pull_mutex)); @@ -143,16 +151,6 @@ ThreadData::ThreadData(ui::ProgressBox* _pbox, s64 size, ReadCallback _rfunc, Wr condvarInit(std::addressof(can_write)); condvarInit(std::addressof(can_pull)); condvarInit(std::addressof(can_pull_write)); - - write_size = size; - - if (App::IsFileBaseEmummc()) { - read_buffer_size = 1024 * 512; - max_buffer_size = 1024 * 512; - } else { - read_buffer_size = READ_BUFFER_MAX; - max_buffer_size = READ_BUFFER_MAX; - } } auto ThreadData::GetResults() -> Result { @@ -251,7 +249,7 @@ Result ThreadData::Pull(void* data, s64 size, u64* bytes_read) { Result ThreadData::readFuncInternal() { // the main buffer which data is read into. std::vector buf; - buf.reserve(this->max_buffer_size); + buf.reserve(this->read_buffer_size); while (this->read_offset < this->write_size && R_SUCCEEDED(this->GetResults())) { // read more data @@ -265,14 +263,14 @@ Result ThreadData::readFuncInternal() { R_TRY(this->SetWriteBuf(buf, buf_size)); } - log_write("read success\n"); + log_write("finished read thread success!\n"); R_SUCCEED(); } // write thread writes data to the nca placeholder. Result ThreadData::writeFuncInternal() { std::vector buf; - buf.reserve(this->max_buffer_size); + buf.reserve(this->read_buffer_size); while (this->write_offset < this->write_size && R_SUCCEEDED(this->GetResults())) { s64 dummy_off; @@ -288,7 +286,7 @@ Result ThreadData::writeFuncInternal() { this->write_offset += size; } - log_write("finished write thread!\n"); + log_write("finished write thread success!\n"); R_SUCCEED(); } @@ -308,87 +306,264 @@ auto GetAlternateCore(int id) { return id == 1 ? 2 : 1; } -Result TransferInternal(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, WriteCallback wfunc, StartCallback2 sfunc) { - App::SetAutoSleepDisabled(true); - ON_SCOPE_EXIT(App::SetAutoSleepDisabled(false)); +Result TransferInternal(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, WriteCallback wfunc, StartCallback2 sfunc, Mode mode, u64 buffer_size = NORMAL_BUFFER_SIZE) { + const auto is_file_based_emummc = App::IsFileBaseEmummc(); - const auto WRITE_THREAD_CORE = sfunc ? pbox->GetCpuId() : GetAlternateCore(pbox->GetCpuId()); - const auto READ_THREAD_CORE = GetAlternateCore(WRITE_THREAD_CORE); + if (is_file_based_emummc) { + buffer_size = SMALL_BUFFER_SIZE; + } - ThreadData t_data{pbox, size, rfunc, wfunc}; + if (mode == Mode::SingleThreadedIfSmaller) { + if (size <= buffer_size) { + mode = Mode::SingleThreaded; + } else { + mode = Mode::MultiThreaded; + } + } - Thread t_read{}; - R_TRY(threadCreate(&t_read, readFunc, std::addressof(t_data), nullptr, 1024*64, PRIO_PREEMPTIVE, READ_THREAD_CORE)); - ON_SCOPE_EXIT(threadClose(&t_read)); + // single threaded pull buffer is not supported. + R_UNLESS(mode != Mode::MultiThreaded || !sfunc, 0x1); - Thread t_write{}; - R_TRY(threadCreate(&t_write, writeFunc, std::addressof(t_data), nullptr, 1024*64, PRIO_PREEMPTIVE, WRITE_THREAD_CORE)); - ON_SCOPE_EXIT(threadClose(&t_write)); + // todo: support single threaded pull buffer. + if (mode == Mode::SingleThreaded) { + std::vector buf(buffer_size); + + s64 offset{}; + while (offset < size) { + R_TRY(pbox->ShouldExitResult()); + + u64 bytes_read; + const auto rsize = std::min(buf.size(), size - offset); + R_TRY(rfunc(buf.data(), offset, rsize, &bytes_read)); + R_TRY(wfunc(buf.data(), offset, bytes_read)); + + offset += bytes_read; + pbox->UpdateTransfer(offset, size); + } - const auto start_threads = [&]() -> Result { - log_write("starting threads\n"); - R_TRY(threadStart(std::addressof(t_read))); - R_TRY(threadStart(std::addressof(t_write))); R_SUCCEED(); - }; - - ON_SCOPE_EXIT(threadWaitForExit(std::addressof(t_read))); - ON_SCOPE_EXIT(threadWaitForExit(std::addressof(t_write))); - - if (sfunc) { - t_data.SetPullResult(sfunc(start_threads, [&](void* data, s64 size, u64* bytes_read) -> Result { - R_TRY(t_data.GetResults()); - return t_data.Pull(data, size, bytes_read); - })); - } else { - R_TRY(start_threads()); - - while (t_data.GetWriteOffset() != t_data.GetWriteSize() && R_SUCCEEDED(t_data.GetResults())) { - pbox->UpdateTransfer(t_data.GetWriteOffset(), t_data.GetWriteSize()); - svcSleepThread(1e+6); - } } + else { + const auto WRITE_THREAD_CORE = sfunc ? pbox->GetCpuId() : GetAlternateCore(pbox->GetCpuId()); + const auto READ_THREAD_CORE = GetAlternateCore(WRITE_THREAD_CORE); - // wait for all threads to close. - log_write("waiting for threads to close\n"); - for (;;) { - t_data.WakeAllThreads(); - pbox->Yield(); + ThreadData t_data{pbox, size, rfunc, wfunc, buffer_size}; - if (R_FAILED(waitSingleHandle(t_read.handle, 1000))) { - continue; - } else if (R_FAILED(waitSingleHandle(t_write.handle, 1000))) { - continue; + Thread t_read{}; + R_TRY(threadCreate(&t_read, readFunc, std::addressof(t_data), nullptr, 1024*256, 0x3B, READ_THREAD_CORE)); + ON_SCOPE_EXIT(threadClose(&t_read)); + + Thread t_write{}; + R_TRY(threadCreate(&t_write, writeFunc, std::addressof(t_data), nullptr, 1024*256, 0x3B, WRITE_THREAD_CORE)); + ON_SCOPE_EXIT(threadClose(&t_write)); + + const auto start_threads = [&]() -> Result { + log_write("starting threads\n"); + R_TRY(threadStart(std::addressof(t_read))); + R_TRY(threadStart(std::addressof(t_write))); + R_SUCCEED(); + }; + + ON_SCOPE_EXIT(threadWaitForExit(std::addressof(t_read))); + ON_SCOPE_EXIT(threadWaitForExit(std::addressof(t_write))); + + if (sfunc) { + log_write("[THREAD] doing sfuncn\n"); + t_data.SetPullResult(sfunc(start_threads, [&](void* data, s64 size, u64* bytes_read) -> Result { + R_TRY(t_data.GetResults()); + return t_data.Pull(data, size, bytes_read); + })); } - break; - } - log_write("threads closed\n"); + else { + log_write("[THREAD] doing normal\n"); + R_TRY(start_threads()); + log_write("[THREAD] started threads\n"); - // if any of the threads failed, wake up all threads so they can exit. - if (R_FAILED(t_data.GetResults())) { - log_write("some reads failed, waking threads\n"); - log_write("returning due to fail\n"); + while (t_data.GetWriteOffset() != t_data.GetWriteSize() && R_SUCCEEDED(t_data.GetResults())) { + pbox->UpdateTransfer(t_data.GetWriteOffset(), t_data.GetWriteSize()); + svcSleepThread(1e+6); + } + } + + // wait for all threads to close. + log_write("waiting for threads to close\n"); + for (;;) { + t_data.WakeAllThreads(); + pbox->Yield(); + + if (R_FAILED(waitSingleHandle(t_read.handle, 1000))) { + continue; + } else if (R_FAILED(waitSingleHandle(t_write.handle, 1000))) { + continue; + } + break; + } + log_write("threads closed\n"); + + // if any of the threads failed, wake up all threads so they can exit. + if (R_FAILED(t_data.GetResults())) { + log_write("some reads failed, waking threads\n"); + log_write("returning due to fail\n"); + return t_data.GetResults(); + } + + log_write("returning from thread func\n"); return t_data.GetResults(); } - - return t_data.GetResults(); } } // namespace -Result Transfer(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, WriteCallback wfunc) { - return TransferInternal(pbox, size, rfunc, wfunc, nullptr); +Result Transfer(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, WriteCallback wfunc, Mode mode) { + return TransferInternal(pbox, size, rfunc, wfunc, nullptr, Mode::MultiThreaded); } -Result TransferPull(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, StartCallback sfunc) { +Result TransferPull(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, StartCallback sfunc, Mode mode) { return TransferInternal(pbox, size, rfunc, nullptr, [sfunc](StartThreadCallback start, PullCallback pull) -> Result { R_TRY(start()); return sfunc(pull); - }); + }, Mode::MultiThreaded); } -Result TransferPull(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, StartCallback2 sfunc) { - return TransferInternal(pbox, size, rfunc, nullptr, sfunc); +Result TransferPull(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, StartCallback2 sfunc, Mode mode) { + return TransferInternal(pbox, size, rfunc, nullptr, sfunc, Mode::MultiThreaded); +} + +Result TransferUnzip(ui::ProgressBox* pbox, void* zfile, fs::Fs* fs, const fs::FsPath& path, s64 size, u32 crc32) { + Result rc; + if (R_FAILED(rc = fs->CreateDirectoryRecursivelyWithPath(path)) && rc != FsError_PathAlreadyExists) { + log_write("failed to create folder: %s 0x%04X\n", path.s, rc); + R_THROW(rc); + } + + if (R_FAILED(rc = fs->CreateFile(path, size, 0)) && rc != FsError_PathAlreadyExists) { + log_write("failed to create file: %s 0x%04X\n", path.s, rc); + R_THROW(rc); + } + + fs::File f; + R_TRY(fs->OpenFile(path, FsOpenMode_Write, &f)); + + // only update the size if this is an existing file. + if (rc == FsError_PathAlreadyExists) { + R_TRY(f.SetSize(size)); + } + + // NOTES: do not use temp file with rename / delete after as it massively slows + // down small file transfers (RA 21s -> 50s). + u32 crc32_out{}; + R_TRY(thread::TransferInternal(pbox, size, + [&](void* data, s64 off, s64 size, u64* bytes_read) -> Result { + const auto result = unzReadCurrentFile(zfile, data, size); + if (result <= 0) { + // log_write("failed to read zip file: %s\n", inzip.c_str()); + R_THROW(0x1); + } + + if (crc32) { + crc32_out = crc32CalculateWithSeed(crc32_out, data, result); + } + + *bytes_read = result; + R_SUCCEED(); + }, + [&](const void* data, s64 off, s64 size) -> Result { + return f.Write(off, data, size, FsWriteOption_None); + }, + nullptr, Mode::SingleThreadedIfSmaller, SMALL_BUFFER_SIZE + )); + + // validate crc32 (if set in the info). + R_UNLESS(!crc32 || crc32 == crc32_out, 0x1); + + R_SUCCEED(); +} + +Result TransferZip(ui::ProgressBox* pbox, void* zfile, fs::Fs* fs, const fs::FsPath& path) { + fs::File f; + R_TRY(fs->OpenFile(path, FsOpenMode_Read, &f)); + + s64 file_size; + R_TRY(f.GetSize(&file_size)); + + return thread::TransferInternal(pbox, file_size, + [&](void* data, s64 off, s64 size, u64* bytes_read) -> Result { + return f.Read(off, data, size, FsReadOption_None, bytes_read); + }, + [&](const void* data, s64 off, s64 size) -> Result { + if (ZIP_OK != zipWriteInFileInZip(zfile, data, size)) { + log_write("failed to write zip file: %s\n", path.s); + R_THROW(0x1); + } + R_SUCCEED(); + }, + nullptr, Mode::SingleThreadedIfSmaller, SMALL_BUFFER_SIZE + ); +} + +Result TransferUnzipAll(ui::ProgressBox* pbox, void* zfile, fs::Fs* fs, const fs::FsPath& base_path, UnzipAllFilter filter) { + unz_global_info64 ginfo; + if (UNZ_OK != unzGetGlobalInfo64(zfile, &ginfo)) { + R_THROW(0x1); + } + + if (UNZ_OK != unzGoToFirstFile(zfile)) { + R_THROW(0x1); + } + + for (s64 i = 0; i < ginfo.number_entry; i++) { + R_TRY(pbox->ShouldExitResult()); + + if (i > 0) { + if (UNZ_OK != unzGoToNextFile(zfile)) { + log_write("failed to unzGoToNextFile\n"); + R_THROW(0x1); + } + } + + if (UNZ_OK != unzOpenCurrentFile(zfile)) { + log_write("failed to open current file\n"); + R_THROW(0x1); + } + ON_SCOPE_EXIT(unzCloseCurrentFile(zfile)); + + unz_file_info64 info; + fs::FsPath name; + if (UNZ_OK != unzGetCurrentFileInfo64(zfile, &info, name, sizeof(name), 0, 0, 0, 0)) { + log_write("failed to get current info\n"); + R_THROW(0x1); + } + + // check if we should skip this file. + // don't make const as to allow the function to modify the path + // this function is used for the updater to change sphaira.nro to exe path. + auto path = fs::AppendPath(base_path, name); + if (filter && !filter(name, path)) { + continue; + } + + pbox->NewTransfer(name); + + if (path[std::strlen(path) -1] == '/') { + Result rc; + if (R_FAILED(rc = fs->CreateDirectoryRecursively(path)) && rc != FsError_PathAlreadyExists) { + log_write("failed to create folder: %s 0x%04X\n", path.s, rc); + R_THROW(rc); + } + } else { + R_TRY(TransferUnzip(pbox, zfile, fs, path, info.uncompressed_size, info.crc)); + } + } + + R_SUCCEED(); +} + +Result TransferUnzipAll(ui::ProgressBox* pbox, const fs::FsPath& zip_out, fs::Fs* fs, const fs::FsPath& base_path, UnzipAllFilter filter) { + auto zfile = unzOpen64(zip_out); + R_UNLESS(zfile, 0x1); + ON_SCOPE_EXIT(unzClose(zfile)); + + return TransferUnzipAll(pbox, zfile, fs, base_path, filter); } } // namespace::thread diff --git a/sphaira/source/ui/menus/appstore.cpp b/sphaira/source/ui/menus/appstore.cpp index f5acdfb..68106aa 100644 --- a/sphaira/source/ui/menus/appstore.cpp +++ b/sphaira/source/ui/menus/appstore.cpp @@ -14,6 +14,7 @@ #include "swkbd.hpp" #include "i18n.hpp" #include "hasher.hpp" +#include "threaded_file_transfer.hpp" #include "nro.hpp" #include @@ -75,6 +76,62 @@ constexpr const char* ORDER_STR[] = { "Asc", }; +struct MzMem { + const void* buf; + size_t size; + size_t offset; +}; + +ZPOS64_T minizip_tell_file_func(voidpf opaque, voidpf stream) { + auto mem = static_cast(opaque); + return mem->offset; +} + +long minizip_seek_file_func(voidpf opaque, voidpf stream, ZPOS64_T offset, int origin) { + auto mem = static_cast(opaque); + size_t new_offset = 0; + + switch (origin) { + case ZLIB_FILEFUNC_SEEK_SET: new_offset = offset; break; + case ZLIB_FILEFUNC_SEEK_CUR: new_offset = mem->offset + offset; break; + case ZLIB_FILEFUNC_SEEK_END: new_offset = (mem->size - 1) + offset; break; + default: return -1; + } + + if (new_offset > mem->size) { + return -1; + } + + mem->offset = new_offset; + return 0; +} + +voidpf minizip_open_file_func(voidpf opaque, const void* filename, int mode) { + return opaque; +} + +uLong minizip_read_file_func(voidpf opaque, voidpf stream, void* buf, uLong size) { + auto mem = static_cast(opaque); + + size = std::min(size, mem->size - mem->offset); + std::memcpy(buf, (const u8*)mem->buf + mem->offset, size); + mem->offset += size; + + return size; +} + +int minizip_close_file_func(voidpf opaque, voidpf stream) { + return 0; +} + +constexpr zlib_filefunc64_def zlib_filefunc = { + .zopen64_file = minizip_open_file_func, + .zread_file = minizip_read_file_func, + .ztell64_file = minizip_tell_file_func, + .zseek64_file = minizip_seek_file_func, + .zclose_file = minizip_close_file_func, +}; + auto BuildIconUrl(const Entry& e) -> std::string { char out[0x100]; std::snprintf(out, sizeof(out), "%s/packages/%s/icon.png", URL_BASE, e.name.c_str()); @@ -363,19 +420,30 @@ auto InstallApp(ProgressBox* pbox, const Entry& entry) -> Result { fs::FsNativeSd fs; R_TRY(fs.GetFsOpenResult()); + // check if we can download the entire zip to mem for faster download / extract times. + // current limit is 300MiB, or disabled for applet mode. + const auto file_download = App::IsApplet() || entry.filesize >= 1024 * 1024 * 300; + curl::ApiResult api_result{}; + // 1. download the zip if (!pbox->ShouldExit()) { pbox->NewTransfer("Downloading "_i18n + entry.title); log_write("starting download\n"); const auto url = BuildZipUrl(entry); - const auto result = curl::Api().ToFile( + curl::Api api{ curl::Url{url}, - curl::Path{zip_out}, curl::OnProgress{pbox->OnDownloadProgressCallback()} - ); + }; - R_UNLESS(result.success, 0x1); + if (file_download) { + api.SetOption(curl::Path{zip_out}); + api_result = curl::ToFile(api); + } else { + api_result = curl::ToMemory(api); + } + + R_UNLESS(api_result.success, 0x1); } ON_SCOPE_EXIT(fs.DeleteFile(zip_out)); @@ -386,7 +454,11 @@ auto InstallApp(ProgressBox* pbox, const Entry& entry) -> Result { log_write("starting md5 check\n"); std::string hash_out; - R_TRY(hash::Hash(pbox, hash::Type::Md5, &fs, zip_out, hash_out)); + if (file_download) { + R_TRY(hash::Hash(pbox, hash::Type::Md5, &fs, zip_out, hash_out)); + } else { + R_TRY(hash::Hash(pbox, hash::Type::Md5, api_result.data, hash_out)); + } if (strncasecmp(hash_out.data(), entry.md5.data(), entry.md5.length())) { log_write("bad md5: %.*s vs %.*s\n", 32, hash_out.data(), 32, entry.md5.c_str()); @@ -394,9 +466,20 @@ auto InstallApp(ProgressBox* pbox, const Entry& entry) -> Result { } } + struct MzMem mem{}; + mem.buf = api_result.data.data(); + mem.size = api_result.data.size(); + auto file_func = zlib_filefunc; + file_func.opaque = &mem; + + zlib_filefunc64_def* file_func_ptr{}; + if (!file_download) { + file_func_ptr = &file_func; + } + // 3. extract the zip if (!pbox->ShouldExit()) { - auto zfile = unzOpen64(zip_out); + auto zfile = unzOpen2_64(zip_out, file_func_ptr); R_UNLESS(zfile, 0x1); ON_SCOPE_EXIT(unzClose(zfile)); @@ -434,43 +517,6 @@ auto InstallApp(ProgressBox* pbox, const Entry& entry) -> Result { } } - const auto unzip_to_file = [&](const unz_file_info64& info, const fs::FsPath& inzip, fs::FsPath output) -> Result { - if (output[0] != '/') { - output = fs::AppendPath("/", output); - } - - // create directories - fs.CreateDirectoryRecursivelyWithPath(output); - - Result rc; - if (R_FAILED(rc = fs.CreateFile(output, info.uncompressed_size, 0)) && rc != FsError_PathAlreadyExists) { - log_write("failed to create file: %s 0x%04X\n", output.s, rc); - R_THROW(rc); - } - - fs::File f; - R_TRY(fs.OpenFile(output, FsOpenMode_Write, &f)); - R_TRY(f.SetSize(info.uncompressed_size)); - - u64 offset{}; - while (offset < info.uncompressed_size) { - R_TRY(pbox->ShouldExitResult()); - - const auto bytes_read = unzReadCurrentFile(zfile, buf.data(), buf.size()); - if (bytes_read <= 0) { - log_write("failed to read zip file: %s\n", inzip.s); - R_THROW(0x1); - } - - R_TRY(f.Write(offset, buf.data(), bytes_read, FsWriteOption_None)); - - pbox->UpdateTransfer(offset, info.uncompressed_size); - offset += bytes_read; - } - - R_SUCCEED(); - }; - const auto unzip_to = [&](const fs::FsPath& inzip, const fs::FsPath& output) -> Result { pbox->NewTransfer(inzip); @@ -491,79 +537,46 @@ auto InstallApp(ProgressBox* pbox, const Entry& entry) -> Result { R_THROW(0x1); } - return unzip_to_file(info, inzip, output); - }; - - const auto unzip_all = [&](std::span entries) -> Result { - unz_global_info64 ginfo; - if (UNZ_OK != unzGetGlobalInfo64(zfile, &ginfo)) { - R_THROW(0x1); + auto path = output; + if (path[0] != '/') { + path = fs::AppendPath("/", path); } - if (UNZ_OK != unzGoToFirstFile(zfile)) { - R_THROW(0x1); - } - - for (s64 i = 0; i < ginfo.number_entry; i++) { - R_TRY(pbox->ShouldExitResult()); - - if (i > 0) { - if (UNZ_OK != unzGoToNextFile(zfile)) { - log_write("failed to unzGoToNextFile\n"); - R_THROW(0x1); - } - } - - if (UNZ_OK != unzOpenCurrentFile(zfile)) { - log_write("failed to open current file\n"); - R_THROW(0x1); - } - ON_SCOPE_EXIT(unzCloseCurrentFile(zfile)); - - unz_file_info64 info; - char name[512]; - if (UNZ_OK != unzGetCurrentFileInfo64(zfile, &info, name, sizeof(name), 0, 0, 0, 0)) { - log_write("failed to get current info\n"); - R_THROW(0x1); - } - - const auto it = std::ranges::find_if(entries, [&name](auto& e){ - return !strcasecmp(name, e.path); - }); - - if (it == entries.end()) [[unlikely]] { - continue; - } - - pbox->NewTransfer(it->path); - - switch (it->command) { - case 'E': // both are the same? - case 'U': - break; - - case 'G': { // checks if file exists, if not, extract - if (fs.FileExists(fs::AppendPath("/", it->path))) { - continue; - } - } break; - - default: - log_write("bad command: %c\n", it->command); - continue; - } - - R_TRY(unzip_to_file(info, it->path, it->path)); - } - - R_SUCCEED(); + return thread::TransferUnzip(pbox, zfile, &fs, path, info.uncompressed_size, info.crc); }; // unzip manifest, info and all entries. TimeStamp ts; + #if 1 R_TRY(unzip_to("info.json", BuildInfoCachePath(entry))); R_TRY(unzip_to("manifest.install", BuildManifestCachePath(entry))); - R_TRY(unzip_all(new_manifest)); + #endif + + R_TRY(thread::TransferUnzipAll(pbox, zfile, &fs, "/", [&](const fs::FsPath& name, fs::FsPath& path) -> bool { + const auto it = std::ranges::find_if(new_manifest, [&name](auto& e){ + return !strcasecmp(name, e.path); + }); + + if (it == new_manifest.end()) [[unlikely]] { + return false; + } + + pbox->NewTransfer(it->path); + + switch (it->command) { + case 'E': // both are the same? + case 'U': + return true; + + case 'G': // checks if file exists, if not, extract + return !fs.FileExists(fs::AppendPath("/", it->path)); + + default: + log_write("bad command: %c\n", it->command); + return false; + } + })); + log_write("\n\t[APPSTORE] finished extract new, time taken: %.2fs %zums\n\n", ts.GetSecondsD(), ts.GetMs()); // finally finally, remove files no longer in the manifest diff --git a/sphaira/source/ui/menus/filebrowser.cpp b/sphaira/source/ui/menus/filebrowser.cpp index 1a90239..7e3f72e 100644 --- a/sphaira/source/ui/menus/filebrowser.cpp +++ b/sphaira/source/ui/menus/filebrowser.cpp @@ -708,75 +708,10 @@ void FsView::UnzipFiles(fs::FsPath dir_path) { } App::Push(std::make_shared(0, "Extracting "_i18n, "", [this, dir_path, targets](auto pbox) -> Result { - constexpr auto chunk_size = 1024 * 512; // 512KiB - for (auto& e : targets) { pbox->SetTitle(e.GetName()); - const auto zip_out = GetNewPath(e); - auto zfile = unzOpen64(zip_out); - R_UNLESS(zfile, 0x1); - ON_SCOPE_EXIT(unzClose(zfile)); - - unz_global_info64 pglobal_info; - if (UNZ_OK != unzGetGlobalInfo64(zfile, &pglobal_info)) { - R_THROW(0x1); - } - - for (s64 i = 0; i < pglobal_info.number_entry; i++) { - if (i > 0) { - if (UNZ_OK != unzGoToNextFile(zfile)) { - log_write("failed to unzGoToNextFile\n"); - R_THROW(0x1); - } - } - - if (UNZ_OK != unzOpenCurrentFile(zfile)) { - log_write("failed to open current file\n"); - R_THROW(0x1); - } - ON_SCOPE_EXIT(unzCloseCurrentFile(zfile)); - - unz_file_info64 info; - char name[512]; - if (UNZ_OK != unzGetCurrentFileInfo64(zfile, &info, name, sizeof(name), 0, 0, 0, 0)) { - log_write("failed to get current info\n"); - R_THROW(0x1); - } - - const auto file_path = fs::AppendPath(dir_path, name); - pbox->NewTransfer(name); - - // create directories - m_fs->CreateDirectoryRecursivelyWithPath(file_path); - - Result rc; - if (R_FAILED(rc = m_fs->CreateFile(file_path, info.uncompressed_size, 0)) && rc != FsError_PathAlreadyExists) { - log_write("failed to create file: %s 0x%04X\n", file_path.s, rc); - R_THROW(rc); - } - - fs::File f; - R_TRY(m_fs->OpenFile(file_path, FsOpenMode_Write, &f)); - R_TRY(f.SetSize(info.uncompressed_size)); - - std::vector buf(chunk_size); - s64 offset{}; - while (offset < info.uncompressed_size) { - R_TRY(pbox->ShouldExitResult()); - - const auto bytes_read = unzReadCurrentFile(zfile, buf.data(), buf.size()); - if (bytes_read <= 0) { - log_write("failed to read zip file: %s\n", name); - R_THROW(0x1); - } - - R_TRY(f.Write(offset, buf.data(), bytes_read, FsWriteOption_None)); - - pbox->UpdateTransfer(offset, info.uncompressed_size); - offset += bytes_read; - } - } + R_TRY(thread::TransferUnzipAll(pbox, zip_out, m_fs.get(), dir_path)); } R_SUCCEED(); @@ -829,8 +764,6 @@ void FsView::ZipFiles(fs::FsPath zip_out) { } App::Push(std::make_shared(0, "Compressing "_i18n, "", [this, zip_out, targets](auto pbox) -> Result { - constexpr auto chunk_size = 1024 * 512; // 512KiB - const auto t = std::time(NULL); const auto tm = std::localtime(&t); @@ -851,6 +784,11 @@ void FsView::ZipFiles(fs::FsPath zip_out) { // the file name needs to be relative to the current directory. const char* file_name_in_zip = file_path.s + std::strlen(m_path); + // strip root path (/ or ums0:) + if (!std::strncmp(file_name_in_zip, m_fs->Root(), std::strlen(m_fs->Root()))) { + file_name_in_zip += std::strlen(m_fs->Root()); + } + // root paths are banned in zips, they will warn when extracting otherwise. if (file_name_in_zip[0] == '/') { file_name_in_zip++; @@ -858,38 +796,13 @@ void FsView::ZipFiles(fs::FsPath zip_out) { pbox->NewTransfer(file_name_in_zip); - const auto ext = std::strrchr(file_name_in_zip, '.'); - const auto raw = ext && IsExtension(ext + 1, COMPRESSED_EXTENSIONS); - - if (ZIP_OK != zipOpenNewFileInZip2(zfile, file_name_in_zip, &zip_info, NULL, 0, NULL, 0, NULL, Z_DEFLATED, Z_DEFAULT_COMPRESSION, raw)) { + if (ZIP_OK != zipOpenNewFileInZip(zfile, file_name_in_zip, &zip_info, NULL, 0, NULL, 0, NULL, Z_DEFLATED, Z_DEFAULT_COMPRESSION)) { + log_write("failed to add zip for %s\n", file_path.s); R_THROW(0x1); } ON_SCOPE_EXIT(zipCloseFileInZip(zfile)); - fs::File f; - R_TRY(m_fs->OpenFile(file_path, FsOpenMode_Read, &f)); - - s64 file_size; - R_TRY(f.GetSize(&file_size)); - - std::vector buf(chunk_size); - s64 offset{}; - while (offset < file_size) { - R_TRY(pbox->ShouldExitResult()); - - u64 bytes_read; - R_TRY(f.Read(offset, buf.data(), buf.size(), FsReadOption_None, &bytes_read)); - - if (ZIP_OK != zipWriteInFileInZip(zfile, buf.data(), bytes_read)) { - log_write("failed to write zip file: %s\n", file_path.s); - R_THROW(0x1); - } - - pbox->UpdateTransfer(offset, file_size); - offset += bytes_read; - } - - R_SUCCEED(); + return thread::TransferZip(pbox, zfile, m_fs.get(), file_path); }; for (auto& e : targets) { diff --git a/sphaira/source/ui/menus/ghdl.cpp b/sphaira/source/ui/menus/ghdl.cpp index 93ef510..db92d36 100644 --- a/sphaira/source/ui/menus/ghdl.cpp +++ b/sphaira/source/ui/menus/ghdl.cpp @@ -14,9 +14,9 @@ #include "download.hpp" #include "i18n.hpp" #include "yyjson_helper.hpp" +#include "threaded_file_transfer.hpp" #include -#include #include #include #include @@ -83,7 +83,6 @@ void from_json(const fs::FsPath& path, GhApiEntry& e) { auto DownloadApp(ProgressBox* pbox, const GhApiAsset& gh_asset, const AssetEntry* entry) -> Result { static const fs::FsPath temp_file{"/switch/sphaira/cache/github/ghdl.temp"}; - constexpr auto chunk_size = 1024 * 512; // 512KiB fs::FsNativeSd fs; R_TRY(fs.GetFsOpenResult()); @@ -113,77 +112,7 @@ auto DownloadApp(ProgressBox* pbox, const GhApiAsset& gh_asset, const AssetEntry // 3. extract the zip / file if (gh_asset.content_type.find("zip") != gh_asset.content_type.npos) { log_write("found zip\n"); - auto zfile = unzOpen64(temp_file); - R_UNLESS(zfile, 0x1); - ON_SCOPE_EXIT(unzClose(zfile)); - - unz_global_info64 pglobal_info; - if (UNZ_OK != unzGetGlobalInfo64(zfile, &pglobal_info)) { - R_THROW(0x1); - } - - for (int i = 0; i < pglobal_info.number_entry; i++) { - if (i > 0) { - if (UNZ_OK != unzGoToNextFile(zfile)) { - log_write("failed to unzGoToNextFile\n"); - R_THROW(0x1); - } - } - - if (UNZ_OK != unzOpenCurrentFile(zfile)) { - log_write("failed to open current file\n"); - R_THROW(0x1); - } - ON_SCOPE_EXIT(unzCloseCurrentFile(zfile)); - - unz_file_info64 info; - fs::FsPath file_path; - if (UNZ_OK != unzGetCurrentFileInfo64(zfile, &info, file_path, sizeof(file_path), 0, 0, 0, 0)) { - log_write("failed to get current info\n"); - R_THROW(0x1); - } - - file_path = fs::AppendPath(root_path, file_path); - - Result rc; - if (file_path[strlen(file_path) -1] == '/') { - if (R_FAILED(rc = fs.CreateDirectoryRecursively(file_path)) && rc != FsError_PathAlreadyExists) { - log_write("failed to create folder: %s 0x%04X\n", file_path.s, rc); - R_THROW(rc); - } - } else { - if (R_FAILED(rc = fs.CreateDirectoryRecursivelyWithPath(file_path)) && rc != FsError_PathAlreadyExists) { - log_write("failed to create folder: %s 0x%04X\n", file_path.s, rc); - R_THROW(rc); - } - - if (R_FAILED(rc = fs.CreateFile(file_path, info.uncompressed_size, 0)) && rc != FsError_PathAlreadyExists) { - log_write("failed to create file: %s 0x%04X\n", file_path.s, rc); - R_THROW(rc); - } - - fs::File f; - R_TRY(fs.OpenFile(file_path, FsOpenMode_Write, &f)); - R_TRY(f.SetSize(info.uncompressed_size)); - - std::vector buf(chunk_size); - s64 offset{}; - while (offset < info.uncompressed_size) { - R_TRY(pbox->ShouldExitResult()); - - const auto bytes_read = unzReadCurrentFile(zfile, buf.data(), buf.size()); - if (bytes_read <= 0) { - log_write("failed to read zip file: %s\n", file_path.s); - R_THROW(0x1); - } - - R_TRY(f.Write(offset, buf.data(), bytes_read, FsWriteOption_None)); - - pbox->UpdateTransfer(offset, info.uncompressed_size); - offset += bytes_read; - } - } - } + R_TRY(thread::TransferUnzipAll(pbox, temp_file, &fs, root_path)); } else { fs.CreateDirectoryRecursivelyWithPath(root_path); fs.DeleteFile(root_path); diff --git a/sphaira/source/ui/menus/main_menu.cpp b/sphaira/source/ui/menus/main_menu.cpp index 445edca..0d8a2db 100644 --- a/sphaira/source/ui/menus/main_menu.cpp +++ b/sphaira/source/ui/menus/main_menu.cpp @@ -22,9 +22,9 @@ #include "download.hpp" #include "defines.hpp" #include "i18n.hpp" +#include "threaded_file_transfer.hpp" #include -#include #include namespace sphaira::ui::menu::main { @@ -59,7 +59,6 @@ const MiscMenuEntry MISC_MENU_ENTRIES[] = { auto InstallUpdate(ProgressBox* pbox, const std::string url, const std::string version) -> Result { static fs::FsPath zip_out{"/switch/sphaira/cache/update.zip"}; - constexpr auto chunk_size = 1024 * 512; // 512KiB fs::FsNativeSd fs; R_TRY(fs.GetFsOpenResult()); @@ -82,95 +81,34 @@ auto InstallUpdate(ProgressBox* pbox, const std::string url, const std::string v // 2. extract the zip if (!pbox->ShouldExit()) { - auto zfile = unzOpen64(zip_out); - R_UNLESS(zfile, 0x1); - ON_SCOPE_EXIT(unzClose(zfile)); + const auto exe_path = App::GetExePath(); + bool found_exe{}; - unz_global_info64 pglobal_info; - if (UNZ_OK != unzGetGlobalInfo64(zfile, &pglobal_info)) { - R_THROW(0x1); - } - - for (s64 i = 0; i < pglobal_info.number_entry; i++) { - if (i > 0) { - if (UNZ_OK != unzGoToNextFile(zfile)) { - log_write("failed to unzGoToNextFile\n"); - R_THROW(0x1); - } + R_TRY(thread::TransferUnzipAll(pbox, zip_out, &fs, "/", [&](const fs::FsPath& name, fs::FsPath& path) -> bool { + if (std::strstr(path, "sphaira.nro")) { + path = exe_path; + found_exe = true; } + return true; + })); - if (UNZ_OK != unzOpenCurrentFile(zfile)) { - log_write("failed to open current file\n"); - R_THROW(0x1); - } - ON_SCOPE_EXIT(unzCloseCurrentFile(zfile)); - - unz_file_info64 info; - fs::FsPath file_path; - if (UNZ_OK != unzGetCurrentFileInfo64(zfile, &info, file_path, sizeof(file_path), 0, 0, 0, 0)) { - log_write("failed to get current info\n"); - R_THROW(0x1); - } - - if (file_path[0] != '/') { - file_path = fs::AppendPath("/", file_path); - } - - if (std::strstr(file_path, "sphaira.nro")) { - file_path = App::GetExePath(); - } - - Result rc; - if (file_path[std::strlen(file_path) -1] == '/') { - if (R_FAILED(rc = fs.CreateDirectoryRecursively(file_path)) && rc != FsError_PathAlreadyExists) { - log_write("failed to create folder: %s 0x%04X\n", file_path.s, rc); - R_THROW(rc); - } - } else { - Result rc; - if (R_FAILED(rc = fs.CreateFile(file_path, info.uncompressed_size, 0)) && rc != FsError_PathAlreadyExists) { - log_write("failed to create file: %s 0x%04X\n", file_path.s, rc); - R_THROW(rc); + // check if we have sphaira installed in other locations and update them. + if (found_exe) { + for (auto& path : SPHAIRA_PATHS) { + log_write("[UPD] checking path: %s\n", path.s); + // skip if we already updated this path. + if (exe_path == path) { + log_write("[UPD] skipped as already updated\n"); + continue; } - fs::File f; - R_TRY(fs.OpenFile(file_path, FsOpenMode_Write, &f)); - R_TRY(f.SetSize(info.uncompressed_size)); - - std::vector buf(chunk_size); - s64 offset{}; - while (offset < info.uncompressed_size) { - const auto bytes_read = unzReadCurrentFile(zfile, buf.data(), buf.size()); - if (bytes_read <= 0) { - // log_write("failed to read zip file: %s\n", inzip.c_str()); - R_THROW(0x1); - } - - R_TRY(f.Write(offset, buf.data(), bytes_read, FsWriteOption_None)); - - pbox->UpdateTransfer(offset, info.uncompressed_size); - offset += bytes_read; - } - } - - // check if we have sphaira installed in other locations and update them. - if (file_path == App::GetExePath()) { - for (auto& path : SPHAIRA_PATHS) { - log_write("[UPD] checking path: %s\n", path.s); - // skip if we already updated this path. - if (file_path == path) { - log_write("[UPD] skipped as already updated\n"); - continue; - } - - // check that this is really sphaira. - log_write("[UPD] checking nacp\n"); - NacpStruct nacp; - if (R_SUCCEEDED(nro_get_nacp(path, nacp)) && !std::strcmp(nacp.lang[0].name, "sphaira")) { - log_write("[UPD] found, updating\n"); - pbox->NewTransfer(path); - R_TRY(pbox->CopyFile(&fs, file_path, path)); - } + // check that this is really sphaira. + log_write("[UPD] checking nacp\n"); + NacpStruct nacp; + if (R_SUCCEEDED(nro_get_nacp(path, nacp)) && !std::strcmp(nacp.lang[0].name, "sphaira")) { + log_write("[UPD] found, updating\n"); + pbox->NewTransfer(path); + R_TRY(pbox->CopyFile(&fs, exe_path, path)); } } } diff --git a/sphaira/source/ui/menus/themezer.cpp b/sphaira/source/ui/menus/themezer.cpp index 18d4c81..934ce57 100644 --- a/sphaira/source/ui/menus/themezer.cpp +++ b/sphaira/source/ui/menus/themezer.cpp @@ -11,11 +11,11 @@ #include "ui/nvg_util.hpp" #include "swkbd.hpp" #include "i18n.hpp" +#include "threaded_file_transfer.hpp" #include #include #include -#include #include #include "yyjson_helper.hpp" @@ -222,7 +222,6 @@ void from_json(const fs::FsPath& path, PackList& e) { auto InstallTheme(ProgressBox* pbox, const PackListEntry& entry) -> Result { static const fs::FsPath zip_out{"/switch/sphaira/cache/themezer/temp.zip"}; - constexpr auto chunk_size = 1024 * 512; // 512KiB fs::FsNativeSd fs; R_TRY(fs.GetFsOpenResult()); @@ -272,66 +271,7 @@ auto InstallTheme(ProgressBox* pbox, const PackListEntry& entry) -> Result { // 3. extract the zip if (!pbox->ShouldExit()) { - auto zfile = unzOpen64(zip_out); - R_UNLESS(zfile, 0x1); - ON_SCOPE_EXIT(unzClose(zfile)); - - unz_global_info64 pglobal_info; - if (UNZ_OK != unzGetGlobalInfo64(zfile, &pglobal_info)) { - R_THROW(0x1); - } - - for (int i = 0; i < pglobal_info.number_entry; i++) { - if (i > 0) { - if (UNZ_OK != unzGoToNextFile(zfile)) { - log_write("failed to unzGoToNextFile\n"); - R_THROW(0x1); - } - } - - if (UNZ_OK != unzOpenCurrentFile(zfile)) { - log_write("failed to open current file\n"); - R_THROW(0x1); - } - ON_SCOPE_EXIT(unzCloseCurrentFile(zfile)); - - unz_file_info64 info; - char name[512]; - if (UNZ_OK != unzGetCurrentFileInfo64(zfile, &info, name, sizeof(name), 0, 0, 0, 0)) { - log_write("failed to get current info\n"); - R_THROW(0x1); - } - - const auto file_path = fs::AppendPath(dir_path, name); - pbox->NewTransfer(name); - - Result rc; - if (R_FAILED(rc = fs.CreateFile(file_path, info.uncompressed_size, 0)) && rc != FsError_PathAlreadyExists) { - log_write("failed to create file: %s 0x%04X\n", file_path.s, rc); - R_THROW(rc); - } - - fs::File f; - R_TRY(fs.OpenFile(file_path, FsOpenMode_Write, &f)); - R_TRY(f.SetSize(info.uncompressed_size)); - - std::vector buf(chunk_size); - s64 offset{}; - while (offset < info.uncompressed_size) { - R_TRY(pbox->ShouldExitResult()); - - const auto bytes_read = unzReadCurrentFile(zfile, buf.data(), buf.size()); - if (bytes_read <= 0) { - // log_write("failed to read zip file: %s\n", inzip.c_str()); - R_THROW(0x1); - } - - R_TRY(f.Write(offset, buf.data(), bytes_read, FsWriteOption_None)); - - pbox->UpdateTransfer(offset, info.uncompressed_size); - offset += bytes_read; - } - } + R_TRY(thread::TransferUnzipAll(pbox, zip_out, &fs, dir_path)); } log_write("finished install :)\n"); From b46136b959a57d6b9e3fd781748f35afb9a1fb15 Mon Sep 17 00:00:00 2001 From: ITotalJustice <47043333+ITotalJustice@users.noreply.github.com> Date: Fri, 30 May 2025 13:16:39 +0100 Subject: [PATCH 2/3] optimise fs CreateDirectoryRecursively() by checking if the path already exists prior to the loop. --- sphaira/source/fs.cpp | 45 +++++++++++++++++++------------------------ 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/sphaira/source/fs.cpp b/sphaira/source/fs.cpp index c09b825..79e4446 100644 --- a/sphaira/source/fs.cpp +++ b/sphaira/source/fs.cpp @@ -118,6 +118,18 @@ Result CreateDirectory(FsFileSystem* fs, const FsPath& path, bool ignore_read_on Result CreateDirectoryRecursively(FsFileSystem* fs, const FsPath& _path, bool ignore_read_only) { R_UNLESS(ignore_read_only || !is_read_only_root(_path), Fs::ResultReadOnly); + // try and create the directory / see if it already exists before the loop. + Result rc; + if (fs) { + rc = CreateDirectory(fs, _path, ignore_read_only); + } else { + rc = CreateDirectory(_path, ignore_read_only); + } + + if (R_SUCCEEDED(rc) || rc == FsError_PathAlreadyExists) { + R_SUCCEED(); + } + auto path_view = std::string_view{_path}; // todo: fix this for sdmc: and ums0: FsPath path{"/"}; @@ -134,7 +146,6 @@ Result CreateDirectoryRecursively(FsFileSystem* fs, const FsPath& _path, bool ig std::strncat(path, dir.data(), dir.size()); log_write("[FS] dir creation path is now: %s\n", path.s); - Result rc; if (fs) { rc = CreateDirectory(fs, path, ignore_read_only); } else { @@ -155,31 +166,15 @@ Result CreateDirectoryRecursively(FsFileSystem* fs, const FsPath& _path, bool ig Result CreateDirectoryRecursivelyWithPath(FsFileSystem* fs, const FsPath& _path, bool ignore_read_only) { R_UNLESS(ignore_read_only || !is_read_only_root(_path), Fs::ResultReadOnly); - size_t off = 0; - while (true) { - const auto first = std::strchr(_path + off, '/'); - if (!first) { - R_SUCCEED(); - } - - off = (first - _path.s) + 1; - FsPath path; - std::strncpy(path, _path, off); - - Result rc; - if (fs) { - rc = CreateDirectory(fs, path, ignore_read_only); - } else { - rc = CreateDirectory(path, ignore_read_only); - } - - if (R_FAILED(rc) && rc != FsError_PathAlreadyExists) { - log_write("failed to create folder recursively: %s\n", path.s); - return rc; - } - - // log_write("created_directory recursively: %s\n", path); + // strip file name form path. + const auto last_slash = std::strrchr(_path, '/'); + if (!last_slash) { + R_SUCCEED(); } + + FsPath new_path{}; + std::snprintf(new_path, sizeof(new_path), "%.*s", (int)(last_slash - _path.s), _path.s); + return CreateDirectoryRecursively(fs, new_path, ignore_read_only); } Result DeleteFile(FsFileSystem* fs, const FsPath& path, bool ignore_read_only) { From 5893cb575ef04965da69abcd9818e7b33ffa2764 Mon Sep 17 00:00:00 2001 From: ITotalJustice <47043333+ITotalJustice@users.noreply.github.com> Date: Sat, 31 May 2025 17:30:28 +0100 Subject: [PATCH 3/3] fix ncz block installs, fix error module value being out of range, display error on install from filebrowser. the issue with block installs was that i was not tracking the ncz block offset in between transfers. this resulted in the block size being used for each transfer, rather then size-offset. for blocks that were always compressed, this silently worked as zstd stream can handle multiple frames. however, if there existed compressed and uncompressed blocks, then this bug would be exposed. thanks to Marulv for reporting the bug. --- sphaira/include/yati/nx/es.hpp | 2 +- sphaira/include/yati/source/usb.hpp | 2 +- sphaira/include/yati/yati.hpp | 12 ++++++++++- sphaira/source/ui/error_box.cpp | 2 ++ sphaira/source/ui/menus/filebrowser.cpp | 2 ++ sphaira/source/yati/yati.cpp | 28 ++++++++++++++++--------- 6 files changed, 35 insertions(+), 13 deletions(-) diff --git a/sphaira/include/yati/nx/es.hpp b/sphaira/include/yati/nx/es.hpp index 315222c..7261714 100644 --- a/sphaira/include/yati/nx/es.hpp +++ b/sphaira/include/yati/nx/es.hpp @@ -7,7 +7,7 @@ namespace sphaira::es { -enum { TicketModule = 522 }; +enum { TicketModule = 507 }; enum : Result { // found ticket has missmatching rights_id from it's name. diff --git a/sphaira/include/yati/source/usb.hpp b/sphaira/include/yati/source/usb.hpp index 7619c60..7c1ac9c 100644 --- a/sphaira/include/yati/source/usb.hpp +++ b/sphaira/include/yati/source/usb.hpp @@ -11,7 +11,7 @@ namespace sphaira::yati::source { struct Usb final : Base { - enum { USBModule = 523 }; + enum { USBModule = 508 }; enum : Result { Result_BadMagic = MAKERESULT(USBModule, 0), diff --git a/sphaira/include/yati/yati.hpp b/sphaira/include/yati/yati.hpp index 58d1d21..067acf9 100644 --- a/sphaira/include/yati/yati.hpp +++ b/sphaira/include/yati/yati.hpp @@ -16,8 +16,18 @@ namespace sphaira::yati { -enum { YatiModule = 521 }; +enum { YatiModule = 506 }; +/* +Improving compression ratio via block splitting is now enabled by default for high compression levels (16+). +The amount of benefit varies depending on the workload. +Compressing archives comprised of heavily differing files will see more improvement than compression of single files that don’t + vary much entropically (like text files/enwik). At levels 16+, we observe no measurable regression to compression speed. + +The block splitter can be forcibly enabled on lower compression levels as well with the advanced parameter ZSTD_c_splitBlocks. +When forcibly enabled at lower levels, speed regressions can become more notable. +Additionally, since more compressed blocks may be produced, decompression speed on these blobs may also see small regressions. +*/ enum : Result { // unkown container for the source provided. Result_ContainerNotFound = MAKERESULT(YatiModule, 10), diff --git a/sphaira/source/ui/error_box.cpp b/sphaira/source/ui/error_box.cpp index dff51be..325c05c 100644 --- a/sphaira/source/ui/error_box.cpp +++ b/sphaira/source/ui/error_box.cpp @@ -6,6 +6,7 @@ namespace sphaira::ui { ErrorBox::ErrorBox(const std::string& message) : m_message{message} { + log_write("[ERROR] %s\n", m_message.c_str()); m_pos.w = 770.f; m_pos.h = 430.f; @@ -21,6 +22,7 @@ ErrorBox::ErrorBox(const std::string& message) : m_message{message} { ErrorBox::ErrorBox(Result code, const std::string& message) : ErrorBox{message} { m_code = code; + log_write("[ERROR] Code: 0x%X Module: %u Description: %u\n", R_VALUE(code), R_MODULE(code), R_DESCRIPTION(code)); } auto ErrorBox::Update(Controller* controller, TouchInfo* touch) -> void { diff --git a/sphaira/source/ui/menus/filebrowser.cpp b/sphaira/source/ui/menus/filebrowser.cpp index 7e3f72e..8621044 100644 --- a/sphaira/source/ui/menus/filebrowser.cpp +++ b/sphaira/source/ui/menus/filebrowser.cpp @@ -694,6 +694,8 @@ void FsView::InstallFiles() { } R_SUCCEED(); + }, [this](Result rc){ + App::PushErrorBox(rc, "File install failed!"_i18n); })); } })); diff --git a/sphaira/source/yati/yati.cpp b/sphaira/source/yati/yati.cpp index 2ee7799..8d9bc86 100644 --- a/sphaira/source/yati/yati.cpp +++ b/sphaira/source/yati/yati.cpp @@ -433,7 +433,7 @@ Result Yati::decompressFuncInternal(ThreadData* t) { inflate_buf.reserve(t->max_buffer_size); s64 written{}; - s64 decompress_buf_off{}; + s64 block_offset{}; std::vector buf{}; buf.reserve(t->max_buffer_size); @@ -454,14 +454,15 @@ Result Yati::decompressFuncInternal(ThreadData* t) { } for (s64 off = 0; off < size;) { - // log_write("looking for section\n"); if (!ncz_section || !ncz_section->InRange(written)) { + log_write("[NCZ] looking for new section: %zu\n", written); auto it = std::ranges::find_if(t->ncz_sections, [written](auto& e){ return e.InRange(written); }); R_UNLESS(it != t->ncz_sections.cend(), Result_NczSectionNotFound); ncz_section = &(*it); + log_write("[NCZ] found new section: %zu\n", written); if (ncz_section->crypto_type >= nca::EncryptionType_AesCtr) { const auto swp = std::byteswap(u64(written) >> 4); @@ -488,7 +489,7 @@ Result Yati::decompressFuncInternal(ThreadData* t) { // restore remaining data to the swapped buffer. if (!temp_vector.empty()) { - log_write("storing data size: %zu\n", temp_vector.size()); + log_write("[NCZ] storing data size: %zu\n", temp_vector.size()); inflate_buf = temp_vector; } @@ -496,6 +497,7 @@ Result Yati::decompressFuncInternal(ThreadData* t) { }; while (t->decompress_offset < t->write_size && R_SUCCEEDED(t->GetResults())) { + s64 decompress_buf_off{}; R_TRY(t->GetDecompressBuf(buf, decompress_buf_off)); // do we have an nsz? if so, setup buffers. @@ -616,12 +618,14 @@ Result Yati::decompressFuncInternal(ThreadData* t) { // todo: blocks need to use read offset, as the offset + size is compressed range. if (t->ncz_blocks.size()) { if (!ncz_block || !ncz_block->InRange(decompress_buf_off)) { + block_offset = 0; + log_write("[NCZ] looking for new block: %zu\n", decompress_buf_off); auto it = std::ranges::find_if(t->ncz_blocks, [decompress_buf_off](auto& e){ return e.InRange(decompress_buf_off); }); R_UNLESS(it != t->ncz_blocks.cend(), Result_NczBlockNotFound); - // log_write("looking found block\n"); + log_write("[NCZ] found new block: %zu off: %zd size: %zd\n", decompress_buf_off, it->offset, it->size); ncz_block = &(*it); } @@ -629,7 +633,7 @@ Result Yati::decompressFuncInternal(ThreadData* t) { auto decompressedBlockSize = 1 << t->ncz_block_header.block_size_exponent; // special handling for the last block to check it's actually compressed if (ncz_block->offset == t->ncz_blocks.back().offset) { - log_write("last block special handling\n"); + log_write("[NCZ] last block special handling\n"); decompressedBlockSize = t->ncz_block_header.decompressed_size % decompressedBlockSize; } @@ -637,12 +641,12 @@ Result Yati::decompressFuncInternal(ThreadData* t) { compressed = ncz_block->size < decompressedBlockSize; // clip read size as blocks can be up to 32GB in size! - const auto size = std::min(buf.size() - buf_off, ncz_block->size); - buffer = {buf.data() + buf_off, size}; + const auto size = std::min(buffer.size(), ncz_block->size - block_offset); + buffer = buffer.subspan(0, size); } if (compressed) { - // log_write("COMPRESSED block\n"); + log_write("[NCZ] COMPRESSED block\n"); ZSTD_inBuffer input = { buffer.data(), buffer.size(), 0 }; while (input.pos < input.size) { R_TRY(t->GetResults()); @@ -650,12 +654,15 @@ Result Yati::decompressFuncInternal(ThreadData* t) { inflate_buf.resize(inflate_offset + chunk_size); ZSTD_outBuffer output = { inflate_buf.data() + inflate_offset, chunk_size, 0 }; const auto res = ZSTD_decompressStream(dctx, std::addressof(output), std::addressof(input)); + if (ZSTD_isError(res)) { + log_write("[NCZ] ZSTD_decompressStream() pos: %zu size: %zu res: %zd msg: %s\n", input.pos, input.size, res, ZSTD_getErrorName(res)); + } R_UNLESS(!ZSTD_isError(res), Result_InvalidNczZstdError); t->decompress_offset += output.pos; inflate_offset += output.pos; if (inflate_offset >= INFLATE_BUFFER_MAX) { - // log_write("flushing compressed data: %zd vs %zd diff: %zd\n", inflate_offset, INFLATE_BUFFER_MAX, inflate_offset - INFLATE_BUFFER_MAX); + log_write("[NCZ] flushing compressed data: %zd vs %zd diff: %zd\n", inflate_offset, INFLATE_BUFFER_MAX, inflate_offset - INFLATE_BUFFER_MAX); R_TRY(ncz_flush(INFLATE_BUFFER_MAX)); } } @@ -666,13 +673,14 @@ Result Yati::decompressFuncInternal(ThreadData* t) { t->decompress_offset += buffer.size(); inflate_offset += buffer.size(); if (inflate_offset >= INFLATE_BUFFER_MAX) { - // log_write("flushing copy data\n"); + log_write("[NCZ] flushing copy data\n"); R_TRY(ncz_flush(INFLATE_BUFFER_MAX)); } } buf_off += buffer.size(); decompress_buf_off += buffer.size(); + block_offset += buffer.size(); } } }