OmniNX/sphaira
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'multi_thread_everything'

Browse Source
This commit is contained in:
ITotalJustice
2025-05-31 18:08:17 +01:00
parent f98135325a 5893cb575e
commit 2ef7742903
15 changed files with 523 additions and 548 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
sphaira/include/hasher.hpp
View File

@@ -4,6 +4,7 @@
#include "ui/progress_box.hpp"
#include <string>
#include <memory>
#include <span>
#include <switch.h>
namespace sphaira::hash {
@@ -26,5 +27,6 @@ auto GetTypeStr(Type type) -> const char*;
// returns the hash string.
Result Hash(ui::ProgressBox* pbox, Type type, std::shared_ptr<BaseSource> source, std::string& out);
Result Hash(ui::ProgressBox* pbox, Type type, fs::Fs* fs, const fs::FsPath& path, std::string& out);
Result Hash(ui::ProgressBox* pbox, Type type, std::span<const u8> data, std::string& out);
} // namespace sphaira::hash

30
sphaira/include/threaded_file_transfer.hpp
View File

@@ -6,6 +6,15 @@
namespace sphaira::thread {
enum class Mode {
// default, always multi-thread.
MultiThreaded,
// always single-thread.
SingleThreaded,
// check buffer size, if smaller, single thread.
SingleThreadedIfSmaller,
};
using ReadCallback = std::function<Result(void* data, s64 off, s64 size, u64* bytes_read)>;
using WriteCallback = std::function<Result(const void* data, s64 off, s64 size)>;
@@ -23,10 +32,25 @@ using StartCallback = std::function<Result(PullCallback pull)>;
using StartCallback2 = std::function<Result(StartThreadCallback start, PullCallback pull)>;
// reads data from rfunc into wfunc.
Result Transfer(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, WriteCallback wfunc);
Result Transfer(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, WriteCallback wfunc, Mode mode = Mode::MultiThreaded);
// reads data from rfunc, pull data from provided pull() callback.
Result TransferPull(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, StartCallback sfunc);
Result TransferPull(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, StartCallback2 sfunc);
Result TransferPull(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, StartCallback sfunc, Mode mode = Mode::MultiThreaded);
Result TransferPull(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, StartCallback2 sfunc, Mode mode = Mode::MultiThreaded);
// helper for extract zips.
// this will multi-thread unzip if size >= 512KiB, otherwise it'll single pass.
Result TransferUnzip(ui::ProgressBox* pbox, void* zfile, fs::Fs* fs, const fs::FsPath& path, s64 size, u32 crc32 = 0);
// same as above but for zipping files.
Result TransferZip(ui::ProgressBox* pbox, void* zfile, fs::Fs* fs, const fs::FsPath& path);
// passes the name inside the zip an final output path.
using UnzipAllFilter = std::function<bool(const fs::FsPath& name, fs::FsPath& path)>;
// helper all-in-one unzip function that unzips a zip (either open or path provided).
// the filter function can be used to modify the path and filter out unwanted files.
Result TransferUnzipAll(ui::ProgressBox* pbox, void* zfile, fs::Fs* fs, const fs::FsPath& base_path, UnzipAllFilter filter = nullptr);
Result TransferUnzipAll(ui::ProgressBox* pbox, const fs::FsPath& zip_out, fs::Fs* fs, const fs::FsPath& base_path, UnzipAllFilter filter = nullptr);
} // namespace sphaira::thread

2
sphaira/include/yati/nx/es.hpp
View File

@@ -7,7 +7,7 @@
namespace sphaira::es {
enum { TicketModule = 522 };
enum { TicketModule = 507 };
enum : Result {
// found ticket has missmatching rights_id from it's name.

2
sphaira/include/yati/source/usb.hpp
View File

@@ -11,7 +11,7 @@
namespace sphaira::yati::source {
struct Usb final : Base {
enum { USBModule = 523 };
enum { USBModule = 508 };
enum : Result {
Result_BadMagic = MAKERESULT(USBModule, 0),

12
sphaira/include/yati/yati.hpp
View File

@@ -16,8 +16,18 @@
namespace sphaira::yati {
enum { YatiModule = 521 };
enum { YatiModule = 506 };
/*
Improving compression ratio via block splitting is now enabled by default for high compression levels (16+).
The amount of benefit varies depending on the workload.
Compressing archives comprised of heavily differing files will see more improvement than compression of single files that don’t
vary much entropically (like text files/enwik). At levels 16+, we observe no measurable regression to compression speed.
The block splitter can be forcibly enabled on lower compression levels as well with the advanced parameter ZSTD_c_splitBlocks.
When forcibly enabled at lower levels, speed regressions can become more notable.
Additionally, since more compressed blocks may be produced, decompression speed on these blobs may also see small regressions.
*/
enum : Result {
// unkown container for the source provided.
Result_ContainerNotFound = MAKERESULT(YatiModule, 10),

45
sphaira/source/fs.cpp
View File

@@ -118,6 +118,18 @@ Result CreateDirectory(FsFileSystem* fs, const FsPath& path, bool ignore_read_on
Result CreateDirectoryRecursively(FsFileSystem* fs, const FsPath& _path, bool ignore_read_only) {
R_UNLESS(ignore_read_only || !is_read_only_root(_path), Fs::ResultReadOnly);
// try and create the directory / see if it already exists before the loop.
Result rc;
if (fs) {
rc = CreateDirectory(fs, _path, ignore_read_only);
} else {
rc = CreateDirectory(_path, ignore_read_only);
}
if (R_SUCCEEDED(rc) || rc == FsError_PathAlreadyExists) {
R_SUCCEED();
}
auto path_view = std::string_view{_path};
// todo: fix this for sdmc: and ums0:
FsPath path{"/"};
@@ -134,7 +146,6 @@ Result CreateDirectoryRecursively(FsFileSystem* fs, const FsPath& _path, bool ig
std::strncat(path, dir.data(), dir.size());
log_write("[FS] dir creation path is now: %s\n", path.s);
Result rc;
if (fs) {
rc = CreateDirectory(fs, path, ignore_read_only);
} else {
@@ -155,31 +166,15 @@ Result CreateDirectoryRecursively(FsFileSystem* fs, const FsPath& _path, bool ig
Result CreateDirectoryRecursivelyWithPath(FsFileSystem* fs, const FsPath& _path, bool ignore_read_only) {
R_UNLESS(ignore_read_only || !is_read_only_root(_path), Fs::ResultReadOnly);
size_t off = 0;
while (true) {
const auto first = std::strchr(_path + off, '/');
if (!first) {
R_SUCCEED();
}
off = (first - _path.s) + 1;
FsPath path;
std::strncpy(path, _path, off);
Result rc;
if (fs) {
rc = CreateDirectory(fs, path, ignore_read_only);
} else {
rc = CreateDirectory(path, ignore_read_only);
}
if (R_FAILED(rc) && rc != FsError_PathAlreadyExists) {
log_write("failed to create folder recursively: %s\n", path.s);
return rc;
}
// log_write("created_directory recursively: %s\n", path);
// strip file name form path.
const auto last_slash = std::strrchr(_path, '/');
if (!last_slash) {
R_SUCCEED();
}
FsPath new_path{};
std::snprintf(new_path, sizeof(new_path), "%.*s", (int)(last_slash - _path.s), _path.s);
return CreateDirectoryRecursively(fs, new_path, ignore_read_only);
}
Result DeleteFile(FsFileSystem* fs, const FsPath& path, bool ignore_read_only) {

24
sphaira/source/hasher.cpp
View File

@@ -35,6 +35,25 @@ private:
bool m_is_file_based_emummc{};
};
struct MemSource final : BaseSource {
MemSource(std::span<const u8> data) : m_data{data} { }
Result Size(s64* out) {
*out = m_data.size();
R_SUCCEED();
}
Result Read(void* buf, s64 off, s64 size, u64* bytes_read) {
size = std::min<s64>(size, m_data.size() - off);
std::memcpy(buf, m_data.data() + off, size);
*bytes_read = size;
R_SUCCEED();
}
private:
const std::span<const u8> m_data;
};
struct HashSource {
virtual ~HashSource() = default;
virtual void Update(const void* buf, s64 size) = 0;
@@ -181,4 +200,9 @@ Result Hash(ui::ProgressBox* pbox, Type type, fs::Fs* fs, const fs::FsPath& path
return Hash(pbox, type, source, out);
}
Result Hash(ui::ProgressBox* pbox, Type type, std::span<const u8> data, std::string& out) {
auto source = std::make_shared<MemSource>(data);
return Hash(pbox, type, source, out);
}
} // namespace sphaira::has

339
sphaira/source/threaded_file_transfer.cpp
View File

@@ -6,15 +6,20 @@
#include <vector>
#include <algorithm>
#include <cstring>
#include <minizip/unzip.h>
#include <minizip/zip.h>
namespace sphaira::thread {
namespace {
constexpr u64 READ_BUFFER_MAX = 1024*1024*4;
// used for file based emummc and zip/unzip.
constexpr u64 SMALL_BUFFER_SIZE = 1024 * 512;
// used for everything else.
constexpr u64 NORMAL_BUFFER_SIZE = 1024*1024*4;
struct ThreadBuffer {
ThreadBuffer() {
buf.reserve(READ_BUFFER_MAX);
buf.reserve(NORMAL_BUFFER_SIZE);
}
std::vector<u8> buf;
@@ -65,7 +70,7 @@ public:
};
struct ThreadData {
ThreadData(ui::ProgressBox* _pbox, s64 size, ReadCallback _rfunc, WriteCallback _wfunc);
ThreadData(ui::ProgressBox* _pbox, s64 size, ReadCallback _rfunc, WriteCallback _wfunc, u64 buffer_size);
auto GetResults() -> Result;
void WakeAllThreads();
@@ -104,9 +109,9 @@ private:
private:
// these need to be copied
ui::ProgressBox* pbox{};
ReadCallback rfunc{};
WriteCallback wfunc{};
ui::ProgressBox* const pbox;
const ReadCallback rfunc;
const WriteCallback wfunc;
// these need to be created
Mutex mutex{};
@@ -121,21 +126,24 @@ private:
std::vector<u8> pull_buffer{};
s64 pull_buffer_offset{};
u64 read_buffer_size{};
u64 max_buffer_size{};
const u64 read_buffer_size;
const s64 write_size;
// these are shared between threads
volatile s64 read_offset{};
volatile s64 write_offset{};
volatile s64 write_size{};
volatile Result read_result{};
volatile Result write_result{};
volatile Result pull_result{};
};
ThreadData::ThreadData(ui::ProgressBox* _pbox, s64 size, ReadCallback _rfunc, WriteCallback _wfunc)
: pbox{_pbox}, rfunc{_rfunc}, wfunc{_wfunc} {
ThreadData::ThreadData(ui::ProgressBox* _pbox, s64 size, ReadCallback _rfunc, WriteCallback _wfunc, u64 buffer_size)
: pbox{_pbox}
, rfunc{_rfunc}
, wfunc{_wfunc}
, read_buffer_size{buffer_size}
, write_size{size} {
mutexInit(std::addressof(mutex));
mutexInit(std::addressof(pull_mutex));
@@ -143,16 +151,6 @@ ThreadData::ThreadData(ui::ProgressBox* _pbox, s64 size, ReadCallback _rfunc, Wr
condvarInit(std::addressof(can_write));
condvarInit(std::addressof(can_pull));
condvarInit(std::addressof(can_pull_write));
write_size = size;
if (App::IsFileBaseEmummc()) {
read_buffer_size = 1024 * 512;
max_buffer_size = 1024 * 512;
} else {
read_buffer_size = READ_BUFFER_MAX;
max_buffer_size = READ_BUFFER_MAX;
}
}
auto ThreadData::GetResults() -> Result {
@@ -251,7 +249,7 @@ Result ThreadData::Pull(void* data, s64 size, u64* bytes_read) {
Result ThreadData::readFuncInternal() {
// the main buffer which data is read into.
std::vector<u8> buf;
buf.reserve(this->max_buffer_size);
buf.reserve(this->read_buffer_size);
while (this->read_offset < this->write_size && R_SUCCEEDED(this->GetResults())) {
// read more data
@@ -265,14 +263,14 @@ Result ThreadData::readFuncInternal() {
R_TRY(this->SetWriteBuf(buf, buf_size));
}
log_write("read success\n");
log_write("finished read thread success!\n");
R_SUCCEED();
}
// write thread writes data to the nca placeholder.
Result ThreadData::writeFuncInternal() {
std::vector<u8> buf;
buf.reserve(this->max_buffer_size);
buf.reserve(this->read_buffer_size);
while (this->write_offset < this->write_size && R_SUCCEEDED(this->GetResults())) {
s64 dummy_off;
@@ -288,7 +286,7 @@ Result ThreadData::writeFuncInternal() {
this->write_offset += size;
}
log_write("finished write thread!\n");
log_write("finished write thread success!\n");
R_SUCCEED();
}
@@ -308,87 +306,264 @@ auto GetAlternateCore(int id) {
return id == 1 ? 2 : 1;
}
Result TransferInternal(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, WriteCallback wfunc, StartCallback2 sfunc) {
App::SetAutoSleepDisabled(true);
ON_SCOPE_EXIT(App::SetAutoSleepDisabled(false));
Result TransferInternal(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, WriteCallback wfunc, StartCallback2 sfunc, Mode mode, u64 buffer_size = NORMAL_BUFFER_SIZE) {
const auto is_file_based_emummc = App::IsFileBaseEmummc();
const auto WRITE_THREAD_CORE = sfunc ? pbox->GetCpuId() : GetAlternateCore(pbox->GetCpuId());
const auto READ_THREAD_CORE = GetAlternateCore(WRITE_THREAD_CORE);
if (is_file_based_emummc) {
buffer_size = SMALL_BUFFER_SIZE;
}
ThreadData t_data{pbox, size, rfunc, wfunc};
if (mode == Mode::SingleThreadedIfSmaller) {
if (size <= buffer_size) {
mode = Mode::SingleThreaded;
} else {
mode = Mode::MultiThreaded;
}
}
Thread t_read{};
R_TRY(threadCreate(&t_read, readFunc, std::addressof(t_data), nullptr, 1024*64, PRIO_PREEMPTIVE, READ_THREAD_CORE));
ON_SCOPE_EXIT(threadClose(&t_read));
// single threaded pull buffer is not supported.
R_UNLESS(mode != Mode::MultiThreaded || !sfunc, 0x1);
Thread t_write{};
R_TRY(threadCreate(&t_write, writeFunc, std::addressof(t_data), nullptr, 1024*64, PRIO_PREEMPTIVE, WRITE_THREAD_CORE));
ON_SCOPE_EXIT(threadClose(&t_write));
// todo: support single threaded pull buffer.
if (mode == Mode::SingleThreaded) {
std::vector<u8> buf(buffer_size);
s64 offset{};
while (offset < size) {
R_TRY(pbox->ShouldExitResult());
u64 bytes_read;
const auto rsize = std::min<s64>(buf.size(), size - offset);
R_TRY(rfunc(buf.data(), offset, rsize, &bytes_read));
R_TRY(wfunc(buf.data(), offset, bytes_read));
offset += bytes_read;
pbox->UpdateTransfer(offset, size);
}
const auto start_threads = [&]() -> Result {
log_write("starting threads\n");
R_TRY(threadStart(std::addressof(t_read)));
R_TRY(threadStart(std::addressof(t_write)));
R_SUCCEED();
};
ON_SCOPE_EXIT(threadWaitForExit(std::addressof(t_read)));
ON_SCOPE_EXIT(threadWaitForExit(std::addressof(t_write)));
if (sfunc) {
t_data.SetPullResult(sfunc(start_threads, [&](void* data, s64 size, u64* bytes_read) -> Result {
R_TRY(t_data.GetResults());
return t_data.Pull(data, size, bytes_read);
}));
} else {
R_TRY(start_threads());
while (t_data.GetWriteOffset() != t_data.GetWriteSize() && R_SUCCEEDED(t_data.GetResults())) {
pbox->UpdateTransfer(t_data.GetWriteOffset(), t_data.GetWriteSize());
svcSleepThread(1e+6);
}
}
else {
const auto WRITE_THREAD_CORE = sfunc ? pbox->GetCpuId() : GetAlternateCore(pbox->GetCpuId());
const auto READ_THREAD_CORE = GetAlternateCore(WRITE_THREAD_CORE);
// wait for all threads to close.
log_write("waiting for threads to close\n");
for (;;) {
t_data.WakeAllThreads();
pbox->Yield();
ThreadData t_data{pbox, size, rfunc, wfunc, buffer_size};
if (R_FAILED(waitSingleHandle(t_read.handle, 1000))) {
continue;
} else if (R_FAILED(waitSingleHandle(t_write.handle, 1000))) {
continue;
Thread t_read{};
R_TRY(threadCreate(&t_read, readFunc, std::addressof(t_data), nullptr, 1024*256, 0x3B, READ_THREAD_CORE));
ON_SCOPE_EXIT(threadClose(&t_read));
Thread t_write{};
R_TRY(threadCreate(&t_write, writeFunc, std::addressof(t_data), nullptr, 1024*256, 0x3B, WRITE_THREAD_CORE));
ON_SCOPE_EXIT(threadClose(&t_write));
const auto start_threads = [&]() -> Result {
log_write("starting threads\n");
R_TRY(threadStart(std::addressof(t_read)));
R_TRY(threadStart(std::addressof(t_write)));
R_SUCCEED();
};
ON_SCOPE_EXIT(threadWaitForExit(std::addressof(t_read)));
ON_SCOPE_EXIT(threadWaitForExit(std::addressof(t_write)));
if (sfunc) {
log_write("[THREAD] doing sfuncn\n");
t_data.SetPullResult(sfunc(start_threads, [&](void* data, s64 size, u64* bytes_read) -> Result {
R_TRY(t_data.GetResults());
return t_data.Pull(data, size, bytes_read);
}));
}
break;
}
log_write("threads closed\n");
else {
log_write("[THREAD] doing normal\n");
R_TRY(start_threads());
log_write("[THREAD] started threads\n");
// if any of the threads failed, wake up all threads so they can exit.
if (R_FAILED(t_data.GetResults())) {
log_write("some reads failed, waking threads\n");
log_write("returning due to fail\n");
while (t_data.GetWriteOffset() != t_data.GetWriteSize() && R_SUCCEEDED(t_data.GetResults())) {
pbox->UpdateTransfer(t_data.GetWriteOffset(), t_data.GetWriteSize());
svcSleepThread(1e+6);
}
}
// wait for all threads to close.
log_write("waiting for threads to close\n");
for (;;) {
t_data.WakeAllThreads();
pbox->Yield();
if (R_FAILED(waitSingleHandle(t_read.handle, 1000))) {
continue;
} else if (R_FAILED(waitSingleHandle(t_write.handle, 1000))) {
continue;
}
break;
}
log_write("threads closed\n");
// if any of the threads failed, wake up all threads so they can exit.
if (R_FAILED(t_data.GetResults())) {
log_write("some reads failed, waking threads\n");
log_write("returning due to fail\n");
return t_data.GetResults();
}
log_write("returning from thread func\n");
return t_data.GetResults();
}
return t_data.GetResults();
}
} // namespace
Result Transfer(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, WriteCallback wfunc) {
return TransferInternal(pbox, size, rfunc, wfunc, nullptr);
Result Transfer(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, WriteCallback wfunc, Mode mode) {
return TransferInternal(pbox, size, rfunc, wfunc, nullptr, Mode::MultiThreaded);
}
Result TransferPull(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, StartCallback sfunc) {
Result TransferPull(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, StartCallback sfunc, Mode mode) {
return TransferInternal(pbox, size, rfunc, nullptr, [sfunc](StartThreadCallback start, PullCallback pull) -> Result {
R_TRY(start());
return sfunc(pull);
});
}, Mode::MultiThreaded);
}
Result TransferPull(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, StartCallback2 sfunc) {
return TransferInternal(pbox, size, rfunc, nullptr, sfunc);
Result TransferPull(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, StartCallback2 sfunc, Mode mode) {
return TransferInternal(pbox, size, rfunc, nullptr, sfunc, Mode::MultiThreaded);
}
Result TransferUnzip(ui::ProgressBox* pbox, void* zfile, fs::Fs* fs, const fs::FsPath& path, s64 size, u32 crc32) {
Result rc;
if (R_FAILED(rc = fs->CreateDirectoryRecursivelyWithPath(path)) && rc != FsError_PathAlreadyExists) {
log_write("failed to create folder: %s 0x%04X\n", path.s, rc);
R_THROW(rc);
}
if (R_FAILED(rc = fs->CreateFile(path, size, 0)) && rc != FsError_PathAlreadyExists) {
log_write("failed to create file: %s 0x%04X\n", path.s, rc);
R_THROW(rc);
}
fs::File f;
R_TRY(fs->OpenFile(path, FsOpenMode_Write, &f));
// only update the size if this is an existing file.
if (rc == FsError_PathAlreadyExists) {
R_TRY(f.SetSize(size));
}
// NOTES: do not use temp file with rename / delete after as it massively slows
// down small file transfers (RA 21s -> 50s).
u32 crc32_out{};
R_TRY(thread::TransferInternal(pbox, size,
[&](void* data, s64 off, s64 size, u64* bytes_read) -> Result {
const auto result = unzReadCurrentFile(zfile, data, size);
if (result <= 0) {
// log_write("failed to read zip file: %s\n", inzip.c_str());
R_THROW(0x1);
}
if (crc32) {
crc32_out = crc32CalculateWithSeed(crc32_out, data, result);
}
*bytes_read = result;
R_SUCCEED();
},
[&](const void* data, s64 off, s64 size) -> Result {
return f.Write(off, data, size, FsWriteOption_None);
},
nullptr, Mode::SingleThreadedIfSmaller, SMALL_BUFFER_SIZE
));
// validate crc32 (if set in the info).
R_UNLESS(!crc32 || crc32 == crc32_out, 0x1);
R_SUCCEED();
}
Result TransferZip(ui::ProgressBox* pbox, void* zfile, fs::Fs* fs, const fs::FsPath& path) {
fs::File f;
R_TRY(fs->OpenFile(path, FsOpenMode_Read, &f));
s64 file_size;
R_TRY(f.GetSize(&file_size));
return thread::TransferInternal(pbox, file_size,
[&](void* data, s64 off, s64 size, u64* bytes_read) -> Result {
return f.Read(off, data, size, FsReadOption_None, bytes_read);
},
[&](const void* data, s64 off, s64 size) -> Result {
if (ZIP_OK != zipWriteInFileInZip(zfile, data, size)) {
log_write("failed to write zip file: %s\n", path.s);
R_THROW(0x1);
}
R_SUCCEED();
},
nullptr, Mode::SingleThreadedIfSmaller, SMALL_BUFFER_SIZE
);
}
Result TransferUnzipAll(ui::ProgressBox* pbox, void* zfile, fs::Fs* fs, const fs::FsPath& base_path, UnzipAllFilter filter) {
unz_global_info64 ginfo;
if (UNZ_OK != unzGetGlobalInfo64(zfile, &ginfo)) {
R_THROW(0x1);
}
if (UNZ_OK != unzGoToFirstFile(zfile)) {
R_THROW(0x1);
}
for (s64 i = 0; i < ginfo.number_entry; i++) {
R_TRY(pbox->ShouldExitResult());
if (i > 0) {
if (UNZ_OK != unzGoToNextFile(zfile)) {
log_write("failed to unzGoToNextFile\n");
R_THROW(0x1);
}
}
if (UNZ_OK != unzOpenCurrentFile(zfile)) {
log_write("failed to open current file\n");
R_THROW(0x1);
}
ON_SCOPE_EXIT(unzCloseCurrentFile(zfile));
unz_file_info64 info;
fs::FsPath name;
if (UNZ_OK != unzGetCurrentFileInfo64(zfile, &info, name, sizeof(name), 0, 0, 0, 0)) {
log_write("failed to get current info\n");
R_THROW(0x1);
}
// check if we should skip this file.
// don't make const as to allow the function to modify the path
// this function is used for the updater to change sphaira.nro to exe path.
auto path = fs::AppendPath(base_path, name);
if (filter && !filter(name, path)) {
continue;
}
pbox->NewTransfer(name);
if (path[std::strlen(path) -1] == '/') {
Result rc;
if (R_FAILED(rc = fs->CreateDirectoryRecursively(path)) && rc != FsError_PathAlreadyExists) {
log_write("failed to create folder: %s 0x%04X\n", path.s, rc);
R_THROW(rc);
}
} else {
R_TRY(TransferUnzip(pbox, zfile, fs, path, info.uncompressed_size, info.crc));
}
}
R_SUCCEED();
}
Result TransferUnzipAll(ui::ProgressBox* pbox, const fs::FsPath& zip_out, fs::Fs* fs, const fs::FsPath& base_path, UnzipAllFilter filter) {
auto zfile = unzOpen64(zip_out);
R_UNLESS(zfile, 0x1);
ON_SCOPE_EXIT(unzClose(zfile));
return TransferUnzipAll(pbox, zfile, fs, base_path, filter);
}
} // namespace::thread

2
sphaira/source/ui/error_box.cpp
View File

@@ -6,6 +6,7 @@
namespace sphaira::ui {
ErrorBox::ErrorBox(const std::string& message) : m_message{message} {
log_write("[ERROR] %s\n", m_message.c_str());
m_pos.w = 770.f;
m_pos.h = 430.f;
@@ -21,6 +22,7 @@ ErrorBox::ErrorBox(const std::string& message) : m_message{message} {
ErrorBox::ErrorBox(Result code, const std::string& message) : ErrorBox{message} {
m_code = code;
log_write("[ERROR] Code: 0x%X Module: %u Description: %u\n", R_VALUE(code), R_MODULE(code), R_DESCRIPTION(code));
}
auto ErrorBox::Update(Controller* controller, TouchInfo* touch) -> void {

229
sphaira/source/ui/menus/appstore.cpp
View File

@@ -14,6 +14,7 @@
#include "swkbd.hpp"
#include "i18n.hpp"
#include "hasher.hpp"
#include "threaded_file_transfer.hpp"
#include "nro.hpp"
#include <minIni.h>
@@ -75,6 +76,62 @@ constexpr const char* ORDER_STR[] = {
"Asc",
};
struct MzMem {
const void* buf;
size_t size;
size_t offset;
};
ZPOS64_T minizip_tell_file_func(voidpf opaque, voidpf stream) {
auto mem = static_cast<const MzMem*>(opaque);
return mem->offset;
}
long minizip_seek_file_func(voidpf opaque, voidpf stream, ZPOS64_T offset, int origin) {
auto mem = static_cast<MzMem*>(opaque);
size_t new_offset = 0;
switch (origin) {
case ZLIB_FILEFUNC_SEEK_SET: new_offset = offset; break;
case ZLIB_FILEFUNC_SEEK_CUR: new_offset = mem->offset + offset; break;
case ZLIB_FILEFUNC_SEEK_END: new_offset = (mem->size - 1) + offset; break;
default: return -1;
}
if (new_offset > mem->size) {
return -1;
}
mem->offset = new_offset;
return 0;
}
voidpf minizip_open_file_func(voidpf opaque, const void* filename, int mode) {
return opaque;
}
uLong minizip_read_file_func(voidpf opaque, voidpf stream, void* buf, uLong size) {
auto mem = static_cast<MzMem*>(opaque);
size = std::min(size, mem->size - mem->offset);
std::memcpy(buf, (const u8*)mem->buf + mem->offset, size);
mem->offset += size;
return size;
}
int minizip_close_file_func(voidpf opaque, voidpf stream) {
return 0;
}
constexpr zlib_filefunc64_def zlib_filefunc = {
.zopen64_file = minizip_open_file_func,
.zread_file = minizip_read_file_func,
.ztell64_file = minizip_tell_file_func,
.zseek64_file = minizip_seek_file_func,
.zclose_file = minizip_close_file_func,
};
auto BuildIconUrl(const Entry& e) -> std::string {
char out[0x100];
std::snprintf(out, sizeof(out), "%s/packages/%s/icon.png", URL_BASE, e.name.c_str());
@@ -363,19 +420,30 @@ auto InstallApp(ProgressBox* pbox, const Entry& entry) -> Result {
fs::FsNativeSd fs;
R_TRY(fs.GetFsOpenResult());
// check if we can download the entire zip to mem for faster download / extract times.
// current limit is 300MiB, or disabled for applet mode.
const auto file_download = App::IsApplet() || entry.filesize >= 1024 * 1024 * 300;
curl::ApiResult api_result{};
// 1. download the zip
if (!pbox->ShouldExit()) {
pbox->NewTransfer("Downloading "_i18n + entry.title);
log_write("starting download\n");
const auto url = BuildZipUrl(entry);
const auto result = curl::Api().ToFile(
curl::Api api{
curl::Url{url},
curl::Path{zip_out},
curl::OnProgress{pbox->OnDownloadProgressCallback()}
);
};
R_UNLESS(result.success, 0x1);
if (file_download) {
api.SetOption(curl::Path{zip_out});
api_result = curl::ToFile(api);
} else {
api_result = curl::ToMemory(api);
}
R_UNLESS(api_result.success, 0x1);
}
ON_SCOPE_EXIT(fs.DeleteFile(zip_out));
@@ -386,7 +454,11 @@ auto InstallApp(ProgressBox* pbox, const Entry& entry) -> Result {
log_write("starting md5 check\n");
std::string hash_out;
R_TRY(hash::Hash(pbox, hash::Type::Md5, &fs, zip_out, hash_out));
if (file_download) {
R_TRY(hash::Hash(pbox, hash::Type::Md5, &fs, zip_out, hash_out));
} else {
R_TRY(hash::Hash(pbox, hash::Type::Md5, api_result.data, hash_out));
}
if (strncasecmp(hash_out.data(), entry.md5.data(), entry.md5.length())) {
log_write("bad md5: %.*s vs %.*s\n", 32, hash_out.data(), 32, entry.md5.c_str());
@@ -394,9 +466,20 @@ auto InstallApp(ProgressBox* pbox, const Entry& entry) -> Result {
}
}
struct MzMem mem{};
mem.buf = api_result.data.data();
mem.size = api_result.data.size();
auto file_func = zlib_filefunc;
file_func.opaque = &mem;
zlib_filefunc64_def* file_func_ptr{};
if (!file_download) {
file_func_ptr = &file_func;
}
// 3. extract the zip
if (!pbox->ShouldExit()) {
auto zfile = unzOpen64(zip_out);
auto zfile = unzOpen2_64(zip_out, file_func_ptr);
R_UNLESS(zfile, 0x1);
ON_SCOPE_EXIT(unzClose(zfile));
@@ -434,43 +517,6 @@ auto InstallApp(ProgressBox* pbox, const Entry& entry) -> Result {
}
}
const auto unzip_to_file = [&](const unz_file_info64& info, const fs::FsPath& inzip, fs::FsPath output) -> Result {
if (output[0] != '/') {
output = fs::AppendPath("/", output);
}
// create directories
fs.CreateDirectoryRecursivelyWithPath(output);
Result rc;
if (R_FAILED(rc = fs.CreateFile(output, info.uncompressed_size, 0)) && rc != FsError_PathAlreadyExists) {
log_write("failed to create file: %s 0x%04X\n", output.s, rc);
R_THROW(rc);
}
fs::File f;
R_TRY(fs.OpenFile(output, FsOpenMode_Write, &f));
R_TRY(f.SetSize(info.uncompressed_size));
u64 offset{};
while (offset < info.uncompressed_size) {
R_TRY(pbox->ShouldExitResult());
const auto bytes_read = unzReadCurrentFile(zfile, buf.data(), buf.size());
if (bytes_read <= 0) {
log_write("failed to read zip file: %s\n", inzip.s);
R_THROW(0x1);
}
R_TRY(f.Write(offset, buf.data(), bytes_read, FsWriteOption_None));
pbox->UpdateTransfer(offset, info.uncompressed_size);
offset += bytes_read;
}
R_SUCCEED();
};
const auto unzip_to = [&](const fs::FsPath& inzip, const fs::FsPath& output) -> Result {
pbox->NewTransfer(inzip);
@@ -491,79 +537,46 @@ auto InstallApp(ProgressBox* pbox, const Entry& entry) -> Result {
R_THROW(0x1);
}
return unzip_to_file(info, inzip, output);
};
const auto unzip_all = [&](std::span<const ManifestEntry> entries) -> Result {
unz_global_info64 ginfo;
if (UNZ_OK != unzGetGlobalInfo64(zfile, &ginfo)) {
R_THROW(0x1);
auto path = output;
if (path[0] != '/') {
path = fs::AppendPath("/", path);
}
if (UNZ_OK != unzGoToFirstFile(zfile)) {
R_THROW(0x1);
}
for (s64 i = 0; i < ginfo.number_entry; i++) {
R_TRY(pbox->ShouldExitResult());
if (i > 0) {
if (UNZ_OK != unzGoToNextFile(zfile)) {
log_write("failed to unzGoToNextFile\n");
R_THROW(0x1);
}
}
if (UNZ_OK != unzOpenCurrentFile(zfile)) {
log_write("failed to open current file\n");
R_THROW(0x1);
}
ON_SCOPE_EXIT(unzCloseCurrentFile(zfile));
unz_file_info64 info;
char name[512];
if (UNZ_OK != unzGetCurrentFileInfo64(zfile, &info, name, sizeof(name), 0, 0, 0, 0)) {
log_write("failed to get current info\n");
R_THROW(0x1);
}
const auto it = std::ranges::find_if(entries, [&name](auto& e){
return !strcasecmp(name, e.path);
});
if (it == entries.end()) [[unlikely]] {
continue;
}
pbox->NewTransfer(it->path);
switch (it->command) {
case 'E': // both are the same?
case 'U':
break;
case 'G': { // checks if file exists, if not, extract
if (fs.FileExists(fs::AppendPath("/", it->path))) {
continue;
}
} break;
default:
log_write("bad command: %c\n", it->command);
continue;
}
R_TRY(unzip_to_file(info, it->path, it->path));
}
R_SUCCEED();
return thread::TransferUnzip(pbox, zfile, &fs, path, info.uncompressed_size, info.crc);
};
// unzip manifest, info and all entries.
TimeStamp ts;
#if 1
R_TRY(unzip_to("info.json", BuildInfoCachePath(entry)));
R_TRY(unzip_to("manifest.install", BuildManifestCachePath(entry)));
R_TRY(unzip_all(new_manifest));
#endif
R_TRY(thread::TransferUnzipAll(pbox, zfile, &fs, "/", [&](const fs::FsPath& name, fs::FsPath& path) -> bool {
const auto it = std::ranges::find_if(new_manifest, [&name](auto& e){
return !strcasecmp(name, e.path);
});
if (it == new_manifest.end()) [[unlikely]] {
return false;
}
pbox->NewTransfer(it->path);
switch (it->command) {
case 'E': // both are the same?
case 'U':
return true;
case 'G': // checks if file exists, if not, extract
return !fs.FileExists(fs::AppendPath("/", it->path));
default:
log_write("bad command: %c\n", it->command);
return false;
}
}));
log_write("\n\t[APPSTORE] finished extract new, time taken: %.2fs %zums\n\n", ts.GetSecondsD(), ts.GetMs());
// finally finally, remove files no longer in the manifest

107
sphaira/source/ui/menus/filebrowser.cpp
View File

@@ -694,6 +694,8 @@ void FsView::InstallFiles() {
}
R_SUCCEED();
}, [this](Result rc){
App::PushErrorBox(rc, "File install failed!"_i18n);
}));
}
}));
@@ -708,75 +710,10 @@ void FsView::UnzipFiles(fs::FsPath dir_path) {
}
App::Push(std::make_shared<ui::ProgressBox>(0, "Extracting "_i18n, "", [this, dir_path, targets](auto pbox) -> Result {
constexpr auto chunk_size = 1024 * 512; // 512KiB
for (auto& e : targets) {
pbox->SetTitle(e.GetName());
const auto zip_out = GetNewPath(e);
auto zfile = unzOpen64(zip_out);
R_UNLESS(zfile, 0x1);
ON_SCOPE_EXIT(unzClose(zfile));
unz_global_info64 pglobal_info;
if (UNZ_OK != unzGetGlobalInfo64(zfile, &pglobal_info)) {
R_THROW(0x1);
}
for (s64 i = 0; i < pglobal_info.number_entry; i++) {
if (i > 0) {
if (UNZ_OK != unzGoToNextFile(zfile)) {
log_write("failed to unzGoToNextFile\n");
R_THROW(0x1);
}
}
if (UNZ_OK != unzOpenCurrentFile(zfile)) {
log_write("failed to open current file\n");
R_THROW(0x1);
}
ON_SCOPE_EXIT(unzCloseCurrentFile(zfile));
unz_file_info64 info;
char name[512];
if (UNZ_OK != unzGetCurrentFileInfo64(zfile, &info, name, sizeof(name), 0, 0, 0, 0)) {
log_write("failed to get current info\n");
R_THROW(0x1);
}
const auto file_path = fs::AppendPath(dir_path, name);
pbox->NewTransfer(name);
// create directories
m_fs->CreateDirectoryRecursivelyWithPath(file_path);
Result rc;
if (R_FAILED(rc = m_fs->CreateFile(file_path, info.uncompressed_size, 0)) && rc != FsError_PathAlreadyExists) {
log_write("failed to create file: %s 0x%04X\n", file_path.s, rc);
R_THROW(rc);
}
fs::File f;
R_TRY(m_fs->OpenFile(file_path, FsOpenMode_Write, &f));
R_TRY(f.SetSize(info.uncompressed_size));
std::vector<char> buf(chunk_size);
s64 offset{};
while (offset < info.uncompressed_size) {
R_TRY(pbox->ShouldExitResult());
const auto bytes_read = unzReadCurrentFile(zfile, buf.data(), buf.size());
if (bytes_read <= 0) {
log_write("failed to read zip file: %s\n", name);
R_THROW(0x1);
}
R_TRY(f.Write(offset, buf.data(), bytes_read, FsWriteOption_None));
pbox->UpdateTransfer(offset, info.uncompressed_size);
offset += bytes_read;
}
}
R_TRY(thread::TransferUnzipAll(pbox, zip_out, m_fs.get(), dir_path));
}
R_SUCCEED();
@@ -829,8 +766,6 @@ void FsView::ZipFiles(fs::FsPath zip_out) {
}
App::Push(std::make_shared<ui::ProgressBox>(0, "Compressing "_i18n, "", [this, zip_out, targets](auto pbox) -> Result {
constexpr auto chunk_size = 1024 * 512; // 512KiB
const auto t = std::time(NULL);
const auto tm = std::localtime(&t);
@@ -851,6 +786,11 @@ void FsView::ZipFiles(fs::FsPath zip_out) {
// the file name needs to be relative to the current directory.
const char* file_name_in_zip = file_path.s + std::strlen(m_path);
// strip root path (/ or ums0:)
if (!std::strncmp(file_name_in_zip, m_fs->Root(), std::strlen(m_fs->Root()))) {
file_name_in_zip += std::strlen(m_fs->Root());
}
// root paths are banned in zips, they will warn when extracting otherwise.
if (file_name_in_zip[0] == '/') {
file_name_in_zip++;
@@ -858,38 +798,13 @@ void FsView::ZipFiles(fs::FsPath zip_out) {
pbox->NewTransfer(file_name_in_zip);
const auto ext = std::strrchr(file_name_in_zip, '.');
const auto raw = ext && IsExtension(ext + 1, COMPRESSED_EXTENSIONS);
if (ZIP_OK != zipOpenNewFileInZip2(zfile, file_name_in_zip, &zip_info, NULL, 0, NULL, 0, NULL, Z_DEFLATED, Z_DEFAULT_COMPRESSION, raw)) {
if (ZIP_OK != zipOpenNewFileInZip(zfile, file_name_in_zip, &zip_info, NULL, 0, NULL, 0, NULL, Z_DEFLATED, Z_DEFAULT_COMPRESSION)) {
log_write("failed to add zip for %s\n", file_path.s);
R_THROW(0x1);
}
ON_SCOPE_EXIT(zipCloseFileInZip(zfile));
fs::File f;
R_TRY(m_fs->OpenFile(file_path, FsOpenMode_Read, &f));
s64 file_size;
R_TRY(f.GetSize(&file_size));
std::vector<char> buf(chunk_size);
s64 offset{};
while (offset < file_size) {
R_TRY(pbox->ShouldExitResult());
u64 bytes_read;
R_TRY(f.Read(offset, buf.data(), buf.size(), FsReadOption_None, &bytes_read));
if (ZIP_OK != zipWriteInFileInZip(zfile, buf.data(), bytes_read)) {
log_write("failed to write zip file: %s\n", file_path.s);
R_THROW(0x1);
}
pbox->UpdateTransfer(offset, file_size);
offset += bytes_read;
}
R_SUCCEED();
return thread::TransferZip(pbox, zfile, m_fs.get(), file_path);
};
for (auto& e : targets) {

75
sphaira/source/ui/menus/ghdl.cpp
View File

@@ -14,9 +14,9 @@
#include "download.hpp"
#include "i18n.hpp"
#include "yyjson_helper.hpp"
#include "threaded_file_transfer.hpp"
#include <minIni.h>
#include <minizip/unzip.h>
#include <dirent.h>
#include <cstring>
#include <string>
@@ -83,7 +83,6 @@ void from_json(const fs::FsPath& path, GhApiEntry& e) {
auto DownloadApp(ProgressBox* pbox, const GhApiAsset& gh_asset, const AssetEntry* entry) -> Result {
static const fs::FsPath temp_file{"/switch/sphaira/cache/github/ghdl.temp"};
constexpr auto chunk_size = 1024 * 512; // 512KiB
fs::FsNativeSd fs;
R_TRY(fs.GetFsOpenResult());
@@ -113,77 +112,7 @@ auto DownloadApp(ProgressBox* pbox, const GhApiAsset& gh_asset, const AssetEntry
// 3. extract the zip / file
if (gh_asset.content_type.find("zip") != gh_asset.content_type.npos) {
log_write("found zip\n");
auto zfile = unzOpen64(temp_file);
R_UNLESS(zfile, 0x1);
ON_SCOPE_EXIT(unzClose(zfile));
unz_global_info64 pglobal_info;
if (UNZ_OK != unzGetGlobalInfo64(zfile, &pglobal_info)) {
R_THROW(0x1);
}
for (int i = 0; i < pglobal_info.number_entry; i++) {
if (i > 0) {
if (UNZ_OK != unzGoToNextFile(zfile)) {
log_write("failed to unzGoToNextFile\n");
R_THROW(0x1);
}
}
if (UNZ_OK != unzOpenCurrentFile(zfile)) {
log_write("failed to open current file\n");
R_THROW(0x1);
}
ON_SCOPE_EXIT(unzCloseCurrentFile(zfile));
unz_file_info64 info;
fs::FsPath file_path;
if (UNZ_OK != unzGetCurrentFileInfo64(zfile, &info, file_path, sizeof(file_path), 0, 0, 0, 0)) {
log_write("failed to get current info\n");
R_THROW(0x1);
}
file_path = fs::AppendPath(root_path, file_path);
Result rc;
if (file_path[strlen(file_path) -1] == '/') {
if (R_FAILED(rc = fs.CreateDirectoryRecursively(file_path)) && rc != FsError_PathAlreadyExists) {
log_write("failed to create folder: %s 0x%04X\n", file_path.s, rc);
R_THROW(rc);
}
} else {
if (R_FAILED(rc = fs.CreateDirectoryRecursivelyWithPath(file_path)) && rc != FsError_PathAlreadyExists) {
log_write("failed to create folder: %s 0x%04X\n", file_path.s, rc);
R_THROW(rc);
}
if (R_FAILED(rc = fs.CreateFile(file_path, info.uncompressed_size, 0)) && rc != FsError_PathAlreadyExists) {
log_write("failed to create file: %s 0x%04X\n", file_path.s, rc);
R_THROW(rc);
}
fs::File f;
R_TRY(fs.OpenFile(file_path, FsOpenMode_Write, &f));
R_TRY(f.SetSize(info.uncompressed_size));
std::vector<char> buf(chunk_size);
s64 offset{};
while (offset < info.uncompressed_size) {
R_TRY(pbox->ShouldExitResult());
const auto bytes_read = unzReadCurrentFile(zfile, buf.data(), buf.size());
if (bytes_read <= 0) {
log_write("failed to read zip file: %s\n", file_path.s);
R_THROW(0x1);
}
R_TRY(f.Write(offset, buf.data(), bytes_read, FsWriteOption_None));
pbox->UpdateTransfer(offset, info.uncompressed_size);
offset += bytes_read;
}
}
}
R_TRY(thread::TransferUnzipAll(pbox, temp_file, &fs, root_path));
} else {
fs.CreateDirectoryRecursivelyWithPath(root_path);
fs.DeleteFile(root_path);

110
sphaira/source/ui/menus/main_menu.cpp
View File

@@ -22,9 +22,9 @@
#include "download.hpp"
#include "defines.hpp"
#include "i18n.hpp"
#include "threaded_file_transfer.hpp"
#include <cstring>
#include <minizip/unzip.h>
#include <yyjson.h>
namespace sphaira::ui::menu::main {
@@ -59,7 +59,6 @@ const MiscMenuEntry MISC_MENU_ENTRIES[] = {
auto InstallUpdate(ProgressBox* pbox, const std::string url, const std::string version) -> Result {
static fs::FsPath zip_out{"/switch/sphaira/cache/update.zip"};
constexpr auto chunk_size = 1024 * 512; // 512KiB
fs::FsNativeSd fs;
R_TRY(fs.GetFsOpenResult());
@@ -82,95 +81,34 @@ auto InstallUpdate(ProgressBox* pbox, const std::string url, const std::string v
// 2. extract the zip
if (!pbox->ShouldExit()) {
auto zfile = unzOpen64(zip_out);
R_UNLESS(zfile, 0x1);
ON_SCOPE_EXIT(unzClose(zfile));
const auto exe_path = App::GetExePath();
bool found_exe{};
unz_global_info64 pglobal_info;
if (UNZ_OK != unzGetGlobalInfo64(zfile, &pglobal_info)) {
R_THROW(0x1);
}
for (s64 i = 0; i < pglobal_info.number_entry; i++) {
if (i > 0) {
if (UNZ_OK != unzGoToNextFile(zfile)) {
log_write("failed to unzGoToNextFile\n");
R_THROW(0x1);
}
R_TRY(thread::TransferUnzipAll(pbox, zip_out, &fs, "/", [&](const fs::FsPath& name, fs::FsPath& path) -> bool {
if (std::strstr(path, "sphaira.nro")) {
path = exe_path;
found_exe = true;
}
return true;
}));
if (UNZ_OK != unzOpenCurrentFile(zfile)) {
log_write("failed to open current file\n");
R_THROW(0x1);
}
ON_SCOPE_EXIT(unzCloseCurrentFile(zfile));
unz_file_info64 info;
fs::FsPath file_path;
if (UNZ_OK != unzGetCurrentFileInfo64(zfile, &info, file_path, sizeof(file_path), 0, 0, 0, 0)) {
log_write("failed to get current info\n");
R_THROW(0x1);
}
if (file_path[0] != '/') {
file_path = fs::AppendPath("/", file_path);
}
if (std::strstr(file_path, "sphaira.nro")) {
file_path = App::GetExePath();
}
Result rc;
if (file_path[std::strlen(file_path) -1] == '/') {
if (R_FAILED(rc = fs.CreateDirectoryRecursively(file_path)) && rc != FsError_PathAlreadyExists) {
log_write("failed to create folder: %s 0x%04X\n", file_path.s, rc);
R_THROW(rc);
}
} else {
Result rc;
if (R_FAILED(rc = fs.CreateFile(file_path, info.uncompressed_size, 0)) && rc != FsError_PathAlreadyExists) {
log_write("failed to create file: %s 0x%04X\n", file_path.s, rc);
R_THROW(rc);
// check if we have sphaira installed in other locations and update them.
if (found_exe) {
for (auto& path : SPHAIRA_PATHS) {
log_write("[UPD] checking path: %s\n", path.s);
// skip if we already updated this path.
if (exe_path == path) {
log_write("[UPD] skipped as already updated\n");
continue;
}
fs::File f;
R_TRY(fs.OpenFile(file_path, FsOpenMode_Write, &f));
R_TRY(f.SetSize(info.uncompressed_size));
std::vector<char> buf(chunk_size);
s64 offset{};
while (offset < info.uncompressed_size) {
const auto bytes_read = unzReadCurrentFile(zfile, buf.data(), buf.size());
if (bytes_read <= 0) {
// log_write("failed to read zip file: %s\n", inzip.c_str());
R_THROW(0x1);
}
R_TRY(f.Write(offset, buf.data(), bytes_read, FsWriteOption_None));
pbox->UpdateTransfer(offset, info.uncompressed_size);
offset += bytes_read;
}
}
// check if we have sphaira installed in other locations and update them.
if (file_path == App::GetExePath()) {
for (auto& path : SPHAIRA_PATHS) {
log_write("[UPD] checking path: %s\n", path.s);
// skip if we already updated this path.
if (file_path == path) {
log_write("[UPD] skipped as already updated\n");
continue;
}
// check that this is really sphaira.
log_write("[UPD] checking nacp\n");
NacpStruct nacp;
if (R_SUCCEEDED(nro_get_nacp(path, nacp)) && !std::strcmp(nacp.lang[0].name, "sphaira")) {
log_write("[UPD] found, updating\n");
pbox->NewTransfer(path);
R_TRY(pbox->CopyFile(&fs, file_path, path));
}
// check that this is really sphaira.
log_write("[UPD] checking nacp\n");
NacpStruct nacp;
if (R_SUCCEEDED(nro_get_nacp(path, nacp)) && !std::strcmp(nacp.lang[0].name, "sphaira")) {
log_write("[UPD] found, updating\n");
pbox->NewTransfer(path);
R_TRY(pbox->CopyFile(&fs, exe_path, path));
}
}
}

64
sphaira/source/ui/menus/themezer.cpp
View File

@@ -11,11 +11,11 @@
#include "ui/nvg_util.hpp"
#include "swkbd.hpp"
#include "i18n.hpp"
#include "threaded_file_transfer.hpp"
#include <minIni.h>
#include <stb_image.h>
#include <cstring>
#include <minizip/unzip.h>
#include <yyjson.h>
#include "yyjson_helper.hpp"
@@ -222,7 +222,6 @@ void from_json(const fs::FsPath& path, PackList& e) {
auto InstallTheme(ProgressBox* pbox, const PackListEntry& entry) -> Result {
static const fs::FsPath zip_out{"/switch/sphaira/cache/themezer/temp.zip"};
constexpr auto chunk_size = 1024 * 512; // 512KiB
fs::FsNativeSd fs;
R_TRY(fs.GetFsOpenResult());
@@ -272,66 +271,7 @@ auto InstallTheme(ProgressBox* pbox, const PackListEntry& entry) -> Result {
// 3. extract the zip
if (!pbox->ShouldExit()) {
auto zfile = unzOpen64(zip_out);
R_UNLESS(zfile, 0x1);
ON_SCOPE_EXIT(unzClose(zfile));
unz_global_info64 pglobal_info;
if (UNZ_OK != unzGetGlobalInfo64(zfile, &pglobal_info)) {
R_THROW(0x1);
}
for (int i = 0; i < pglobal_info.number_entry; i++) {
if (i > 0) {
if (UNZ_OK != unzGoToNextFile(zfile)) {
log_write("failed to unzGoToNextFile\n");
R_THROW(0x1);
}
}
if (UNZ_OK != unzOpenCurrentFile(zfile)) {
log_write("failed to open current file\n");
R_THROW(0x1);
}
ON_SCOPE_EXIT(unzCloseCurrentFile(zfile));
unz_file_info64 info;
char name[512];
if (UNZ_OK != unzGetCurrentFileInfo64(zfile, &info, name, sizeof(name), 0, 0, 0, 0)) {
log_write("failed to get current info\n");
R_THROW(0x1);
}
const auto file_path = fs::AppendPath(dir_path, name);
pbox->NewTransfer(name);
Result rc;
if (R_FAILED(rc = fs.CreateFile(file_path, info.uncompressed_size, 0)) && rc != FsError_PathAlreadyExists) {
log_write("failed to create file: %s 0x%04X\n", file_path.s, rc);
R_THROW(rc);
}
fs::File f;
R_TRY(fs.OpenFile(file_path, FsOpenMode_Write, &f));
R_TRY(f.SetSize(info.uncompressed_size));
std::vector<char> buf(chunk_size);
s64 offset{};
while (offset < info.uncompressed_size) {
R_TRY(pbox->ShouldExitResult());
const auto bytes_read = unzReadCurrentFile(zfile, buf.data(), buf.size());
if (bytes_read <= 0) {
// log_write("failed to read zip file: %s\n", inzip.c_str());
R_THROW(0x1);
}
R_TRY(f.Write(offset, buf.data(), bytes_read, FsWriteOption_None));
pbox->UpdateTransfer(offset, info.uncompressed_size);
offset += bytes_read;
}
}
R_TRY(thread::TransferUnzipAll(pbox, zip_out, &fs, dir_path));
}
log_write("finished install :)\n");

28
sphaira/source/yati/yati.cpp
View File

@@ -433,7 +433,7 @@ Result Yati::decompressFuncInternal(ThreadData* t) {
inflate_buf.reserve(t->max_buffer_size);
s64 written{};
s64 decompress_buf_off{};
s64 block_offset{};
std::vector<u8> buf{};
buf.reserve(t->max_buffer_size);
@@ -454,14 +454,15 @@ Result Yati::decompressFuncInternal(ThreadData* t) {
}
for (s64 off = 0; off < size;) {
// log_write("looking for section\n");
if (!ncz_section || !ncz_section->InRange(written)) {
log_write("[NCZ] looking for new section: %zu\n", written);
auto it = std::ranges::find_if(t->ncz_sections, [written](auto& e){
return e.InRange(written);
});
R_UNLESS(it != t->ncz_sections.cend(), Result_NczSectionNotFound);
ncz_section = &(*it);
log_write("[NCZ] found new section: %zu\n", written);
if (ncz_section->crypto_type >= nca::EncryptionType_AesCtr) {
const auto swp = std::byteswap(u64(written) >> 4);
@@ -488,7 +489,7 @@ Result Yati::decompressFuncInternal(ThreadData* t) {
// restore remaining data to the swapped buffer.
if (!temp_vector.empty()) {
log_write("storing data size: %zu\n", temp_vector.size());
log_write("[NCZ] storing data size: %zu\n", temp_vector.size());
inflate_buf = temp_vector;
}
@@ -496,6 +497,7 @@ Result Yati::decompressFuncInternal(ThreadData* t) {
};
while (t->decompress_offset < t->write_size && R_SUCCEEDED(t->GetResults())) {
s64 decompress_buf_off{};
R_TRY(t->GetDecompressBuf(buf, decompress_buf_off));
// do we have an nsz? if so, setup buffers.
@@ -616,12 +618,14 @@ Result Yati::decompressFuncInternal(ThreadData* t) {
// todo: blocks need to use read offset, as the offset + size is compressed range.
if (t->ncz_blocks.size()) {
if (!ncz_block || !ncz_block->InRange(decompress_buf_off)) {
block_offset = 0;
log_write("[NCZ] looking for new block: %zu\n", decompress_buf_off);
auto it = std::ranges::find_if(t->ncz_blocks, [decompress_buf_off](auto& e){
return e.InRange(decompress_buf_off);
});
R_UNLESS(it != t->ncz_blocks.cend(), Result_NczBlockNotFound);
// log_write("looking found block\n");
log_write("[NCZ] found new block: %zu off: %zd size: %zd\n", decompress_buf_off, it->offset, it->size);
ncz_block = &(*it);
}
@@ -629,7 +633,7 @@ Result Yati::decompressFuncInternal(ThreadData* t) {
auto decompressedBlockSize = 1 << t->ncz_block_header.block_size_exponent;
// special handling for the last block to check it's actually compressed
if (ncz_block->offset == t->ncz_blocks.back().offset) {
log_write("last block special handling\n");
log_write("[NCZ] last block special handling\n");
decompressedBlockSize = t->ncz_block_header.decompressed_size % decompressedBlockSize;
}
@@ -637,12 +641,12 @@ Result Yati::decompressFuncInternal(ThreadData* t) {
compressed = ncz_block->size < decompressedBlockSize;
// clip read size as blocks can be up to 32GB in size!
const auto size = std::min<u64>(buf.size() - buf_off, ncz_block->size);
buffer = {buf.data() + buf_off, size};
const auto size = std::min<u64>(buffer.size(), ncz_block->size - block_offset);
buffer = buffer.subspan(0, size);
}
if (compressed) {
// log_write("COMPRESSED block\n");
log_write("[NCZ] COMPRESSED block\n");
ZSTD_inBuffer input = { buffer.data(), buffer.size(), 0 };
while (input.pos < input.size) {
R_TRY(t->GetResults());
@@ -650,12 +654,15 @@ Result Yati::decompressFuncInternal(ThreadData* t) {
inflate_buf.resize(inflate_offset + chunk_size);
ZSTD_outBuffer output = { inflate_buf.data() + inflate_offset, chunk_size, 0 };
const auto res = ZSTD_decompressStream(dctx, std::addressof(output), std::addressof(input));
if (ZSTD_isError(res)) {
log_write("[NCZ] ZSTD_decompressStream() pos: %zu size: %zu res: %zd msg: %s\n", input.pos, input.size, res, ZSTD_getErrorName(res));
}
R_UNLESS(!ZSTD_isError(res), Result_InvalidNczZstdError);
t->decompress_offset += output.pos;
inflate_offset += output.pos;
if (inflate_offset >= INFLATE_BUFFER_MAX) {
// log_write("flushing compressed data: %zd vs %zd diff: %zd\n", inflate_offset, INFLATE_BUFFER_MAX, inflate_offset - INFLATE_BUFFER_MAX);
log_write("[NCZ] flushing compressed data: %zd vs %zd diff: %zd\n", inflate_offset, INFLATE_BUFFER_MAX, inflate_offset - INFLATE_BUFFER_MAX);
R_TRY(ncz_flush(INFLATE_BUFFER_MAX));
}
}
@@ -666,13 +673,14 @@ Result Yati::decompressFuncInternal(ThreadData* t) {
t->decompress_offset += buffer.size();
inflate_offset += buffer.size();
if (inflate_offset >= INFLATE_BUFFER_MAX) {
// log_write("flushing copy data\n");
log_write("[NCZ] flushing copy data\n");
R_TRY(ncz_flush(INFLATE_BUFFER_MAX));
}
}
buf_off += buffer.size();
decompress_buf_off += buffer.size();
block_offset += buffer.size();
}
}
}