OmniNX/sphaira
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

multi-thread zip and unzip code. option to download appstore zip to mem. hasher mem support.

Browse Source
This commit is contained in:
ITotalJustice
2025-05-30 12:34:29 +01:00
parent 17b341d83a
commit 390c1e870d
9 changed files with 468 additions and 510 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
sphaira/include/hasher.hpp
View File

@@ -4,6 +4,7 @@
#include "ui/progress_box.hpp"
#include <string>
#include <memory>
#include <span>
#include <switch.h>
namespace sphaira::hash {
@@ -26,5 +27,6 @@ auto GetTypeStr(Type type) -> const char*;
// returns the hash string.
Result Hash(ui::ProgressBox* pbox, Type type, std::shared_ptr<BaseSource> source, std::string& out);
Result Hash(ui::ProgressBox* pbox, Type type, fs::Fs* fs, const fs::FsPath& path, std::string& out);
Result Hash(ui::ProgressBox* pbox, Type type, std::span<const u8> data, std::string& out);
} // namespace sphaira::hash

30
sphaira/include/threaded_file_transfer.hpp
View File

@@ -6,6 +6,15 @@
namespace sphaira::thread {
enum class Mode {
// default, always multi-thread.
MultiThreaded,
// always single-thread.
SingleThreaded,
// check buffer size, if smaller, single thread.
SingleThreadedIfSmaller,
};
using ReadCallback = std::function<Result(void* data, s64 off, s64 size, u64* bytes_read)>;
using WriteCallback = std::function<Result(const void* data, s64 off, s64 size)>;
@@ -23,10 +32,25 @@ using StartCallback = std::function<Result(PullCallback pull)>;
using StartCallback2 = std::function<Result(StartThreadCallback start, PullCallback pull)>;
// reads data from rfunc into wfunc.
Result Transfer(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, WriteCallback wfunc);
Result Transfer(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, WriteCallback wfunc, Mode mode = Mode::MultiThreaded);
// reads data from rfunc, pull data from provided pull() callback.
Result TransferPull(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, StartCallback sfunc);
Result TransferPull(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, StartCallback2 sfunc);
Result TransferPull(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, StartCallback sfunc, Mode mode = Mode::MultiThreaded);
Result TransferPull(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, StartCallback2 sfunc, Mode mode = Mode::MultiThreaded);
// helper for extract zips.
// this will multi-thread unzip if size >= 512KiB, otherwise it'll single pass.
Result TransferUnzip(ui::ProgressBox* pbox, void* zfile, fs::Fs* fs, const fs::FsPath& path, s64 size, u32 crc32 = 0);
// same as above but for zipping files.
Result TransferZip(ui::ProgressBox* pbox, void* zfile, fs::Fs* fs, const fs::FsPath& path);
// passes the name inside the zip an final output path.
using UnzipAllFilter = std::function<bool(const fs::FsPath& name, fs::FsPath& path)>;
// helper all-in-one unzip function that unzips a zip (either open or path provided).
// the filter function can be used to modify the path and filter out unwanted files.
Result TransferUnzipAll(ui::ProgressBox* pbox, void* zfile, fs::Fs* fs, const fs::FsPath& base_path, UnzipAllFilter filter = nullptr);
Result TransferUnzipAll(ui::ProgressBox* pbox, const fs::FsPath& zip_out, fs::Fs* fs, const fs::FsPath& base_path, UnzipAllFilter filter = nullptr);
} // namespace sphaira::thread

24
sphaira/source/hasher.cpp
View File

@@ -35,6 +35,25 @@ private:
bool m_is_file_based_emummc{};
};
struct MemSource final : BaseSource {
MemSource(std::span<const u8> data) : m_data{data} { }
Result Size(s64* out) {
*out = m_data.size();
R_SUCCEED();
}
Result Read(void* buf, s64 off, s64 size, u64* bytes_read) {
size = std::min<s64>(size, m_data.size() - off);
std::memcpy(buf, m_data.data() + off, size);
*bytes_read = size;
R_SUCCEED();
}
private:
const std::span<const u8> m_data;
};
struct HashSource {
virtual ~HashSource() = default;
virtual void Update(const void* buf, s64 size) = 0;
@@ -181,4 +200,9 @@ Result Hash(ui::ProgressBox* pbox, Type type, fs::Fs* fs, const fs::FsPath& path
return Hash(pbox, type, source, out);
}
Result Hash(ui::ProgressBox* pbox, Type type, std::span<const u8> data, std::string& out) {
auto source = std::make_shared<MemSource>(data);
return Hash(pbox, type, source, out);
}
} // namespace sphaira::has

339
sphaira/source/threaded_file_transfer.cpp
View File

@@ -6,15 +6,20 @@
#include <vector>
#include <algorithm>
#include <cstring>
#include <minizip/unzip.h>
#include <minizip/zip.h>
namespace sphaira::thread {
namespace {
constexpr u64 READ_BUFFER_MAX = 1024*1024*4;
// used for file based emummc and zip/unzip.
constexpr u64 SMALL_BUFFER_SIZE = 1024 * 512;
// used for everything else.
constexpr u64 NORMAL_BUFFER_SIZE = 1024*1024*4;
struct ThreadBuffer {
ThreadBuffer() {
buf.reserve(READ_BUFFER_MAX);
buf.reserve(NORMAL_BUFFER_SIZE);
}
std::vector<u8> buf;
@@ -65,7 +70,7 @@ public:
};
struct ThreadData {
ThreadData(ui::ProgressBox* _pbox, s64 size, ReadCallback _rfunc, WriteCallback _wfunc);
ThreadData(ui::ProgressBox* _pbox, s64 size, ReadCallback _rfunc, WriteCallback _wfunc, u64 buffer_size);
auto GetResults() -> Result;
void WakeAllThreads();
@@ -104,9 +109,9 @@ private:
private:
// these need to be copied
ui::ProgressBox* pbox{};
ReadCallback rfunc{};
WriteCallback wfunc{};
ui::ProgressBox* const pbox;
const ReadCallback rfunc;
const WriteCallback wfunc;
// these need to be created
Mutex mutex{};
@@ -121,21 +126,24 @@ private:
std::vector<u8> pull_buffer{};
s64 pull_buffer_offset{};
u64 read_buffer_size{};
u64 max_buffer_size{};
const u64 read_buffer_size;
const s64 write_size;
// these are shared between threads
volatile s64 read_offset{};
volatile s64 write_offset{};
volatile s64 write_size{};
volatile Result read_result{};
volatile Result write_result{};
volatile Result pull_result{};
};
ThreadData::ThreadData(ui::ProgressBox* _pbox, s64 size, ReadCallback _rfunc, WriteCallback _wfunc)
: pbox{_pbox}, rfunc{_rfunc}, wfunc{_wfunc} {
ThreadData::ThreadData(ui::ProgressBox* _pbox, s64 size, ReadCallback _rfunc, WriteCallback _wfunc, u64 buffer_size)
: pbox{_pbox}
, rfunc{_rfunc}
, wfunc{_wfunc}
, read_buffer_size{buffer_size}
, write_size{size} {
mutexInit(std::addressof(mutex));
mutexInit(std::addressof(pull_mutex));
@@ -143,16 +151,6 @@ ThreadData::ThreadData(ui::ProgressBox* _pbox, s64 size, ReadCallback _rfunc, Wr
condvarInit(std::addressof(can_write));
condvarInit(std::addressof(can_pull));
condvarInit(std::addressof(can_pull_write));
write_size = size;
if (App::IsFileBaseEmummc()) {
read_buffer_size = 1024 * 512;
max_buffer_size = 1024 * 512;
} else {
read_buffer_size = READ_BUFFER_MAX;
max_buffer_size = READ_BUFFER_MAX;
}
}
auto ThreadData::GetResults() -> Result {
@@ -251,7 +249,7 @@ Result ThreadData::Pull(void* data, s64 size, u64* bytes_read) {
Result ThreadData::readFuncInternal() {
// the main buffer which data is read into.
std::vector<u8> buf;
buf.reserve(this->max_buffer_size);
buf.reserve(this->read_buffer_size);
while (this->read_offset < this->write_size && R_SUCCEEDED(this->GetResults())) {
// read more data
@@ -265,14 +263,14 @@ Result ThreadData::readFuncInternal() {
R_TRY(this->SetWriteBuf(buf, buf_size));
}
log_write("read success\n");
log_write("finished read thread success!\n");
R_SUCCEED();
}
// write thread writes data to the nca placeholder.
Result ThreadData::writeFuncInternal() {
std::vector<u8> buf;
buf.reserve(this->max_buffer_size);
buf.reserve(this->read_buffer_size);
while (this->write_offset < this->write_size && R_SUCCEEDED(this->GetResults())) {
s64 dummy_off;
@@ -288,7 +286,7 @@ Result ThreadData::writeFuncInternal() {
this->write_offset += size;
}
log_write("finished write thread!\n");
log_write("finished write thread success!\n");
R_SUCCEED();
}
@@ -308,87 +306,264 @@ auto GetAlternateCore(int id) {
return id == 1 ? 2 : 1;
}
Result TransferInternal(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, WriteCallback wfunc, StartCallback2 sfunc) {
App::SetAutoSleepDisabled(true);
ON_SCOPE_EXIT(App::SetAutoSleepDisabled(false));
Result TransferInternal(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, WriteCallback wfunc, StartCallback2 sfunc, Mode mode, u64 buffer_size = NORMAL_BUFFER_SIZE) {
const auto is_file_based_emummc = App::IsFileBaseEmummc();
const auto WRITE_THREAD_CORE = sfunc ? pbox->GetCpuId() : GetAlternateCore(pbox->GetCpuId());
const auto READ_THREAD_CORE = GetAlternateCore(WRITE_THREAD_CORE);
if (is_file_based_emummc) {
buffer_size = SMALL_BUFFER_SIZE;
}
ThreadData t_data{pbox, size, rfunc, wfunc};
if (mode == Mode::SingleThreadedIfSmaller) {
if (size <= buffer_size) {
mode = Mode::SingleThreaded;
} else {
mode = Mode::MultiThreaded;
}
}
Thread t_read{};
R_TRY(threadCreate(&t_read, readFunc, std::addressof(t_data), nullptr, 1024*64, PRIO_PREEMPTIVE, READ_THREAD_CORE));
ON_SCOPE_EXIT(threadClose(&t_read));
// single threaded pull buffer is not supported.
R_UNLESS(mode != Mode::MultiThreaded || !sfunc, 0x1);
Thread t_write{};
R_TRY(threadCreate(&t_write, writeFunc, std::addressof(t_data), nullptr, 1024*64, PRIO_PREEMPTIVE, WRITE_THREAD_CORE));
ON_SCOPE_EXIT(threadClose(&t_write));
// todo: support single threaded pull buffer.
if (mode == Mode::SingleThreaded) {
std::vector<u8> buf(buffer_size);
s64 offset{};
while (offset < size) {
R_TRY(pbox->ShouldExitResult());
u64 bytes_read;
const auto rsize = std::min<s64>(buf.size(), size - offset);
R_TRY(rfunc(buf.data(), offset, rsize, &bytes_read));
R_TRY(wfunc(buf.data(), offset, bytes_read));
offset += bytes_read;
pbox->UpdateTransfer(offset, size);
}
const auto start_threads = [&]() -> Result {
log_write("starting threads\n");
R_TRY(threadStart(std::addressof(t_read)));
R_TRY(threadStart(std::addressof(t_write)));
R_SUCCEED();
};
ON_SCOPE_EXIT(threadWaitForExit(std::addressof(t_read)));
ON_SCOPE_EXIT(threadWaitForExit(std::addressof(t_write)));
if (sfunc) {
t_data.SetPullResult(sfunc(start_threads, [&](void* data, s64 size, u64* bytes_read) -> Result {
R_TRY(t_data.GetResults());
return t_data.Pull(data, size, bytes_read);
}));
} else {
R_TRY(start_threads());
while (t_data.GetWriteOffset() != t_data.GetWriteSize() && R_SUCCEEDED(t_data.GetResults())) {
pbox->UpdateTransfer(t_data.GetWriteOffset(), t_data.GetWriteSize());
svcSleepThread(1e+6);
}
}
else {
const auto WRITE_THREAD_CORE = sfunc ? pbox->GetCpuId() : GetAlternateCore(pbox->GetCpuId());
const auto READ_THREAD_CORE = GetAlternateCore(WRITE_THREAD_CORE);
// wait for all threads to close.
log_write("waiting for threads to close\n");
for (;;) {
t_data.WakeAllThreads();
pbox->Yield();
ThreadData t_data{pbox, size, rfunc, wfunc, buffer_size};
if (R_FAILED(waitSingleHandle(t_read.handle, 1000))) {
continue;
} else if (R_FAILED(waitSingleHandle(t_write.handle, 1000))) {
continue;
Thread t_read{};
R_TRY(threadCreate(&t_read, readFunc, std::addressof(t_data), nullptr, 1024*256, 0x3B, READ_THREAD_CORE));
ON_SCOPE_EXIT(threadClose(&t_read));
Thread t_write{};
R_TRY(threadCreate(&t_write, writeFunc, std::addressof(t_data), nullptr, 1024*256, 0x3B, WRITE_THREAD_CORE));
ON_SCOPE_EXIT(threadClose(&t_write));
const auto start_threads = [&]() -> Result {
log_write("starting threads\n");
R_TRY(threadStart(std::addressof(t_read)));
R_TRY(threadStart(std::addressof(t_write)));
R_SUCCEED();
};
ON_SCOPE_EXIT(threadWaitForExit(std::addressof(t_read)));
ON_SCOPE_EXIT(threadWaitForExit(std::addressof(t_write)));
if (sfunc) {
log_write("[THREAD] doing sfuncn\n");
t_data.SetPullResult(sfunc(start_threads, [&](void* data, s64 size, u64* bytes_read) -> Result {
R_TRY(t_data.GetResults());
return t_data.Pull(data, size, bytes_read);
}));
}
break;
}
log_write("threads closed\n");
else {
log_write("[THREAD] doing normal\n");
R_TRY(start_threads());
log_write("[THREAD] started threads\n");
// if any of the threads failed, wake up all threads so they can exit.
if (R_FAILED(t_data.GetResults())) {
log_write("some reads failed, waking threads\n");
log_write("returning due to fail\n");
while (t_data.GetWriteOffset() != t_data.GetWriteSize() && R_SUCCEEDED(t_data.GetResults())) {
pbox->UpdateTransfer(t_data.GetWriteOffset(), t_data.GetWriteSize());
svcSleepThread(1e+6);
}
}
// wait for all threads to close.
log_write("waiting for threads to close\n");
for (;;) {
t_data.WakeAllThreads();
pbox->Yield();
if (R_FAILED(waitSingleHandle(t_read.handle, 1000))) {
continue;
} else if (R_FAILED(waitSingleHandle(t_write.handle, 1000))) {
continue;
}
break;
}
log_write("threads closed\n");
// if any of the threads failed, wake up all threads so they can exit.
if (R_FAILED(t_data.GetResults())) {
log_write("some reads failed, waking threads\n");
log_write("returning due to fail\n");
return t_data.GetResults();
}
log_write("returning from thread func\n");
return t_data.GetResults();
}
return t_data.GetResults();
}
} // namespace
Result Transfer(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, WriteCallback wfunc) {
return TransferInternal(pbox, size, rfunc, wfunc, nullptr);
Result Transfer(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, WriteCallback wfunc, Mode mode) {
return TransferInternal(pbox, size, rfunc, wfunc, nullptr, Mode::MultiThreaded);
}
Result TransferPull(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, StartCallback sfunc) {
Result TransferPull(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, StartCallback sfunc, Mode mode) {
return TransferInternal(pbox, size, rfunc, nullptr, [sfunc](StartThreadCallback start, PullCallback pull) -> Result {
R_TRY(start());
return sfunc(pull);
});
}, Mode::MultiThreaded);
}
Result TransferPull(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, StartCallback2 sfunc) {
return TransferInternal(pbox, size, rfunc, nullptr, sfunc);
Result TransferPull(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, StartCallback2 sfunc, Mode mode) {
return TransferInternal(pbox, size, rfunc, nullptr, sfunc, Mode::MultiThreaded);
}
Result TransferUnzip(ui::ProgressBox* pbox, void* zfile, fs::Fs* fs, const fs::FsPath& path, s64 size, u32 crc32) {
Result rc;
if (R_FAILED(rc = fs->CreateDirectoryRecursivelyWithPath(path)) && rc != FsError_PathAlreadyExists) {
log_write("failed to create folder: %s 0x%04X\n", path.s, rc);
R_THROW(rc);
}
if (R_FAILED(rc = fs->CreateFile(path, size, 0)) && rc != FsError_PathAlreadyExists) {
log_write("failed to create file: %s 0x%04X\n", path.s, rc);
R_THROW(rc);
}
fs::File f;
R_TRY(fs->OpenFile(path, FsOpenMode_Write, &f));
// only update the size if this is an existing file.
if (rc == FsError_PathAlreadyExists) {
R_TRY(f.SetSize(size));
}
// NOTES: do not use temp file with rename / delete after as it massively slows
// down small file transfers (RA 21s -> 50s).
u32 crc32_out{};
R_TRY(thread::TransferInternal(pbox, size,
[&](void* data, s64 off, s64 size, u64* bytes_read) -> Result {
const auto result = unzReadCurrentFile(zfile, data, size);
if (result <= 0) {
// log_write("failed to read zip file: %s\n", inzip.c_str());
R_THROW(0x1);
}
if (crc32) {
crc32_out = crc32CalculateWithSeed(crc32_out, data, result);
}
*bytes_read = result;
R_SUCCEED();
},
[&](const void* data, s64 off, s64 size) -> Result {
return f.Write(off, data, size, FsWriteOption_None);
},
nullptr, Mode::SingleThreadedIfSmaller, SMALL_BUFFER_SIZE
));
// validate crc32 (if set in the info).
R_UNLESS(!crc32 || crc32 == crc32_out, 0x1);
R_SUCCEED();
}
Result TransferZip(ui::ProgressBox* pbox, void* zfile, fs::Fs* fs, const fs::FsPath& path) {
fs::File f;
R_TRY(fs->OpenFile(path, FsOpenMode_Read, &f));
s64 file_size;
R_TRY(f.GetSize(&file_size));
return thread::TransferInternal(pbox, file_size,
[&](void* data, s64 off, s64 size, u64* bytes_read) -> Result {
return f.Read(off, data, size, FsReadOption_None, bytes_read);
},
[&](const void* data, s64 off, s64 size) -> Result {
if (ZIP_OK != zipWriteInFileInZip(zfile, data, size)) {
log_write("failed to write zip file: %s\n", path.s);
R_THROW(0x1);
}
R_SUCCEED();
},
nullptr, Mode::SingleThreadedIfSmaller, SMALL_BUFFER_SIZE
);
}
Result TransferUnzipAll(ui::ProgressBox* pbox, void* zfile, fs::Fs* fs, const fs::FsPath& base_path, UnzipAllFilter filter) {
unz_global_info64 ginfo;
if (UNZ_OK != unzGetGlobalInfo64(zfile, &ginfo)) {
R_THROW(0x1);
}
if (UNZ_OK != unzGoToFirstFile(zfile)) {
R_THROW(0x1);
}
for (s64 i = 0; i < ginfo.number_entry; i++) {
R_TRY(pbox->ShouldExitResult());
if (i > 0) {
if (UNZ_OK != unzGoToNextFile(zfile)) {
log_write("failed to unzGoToNextFile\n");
R_THROW(0x1);
}
}
if (UNZ_OK != unzOpenCurrentFile(zfile)) {
log_write("failed to open current file\n");
R_THROW(0x1);
}
ON_SCOPE_EXIT(unzCloseCurrentFile(zfile));
unz_file_info64 info;
fs::FsPath name;
if (UNZ_OK != unzGetCurrentFileInfo64(zfile, &info, name, sizeof(name), 0, 0, 0, 0)) {
log_write("failed to get current info\n");
R_THROW(0x1);
}
// check if we should skip this file.
// don't make const as to allow the function to modify the path
// this function is used for the updater to change sphaira.nro to exe path.
auto path = fs::AppendPath(base_path, name);
if (filter && !filter(name, path)) {
continue;
}
pbox->NewTransfer(name);
if (path[std::strlen(path) -1] == '/') {
Result rc;
if (R_FAILED(rc = fs->CreateDirectoryRecursively(path)) && rc != FsError_PathAlreadyExists) {
log_write("failed to create folder: %s 0x%04X\n", path.s, rc);
R_THROW(rc);
}
} else {
R_TRY(TransferUnzip(pbox, zfile, fs, path, info.uncompressed_size, info.crc));
}
}
R_SUCCEED();
}
Result TransferUnzipAll(ui::ProgressBox* pbox, const fs::FsPath& zip_out, fs::Fs* fs, const fs::FsPath& base_path, UnzipAllFilter filter) {
auto zfile = unzOpen64(zip_out);
R_UNLESS(zfile, 0x1);
ON_SCOPE_EXIT(unzClose(zfile));
return TransferUnzipAll(pbox, zfile, fs, base_path, filter);
}
} // namespace::thread

229
sphaira/source/ui/menus/appstore.cpp
View File

@@ -14,6 +14,7 @@
#include "swkbd.hpp"
#include "i18n.hpp"
#include "hasher.hpp"
#include "threaded_file_transfer.hpp"
#include "nro.hpp"
#include <minIni.h>
@@ -75,6 +76,62 @@ constexpr const char* ORDER_STR[] = {
"Asc",
};
struct MzMem {
const void* buf;
size_t size;
size_t offset;
};
ZPOS64_T minizip_tell_file_func(voidpf opaque, voidpf stream) {
auto mem = static_cast<const MzMem*>(opaque);
return mem->offset;
}
long minizip_seek_file_func(voidpf opaque, voidpf stream, ZPOS64_T offset, int origin) {
auto mem = static_cast<MzMem*>(opaque);
size_t new_offset = 0;
switch (origin) {
case ZLIB_FILEFUNC_SEEK_SET: new_offset = offset; break;
case ZLIB_FILEFUNC_SEEK_CUR: new_offset = mem->offset + offset; break;
case ZLIB_FILEFUNC_SEEK_END: new_offset = (mem->size - 1) + offset; break;
default: return -1;
}
if (new_offset > mem->size) {
return -1;
}
mem->offset = new_offset;
return 0;
}
voidpf minizip_open_file_func(voidpf opaque, const void* filename, int mode) {
return opaque;
}
uLong minizip_read_file_func(voidpf opaque, voidpf stream, void* buf, uLong size) {
auto mem = static_cast<MzMem*>(opaque);
size = std::min(size, mem->size - mem->offset);
std::memcpy(buf, (const u8*)mem->buf + mem->offset, size);
mem->offset += size;
return size;
}
int minizip_close_file_func(voidpf opaque, voidpf stream) {
return 0;
}
constexpr zlib_filefunc64_def zlib_filefunc = {
.zopen64_file = minizip_open_file_func,
.zread_file = minizip_read_file_func,
.ztell64_file = minizip_tell_file_func,
.zseek64_file = minizip_seek_file_func,
.zclose_file = minizip_close_file_func,
};
auto BuildIconUrl(const Entry& e) -> std::string {
char out[0x100];
std::snprintf(out, sizeof(out), "%s/packages/%s/icon.png", URL_BASE, e.name.c_str());
@@ -363,19 +420,30 @@ auto InstallApp(ProgressBox* pbox, const Entry& entry) -> Result {
fs::FsNativeSd fs;
R_TRY(fs.GetFsOpenResult());
// check if we can download the entire zip to mem for faster download / extract times.
// current limit is 300MiB, or disabled for applet mode.
const auto file_download = App::IsApplet() || entry.filesize >= 1024 * 1024 * 300;
curl::ApiResult api_result{};
// 1. download the zip
if (!pbox->ShouldExit()) {
pbox->NewTransfer("Downloading "_i18n + entry.title);
log_write("starting download\n");
const auto url = BuildZipUrl(entry);
const auto result = curl::Api().ToFile(
curl::Api api{
curl::Url{url},
curl::Path{zip_out},
curl::OnProgress{pbox->OnDownloadProgressCallback()}
);
};
R_UNLESS(result.success, 0x1);
if (file_download) {
api.SetOption(curl::Path{zip_out});
api_result = curl::ToFile(api);
} else {
api_result = curl::ToMemory(api);
}
R_UNLESS(api_result.success, 0x1);
}
ON_SCOPE_EXIT(fs.DeleteFile(zip_out));
@@ -386,7 +454,11 @@ auto InstallApp(ProgressBox* pbox, const Entry& entry) -> Result {
log_write("starting md5 check\n");
std::string hash_out;
R_TRY(hash::Hash(pbox, hash::Type::Md5, &fs, zip_out, hash_out));
if (file_download) {
R_TRY(hash::Hash(pbox, hash::Type::Md5, &fs, zip_out, hash_out));
} else {
R_TRY(hash::Hash(pbox, hash::Type::Md5, api_result.data, hash_out));
}
if (strncasecmp(hash_out.data(), entry.md5.data(), entry.md5.length())) {
log_write("bad md5: %.*s vs %.*s\n", 32, hash_out.data(), 32, entry.md5.c_str());
@@ -394,9 +466,20 @@ auto InstallApp(ProgressBox* pbox, const Entry& entry) -> Result {
}
}
struct MzMem mem{};
mem.buf = api_result.data.data();
mem.size = api_result.data.size();
auto file_func = zlib_filefunc;
file_func.opaque = &mem;
zlib_filefunc64_def* file_func_ptr{};
if (!file_download) {
file_func_ptr = &file_func;
}
// 3. extract the zip
if (!pbox->ShouldExit()) {
auto zfile = unzOpen64(zip_out);
auto zfile = unzOpen2_64(zip_out, file_func_ptr);
R_UNLESS(zfile, 0x1);
ON_SCOPE_EXIT(unzClose(zfile));
@@ -434,43 +517,6 @@ auto InstallApp(ProgressBox* pbox, const Entry& entry) -> Result {
}
}
const auto unzip_to_file = [&](const unz_file_info64& info, const fs::FsPath& inzip, fs::FsPath output) -> Result {
if (output[0] != '/') {
output = fs::AppendPath("/", output);
}
// create directories
fs.CreateDirectoryRecursivelyWithPath(output);
Result rc;
if (R_FAILED(rc = fs.CreateFile(output, info.uncompressed_size, 0)) && rc != FsError_PathAlreadyExists) {
log_write("failed to create file: %s 0x%04X\n", output.s, rc);
R_THROW(rc);
}
fs::File f;
R_TRY(fs.OpenFile(output, FsOpenMode_Write, &f));
R_TRY(f.SetSize(info.uncompressed_size));
u64 offset{};
while (offset < info.uncompressed_size) {
R_TRY(pbox->ShouldExitResult());
const auto bytes_read = unzReadCurrentFile(zfile, buf.data(), buf.size());
if (bytes_read <= 0) {
log_write("failed to read zip file: %s\n", inzip.s);
R_THROW(0x1);
}
R_TRY(f.Write(offset, buf.data(), bytes_read, FsWriteOption_None));
pbox->UpdateTransfer(offset, info.uncompressed_size);
offset += bytes_read;
}
R_SUCCEED();
};
const auto unzip_to = [&](const fs::FsPath& inzip, const fs::FsPath& output) -> Result {
pbox->NewTransfer(inzip);
@@ -491,79 +537,46 @@ auto InstallApp(ProgressBox* pbox, const Entry& entry) -> Result {
R_THROW(0x1);
}
return unzip_to_file(info, inzip, output);
};
const auto unzip_all = [&](std::span<const ManifestEntry> entries) -> Result {
unz_global_info64 ginfo;
if (UNZ_OK != unzGetGlobalInfo64(zfile, &ginfo)) {
R_THROW(0x1);
auto path = output;
if (path[0] != '/') {
path = fs::AppendPath("/", path);
}
if (UNZ_OK != unzGoToFirstFile(zfile)) {
R_THROW(0x1);
}
for (s64 i = 0; i < ginfo.number_entry; i++) {
R_TRY(pbox->ShouldExitResult());
if (i > 0) {
if (UNZ_OK != unzGoToNextFile(zfile)) {
log_write("failed to unzGoToNextFile\n");
R_THROW(0x1);
}
}
if (UNZ_OK != unzOpenCurrentFile(zfile)) {
log_write("failed to open current file\n");
R_THROW(0x1);
}
ON_SCOPE_EXIT(unzCloseCurrentFile(zfile));
unz_file_info64 info;
char name[512];
if (UNZ_OK != unzGetCurrentFileInfo64(zfile, &info, name, sizeof(name), 0, 0, 0, 0)) {
log_write("failed to get current info\n");
R_THROW(0x1);
}
const auto it = std::ranges::find_if(entries, [&name](auto& e){
return !strcasecmp(name, e.path);
});
if (it == entries.end()) [[unlikely]] {
continue;
}
pbox->NewTransfer(it->path);
switch (it->command) {
case 'E': // both are the same?
case 'U':
break;
case 'G': { // checks if file exists, if not, extract
if (fs.FileExists(fs::AppendPath("/", it->path))) {
continue;
}
} break;
default:
log_write("bad command: %c\n", it->command);
continue;
}
R_TRY(unzip_to_file(info, it->path, it->path));
}
R_SUCCEED();
return thread::TransferUnzip(pbox, zfile, &fs, path, info.uncompressed_size, info.crc);
};
// unzip manifest, info and all entries.
TimeStamp ts;
#if 1
R_TRY(unzip_to("info.json", BuildInfoCachePath(entry)));
R_TRY(unzip_to("manifest.install", BuildManifestCachePath(entry)));
R_TRY(unzip_all(new_manifest));
#endif
R_TRY(thread::TransferUnzipAll(pbox, zfile, &fs, "/", [&](const fs::FsPath& name, fs::FsPath& path) -> bool {
const auto it = std::ranges::find_if(new_manifest, [&name](auto& e){
return !strcasecmp(name, e.path);
});
if (it == new_manifest.end()) [[unlikely]] {
return false;
}
pbox->NewTransfer(it->path);
switch (it->command) {
case 'E': // both are the same?
case 'U':
return true;
case 'G': // checks if file exists, if not, extract
return !fs.FileExists(fs::AppendPath("/", it->path));
default:
log_write("bad command: %c\n", it->command);
return false;
}
}));
log_write("\n\t[APPSTORE] finished extract new, time taken: %.2fs %zums\n\n", ts.GetSecondsD(), ts.GetMs());
// finally finally, remove files no longer in the manifest

105
sphaira/source/ui/menus/filebrowser.cpp
View File

@@ -708,75 +708,10 @@ void FsView::UnzipFiles(fs::FsPath dir_path) {
}
App::Push(std::make_shared<ui::ProgressBox>(0, "Extracting "_i18n, "", [this, dir_path, targets](auto pbox) -> Result {
constexpr auto chunk_size = 1024 * 512; // 512KiB
for (auto& e : targets) {
pbox->SetTitle(e.GetName());
const auto zip_out = GetNewPath(e);
auto zfile = unzOpen64(zip_out);
R_UNLESS(zfile, 0x1);
ON_SCOPE_EXIT(unzClose(zfile));
unz_global_info64 pglobal_info;
if (UNZ_OK != unzGetGlobalInfo64(zfile, &pglobal_info)) {
R_THROW(0x1);
}
for (s64 i = 0; i < pglobal_info.number_entry; i++) {
if (i > 0) {
if (UNZ_OK != unzGoToNextFile(zfile)) {
log_write("failed to unzGoToNextFile\n");
R_THROW(0x1);
}
}
if (UNZ_OK != unzOpenCurrentFile(zfile)) {
log_write("failed to open current file\n");
R_THROW(0x1);
}
ON_SCOPE_EXIT(unzCloseCurrentFile(zfile));
unz_file_info64 info;
char name[512];
if (UNZ_OK != unzGetCurrentFileInfo64(zfile, &info, name, sizeof(name), 0, 0, 0, 0)) {
log_write("failed to get current info\n");
R_THROW(0x1);
}
const auto file_path = fs::AppendPath(dir_path, name);
pbox->NewTransfer(name);
// create directories
m_fs->CreateDirectoryRecursivelyWithPath(file_path);
Result rc;
if (R_FAILED(rc = m_fs->CreateFile(file_path, info.uncompressed_size, 0)) && rc != FsError_PathAlreadyExists) {
log_write("failed to create file: %s 0x%04X\n", file_path.s, rc);
R_THROW(rc);
}
fs::File f;
R_TRY(m_fs->OpenFile(file_path, FsOpenMode_Write, &f));
R_TRY(f.SetSize(info.uncompressed_size));
std::vector<char> buf(chunk_size);
s64 offset{};
while (offset < info.uncompressed_size) {
R_TRY(pbox->ShouldExitResult());
const auto bytes_read = unzReadCurrentFile(zfile, buf.data(), buf.size());
if (bytes_read <= 0) {
log_write("failed to read zip file: %s\n", name);
R_THROW(0x1);
}
R_TRY(f.Write(offset, buf.data(), bytes_read, FsWriteOption_None));
pbox->UpdateTransfer(offset, info.uncompressed_size);
offset += bytes_read;
}
}
R_TRY(thread::TransferUnzipAll(pbox, zip_out, m_fs.get(), dir_path));
}
R_SUCCEED();
@@ -829,8 +764,6 @@ void FsView::ZipFiles(fs::FsPath zip_out) {
}
App::Push(std::make_shared<ui::ProgressBox>(0, "Compressing "_i18n, "", [this, zip_out, targets](auto pbox) -> Result {
constexpr auto chunk_size = 1024 * 512; // 512KiB
const auto t = std::time(NULL);
const auto tm = std::localtime(&t);
@@ -851,6 +784,11 @@ void FsView::ZipFiles(fs::FsPath zip_out) {
// the file name needs to be relative to the current directory.
const char* file_name_in_zip = file_path.s + std::strlen(m_path);
// strip root path (/ or ums0:)
if (!std::strncmp(file_name_in_zip, m_fs->Root(), std::strlen(m_fs->Root()))) {
file_name_in_zip += std::strlen(m_fs->Root());
}
// root paths are banned in zips, they will warn when extracting otherwise.
if (file_name_in_zip[0] == '/') {
file_name_in_zip++;
@@ -858,38 +796,13 @@ void FsView::ZipFiles(fs::FsPath zip_out) {
pbox->NewTransfer(file_name_in_zip);
const auto ext = std::strrchr(file_name_in_zip, '.');
const auto raw = ext && IsExtension(ext + 1, COMPRESSED_EXTENSIONS);
if (ZIP_OK != zipOpenNewFileInZip2(zfile, file_name_in_zip, &zip_info, NULL, 0, NULL, 0, NULL, Z_DEFLATED, Z_DEFAULT_COMPRESSION, raw)) {
if (ZIP_OK != zipOpenNewFileInZip(zfile, file_name_in_zip, &zip_info, NULL, 0, NULL, 0, NULL, Z_DEFLATED, Z_DEFAULT_COMPRESSION)) {
log_write("failed to add zip for %s\n", file_path.s);
R_THROW(0x1);
}
ON_SCOPE_EXIT(zipCloseFileInZip(zfile));
fs::File f;
R_TRY(m_fs->OpenFile(file_path, FsOpenMode_Read, &f));
s64 file_size;
R_TRY(f.GetSize(&file_size));
std::vector<char> buf(chunk_size);
s64 offset{};
while (offset < file_size) {
R_TRY(pbox->ShouldExitResult());
u64 bytes_read;
R_TRY(f.Read(offset, buf.data(), buf.size(), FsReadOption_None, &bytes_read));
if (ZIP_OK != zipWriteInFileInZip(zfile, buf.data(), bytes_read)) {
log_write("failed to write zip file: %s\n", file_path.s);
R_THROW(0x1);
}
pbox->UpdateTransfer(offset, file_size);
offset += bytes_read;
}
R_SUCCEED();
return thread::TransferZip(pbox, zfile, m_fs.get(), file_path);
};
for (auto& e : targets) {

75
sphaira/source/ui/menus/ghdl.cpp
View File

@@ -14,9 +14,9 @@
#include "download.hpp"
#include "i18n.hpp"
#include "yyjson_helper.hpp"
#include "threaded_file_transfer.hpp"
#include <minIni.h>
#include <minizip/unzip.h>
#include <dirent.h>
#include <cstring>
#include <string>
@@ -83,7 +83,6 @@ void from_json(const fs::FsPath& path, GhApiEntry& e) {
auto DownloadApp(ProgressBox* pbox, const GhApiAsset& gh_asset, const AssetEntry* entry) -> Result {
static const fs::FsPath temp_file{"/switch/sphaira/cache/github/ghdl.temp"};
constexpr auto chunk_size = 1024 * 512; // 512KiB
fs::FsNativeSd fs;
R_TRY(fs.GetFsOpenResult());
@@ -113,77 +112,7 @@ auto DownloadApp(ProgressBox* pbox, const GhApiAsset& gh_asset, const AssetEntry
// 3. extract the zip / file
if (gh_asset.content_type.find("zip") != gh_asset.content_type.npos) {
log_write("found zip\n");
auto zfile = unzOpen64(temp_file);
R_UNLESS(zfile, 0x1);
ON_SCOPE_EXIT(unzClose(zfile));
unz_global_info64 pglobal_info;
if (UNZ_OK != unzGetGlobalInfo64(zfile, &pglobal_info)) {
R_THROW(0x1);
}
for (int i = 0; i < pglobal_info.number_entry; i++) {
if (i > 0) {
if (UNZ_OK != unzGoToNextFile(zfile)) {
log_write("failed to unzGoToNextFile\n");
R_THROW(0x1);
}
}
if (UNZ_OK != unzOpenCurrentFile(zfile)) {
log_write("failed to open current file\n");
R_THROW(0x1);
}
ON_SCOPE_EXIT(unzCloseCurrentFile(zfile));
unz_file_info64 info;
fs::FsPath file_path;
if (UNZ_OK != unzGetCurrentFileInfo64(zfile, &info, file_path, sizeof(file_path), 0, 0, 0, 0)) {
log_write("failed to get current info\n");
R_THROW(0x1);
}
file_path = fs::AppendPath(root_path, file_path);
Result rc;
if (file_path[strlen(file_path) -1] == '/') {
if (R_FAILED(rc = fs.CreateDirectoryRecursively(file_path)) && rc != FsError_PathAlreadyExists) {
log_write("failed to create folder: %s 0x%04X\n", file_path.s, rc);
R_THROW(rc);
}
} else {
if (R_FAILED(rc = fs.CreateDirectoryRecursivelyWithPath(file_path)) && rc != FsError_PathAlreadyExists) {
log_write("failed to create folder: %s 0x%04X\n", file_path.s, rc);
R_THROW(rc);
}
if (R_FAILED(rc = fs.CreateFile(file_path, info.uncompressed_size, 0)) && rc != FsError_PathAlreadyExists) {
log_write("failed to create file: %s 0x%04X\n", file_path.s, rc);
R_THROW(rc);
}
fs::File f;
R_TRY(fs.OpenFile(file_path, FsOpenMode_Write, &f));
R_TRY(f.SetSize(info.uncompressed_size));
std::vector<char> buf(chunk_size);
s64 offset{};
while (offset < info.uncompressed_size) {
R_TRY(pbox->ShouldExitResult());
const auto bytes_read = unzReadCurrentFile(zfile, buf.data(), buf.size());
if (bytes_read <= 0) {
log_write("failed to read zip file: %s\n", file_path.s);
R_THROW(0x1);
}
R_TRY(f.Write(offset, buf.data(), bytes_read, FsWriteOption_None));
pbox->UpdateTransfer(offset, info.uncompressed_size);
offset += bytes_read;
}
}
}
R_TRY(thread::TransferUnzipAll(pbox, temp_file, &fs, root_path));
} else {
fs.CreateDirectoryRecursivelyWithPath(root_path);
fs.DeleteFile(root_path);

110
sphaira/source/ui/menus/main_menu.cpp
View File

@@ -22,9 +22,9 @@
#include "download.hpp"
#include "defines.hpp"
#include "i18n.hpp"
#include "threaded_file_transfer.hpp"
#include <cstring>
#include <minizip/unzip.h>
#include <yyjson.h>
namespace sphaira::ui::menu::main {
@@ -59,7 +59,6 @@ const MiscMenuEntry MISC_MENU_ENTRIES[] = {
auto InstallUpdate(ProgressBox* pbox, const std::string url, const std::string version) -> Result {
static fs::FsPath zip_out{"/switch/sphaira/cache/update.zip"};
constexpr auto chunk_size = 1024 * 512; // 512KiB
fs::FsNativeSd fs;
R_TRY(fs.GetFsOpenResult());
@@ -82,95 +81,34 @@ auto InstallUpdate(ProgressBox* pbox, const std::string url, const std::string v
// 2. extract the zip
if (!pbox->ShouldExit()) {
auto zfile = unzOpen64(zip_out);
R_UNLESS(zfile, 0x1);
ON_SCOPE_EXIT(unzClose(zfile));
const auto exe_path = App::GetExePath();
bool found_exe{};
unz_global_info64 pglobal_info;
if (UNZ_OK != unzGetGlobalInfo64(zfile, &pglobal_info)) {
R_THROW(0x1);
}
for (s64 i = 0; i < pglobal_info.number_entry; i++) {
if (i > 0) {
if (UNZ_OK != unzGoToNextFile(zfile)) {
log_write("failed to unzGoToNextFile\n");
R_THROW(0x1);
}
R_TRY(thread::TransferUnzipAll(pbox, zip_out, &fs, "/", [&](const fs::FsPath& name, fs::FsPath& path) -> bool {
if (std::strstr(path, "sphaira.nro")) {
path = exe_path;
found_exe = true;
}
return true;
}));
if (UNZ_OK != unzOpenCurrentFile(zfile)) {
log_write("failed to open current file\n");
R_THROW(0x1);
}
ON_SCOPE_EXIT(unzCloseCurrentFile(zfile));
unz_file_info64 info;
fs::FsPath file_path;
if (UNZ_OK != unzGetCurrentFileInfo64(zfile, &info, file_path, sizeof(file_path), 0, 0, 0, 0)) {
log_write("failed to get current info\n");
R_THROW(0x1);
}
if (file_path[0] != '/') {
file_path = fs::AppendPath("/", file_path);
}
if (std::strstr(file_path, "sphaira.nro")) {
file_path = App::GetExePath();
}
Result rc;
if (file_path[std::strlen(file_path) -1] == '/') {
if (R_FAILED(rc = fs.CreateDirectoryRecursively(file_path)) && rc != FsError_PathAlreadyExists) {
log_write("failed to create folder: %s 0x%04X\n", file_path.s, rc);
R_THROW(rc);
}
} else {
Result rc;
if (R_FAILED(rc = fs.CreateFile(file_path, info.uncompressed_size, 0)) && rc != FsError_PathAlreadyExists) {
log_write("failed to create file: %s 0x%04X\n", file_path.s, rc);
R_THROW(rc);
// check if we have sphaira installed in other locations and update them.
if (found_exe) {
for (auto& path : SPHAIRA_PATHS) {
log_write("[UPD] checking path: %s\n", path.s);
// skip if we already updated this path.
if (exe_path == path) {
log_write("[UPD] skipped as already updated\n");
continue;
}
fs::File f;
R_TRY(fs.OpenFile(file_path, FsOpenMode_Write, &f));
R_TRY(f.SetSize(info.uncompressed_size));
std::vector<char> buf(chunk_size);
s64 offset{};
while (offset < info.uncompressed_size) {
const auto bytes_read = unzReadCurrentFile(zfile, buf.data(), buf.size());
if (bytes_read <= 0) {
// log_write("failed to read zip file: %s\n", inzip.c_str());
R_THROW(0x1);
}
R_TRY(f.Write(offset, buf.data(), bytes_read, FsWriteOption_None));
pbox->UpdateTransfer(offset, info.uncompressed_size);
offset += bytes_read;
}
}
// check if we have sphaira installed in other locations and update them.
if (file_path == App::GetExePath()) {
for (auto& path : SPHAIRA_PATHS) {
log_write("[UPD] checking path: %s\n", path.s);
// skip if we already updated this path.
if (file_path == path) {
log_write("[UPD] skipped as already updated\n");
continue;
}
// check that this is really sphaira.
log_write("[UPD] checking nacp\n");
NacpStruct nacp;
if (R_SUCCEEDED(nro_get_nacp(path, nacp)) && !std::strcmp(nacp.lang[0].name, "sphaira")) {
log_write("[UPD] found, updating\n");
pbox->NewTransfer(path);
R_TRY(pbox->CopyFile(&fs, file_path, path));
}
// check that this is really sphaira.
log_write("[UPD] checking nacp\n");
NacpStruct nacp;
if (R_SUCCEEDED(nro_get_nacp(path, nacp)) && !std::strcmp(nacp.lang[0].name, "sphaira")) {
log_write("[UPD] found, updating\n");
pbox->NewTransfer(path);
R_TRY(pbox->CopyFile(&fs, exe_path, path));
}
}
}

64
sphaira/source/ui/menus/themezer.cpp
View File

@@ -11,11 +11,11 @@
#include "ui/nvg_util.hpp"
#include "swkbd.hpp"
#include "i18n.hpp"
#include "threaded_file_transfer.hpp"
#include <minIni.h>
#include <stb_image.h>
#include <cstring>
#include <minizip/unzip.h>
#include <yyjson.h>
#include "yyjson_helper.hpp"
@@ -222,7 +222,6 @@ void from_json(const fs::FsPath& path, PackList& e) {
auto InstallTheme(ProgressBox* pbox, const PackListEntry& entry) -> Result {
static const fs::FsPath zip_out{"/switch/sphaira/cache/themezer/temp.zip"};
constexpr auto chunk_size = 1024 * 512; // 512KiB
fs::FsNativeSd fs;
R_TRY(fs.GetFsOpenResult());
@@ -272,66 +271,7 @@ auto InstallTheme(ProgressBox* pbox, const PackListEntry& entry) -> Result {
// 3. extract the zip
if (!pbox->ShouldExit()) {
auto zfile = unzOpen64(zip_out);
R_UNLESS(zfile, 0x1);
ON_SCOPE_EXIT(unzClose(zfile));
unz_global_info64 pglobal_info;
if (UNZ_OK != unzGetGlobalInfo64(zfile, &pglobal_info)) {
R_THROW(0x1);
}
for (int i = 0; i < pglobal_info.number_entry; i++) {
if (i > 0) {
if (UNZ_OK != unzGoToNextFile(zfile)) {
log_write("failed to unzGoToNextFile\n");
R_THROW(0x1);
}
}
if (UNZ_OK != unzOpenCurrentFile(zfile)) {
log_write("failed to open current file\n");
R_THROW(0x1);
}
ON_SCOPE_EXIT(unzCloseCurrentFile(zfile));
unz_file_info64 info;
char name[512];
if (UNZ_OK != unzGetCurrentFileInfo64(zfile, &info, name, sizeof(name), 0, 0, 0, 0)) {
log_write("failed to get current info\n");
R_THROW(0x1);
}
const auto file_path = fs::AppendPath(dir_path, name);
pbox->NewTransfer(name);
Result rc;
if (R_FAILED(rc = fs.CreateFile(file_path, info.uncompressed_size, 0)) && rc != FsError_PathAlreadyExists) {
log_write("failed to create file: %s 0x%04X\n", file_path.s, rc);
R_THROW(rc);
}
fs::File f;
R_TRY(fs.OpenFile(file_path, FsOpenMode_Write, &f));
R_TRY(f.SetSize(info.uncompressed_size));
std::vector<char> buf(chunk_size);
s64 offset{};
while (offset < info.uncompressed_size) {
R_TRY(pbox->ShouldExitResult());
const auto bytes_read = unzReadCurrentFile(zfile, buf.data(), buf.size());
if (bytes_read <= 0) {
// log_write("failed to read zip file: %s\n", inzip.c_str());
R_THROW(0x1);
}
R_TRY(f.Write(offset, buf.data(), bytes_read, FsWriteOption_None));
pbox->UpdateTransfer(offset, info.uncompressed_size);
offset += bytes_read;
}
}
R_TRY(thread::TransferUnzipAll(pbox, zip_out, &fs, dir_path));
}
log_write("finished install :)\n");