diff --git a/sphaira/CMakeLists.txt b/sphaira/CMakeLists.txt index 077b67c..b30e537 100644 --- a/sphaira/CMakeLists.txt +++ b/sphaira/CMakeLists.txt @@ -90,6 +90,7 @@ add_executable(sphaira source/minizip_helper.cpp source/fatfs.cpp + source/utils/devoptab_common.cpp source/utils/devoptab_save.cpp source/utils/devoptab_nsp.cpp source/utils/devoptab_xci.cpp diff --git a/sphaira/include/utils/devoptab.hpp b/sphaira/include/utils/devoptab.hpp index a0f466a..fc401a1 100644 --- a/sphaira/include/utils/devoptab.hpp +++ b/sphaira/include/utils/devoptab.hpp @@ -19,6 +19,4 @@ void UmountNsp(const fs::FsPath& mount); Result MountXci(fs::Fs* fs, const fs::FsPath& path, fs::FsPath& out_path); void UmountXci(const fs::FsPath& mount); -bool fix_path(const char* str, char* out); - } // namespace sphaira::devoptab diff --git a/sphaira/include/utils/devoptab_common.hpp b/sphaira/include/utils/devoptab_common.hpp new file mode 100644 index 0000000..c48a09a --- /dev/null +++ b/sphaira/include/utils/devoptab_common.hpp @@ -0,0 +1,34 @@ +#pragma once + +#include "yati/source/file.hpp" +#include + +namespace sphaira::devoptab::common { + +// buffers data in 512k chunks to maximise throughput. +// not suitable if random access >= 512k is common. +// if that is needed, see the LRU cache varient used for fatfs. +struct BufferedData final : yati::source::Base { + static constexpr inline u64 CHUNK_SIZE = 1024 * 512; + + BufferedData(std::unique_ptr&& _source, u64 _size) + : source{std::forward(_source)} + , capacity{_size} { + + } + + Result Read(void *buf, s64 off, s64 size); + Result Read(void* buf, s64 off, s64 size, u64* bytes_read) override; + +private: + std::unique_ptr source; + const u64 capacity; + + u64 m_off{}; + u64 m_size{}; + u8 m_data[CHUNK_SIZE]{}; +}; + +bool fix_path(const char* str, char* out); + +} // namespace sphaira::devoptab::common diff --git a/sphaira/source/utils/devoptab_common.cpp b/sphaira/source/utils/devoptab_common.cpp new file mode 100644 index 0000000..41fd740 --- /dev/null +++ b/sphaira/source/utils/devoptab_common.cpp @@ -0,0 +1,121 @@ +#include "utils/devoptab_common.hpp" +#include "defines.hpp" +#include "log.hpp" + +#include +#include + +namespace sphaira::devoptab::common { + +Result BufferedData::Read(void* buf, s64 off, s64 size) { + u64 bytes_read; + return Read(buf, off, size, &bytes_read); +} + +// todo: change above function to handle bytes read instead. +Result BufferedData::Read(void *_buffer, s64 file_off, s64 read_size, u64* bytes_read) { + auto dst = static_cast(_buffer); + size_t amount = 0; + *bytes_read = 0; + + R_UNLESS(file_off < capacity, FsError_UnsupportedOperateRangeForFileStorage); + read_size = std::min(read_size, capacity - file_off); + + if (m_size) { + // check if we can read this data into the beginning of dst. + if (file_off < m_off + m_size && file_off >= m_off) { + const auto off = file_off - m_off; + const auto size = std::min(read_size, m_size - off); + if (size) { + std::memcpy(dst, m_data + off, size); + + read_size -= size; + file_off += size; + amount += size; + dst += size; + } + } + } + + if (read_size) { + const auto alloc_size = sizeof(m_data); + m_off = 0; + m_size = 0; + u64 bytes_read; + + // if the dst is big enough, read data in place. + if (read_size > alloc_size) { + R_TRY(source->Read(dst, file_off, read_size, &bytes_read)); + + read_size -= bytes_read; + file_off += bytes_read; + amount += bytes_read; + dst += bytes_read; + + // save the last chunk of data to the m_buffered io. + const auto max_advance = std::min(amount, alloc_size); + m_off = file_off - max_advance; + m_size = max_advance; + std::memcpy(m_data, dst - max_advance, max_advance); + } else { + R_TRY(source->Read(m_data, file_off, alloc_size, &bytes_read)); + const auto bytes_read = alloc_size; + const auto max_advance = std::min(read_size, bytes_read); + std::memcpy(dst, m_data, max_advance); + + m_off = file_off; + m_size = bytes_read; + + read_size -= max_advance; + file_off += max_advance; + amount += max_advance; + dst += max_advance; + } + } + + *bytes_read = amount; + R_SUCCEED(); +} + +bool fix_path(const char* str, char* out) { + // log_write("[SAVE] got path: %s\n", str); + + str = std::strrchr(str, ':'); + if (!str) { + return false; + } + + // skip over ':' + str++; + size_t len = 0; + + // todo: hanle utf8 paths. + for (size_t i = 0; str[i]; i++) { + // skip multiple slashes. + if (i && str[i] == '/' && str[i - 1] == '/') { + continue; + } + + // add leading slash. + if (!i && str[i] != '/') { + out[len++] = '/'; + } + + // save single char. + out[len++] = str[i]; + } + + // skip trailing slash. + if (len > 1 && out[len - 1] == '/') { + out[len - 1] = '\0'; + } + + // null the end. + out[len] = '\0'; + + // log_write("[SAVE] end path: %s\n", out); + + return true; +} + +} // sphaira::devoptab::common diff --git a/sphaira/source/utils/devoptab_nsp.cpp b/sphaira/source/utils/devoptab_nsp.cpp index f245efa..f5f8427 100644 --- a/sphaira/source/utils/devoptab_nsp.cpp +++ b/sphaira/source/utils/devoptab_nsp.cpp @@ -1,5 +1,6 @@ #include "utils/devoptab.hpp" +#include "utils/devoptab_common.hpp" #include "defines.hpp" #include "log.hpp" @@ -17,7 +18,7 @@ namespace sphaira::devoptab { namespace { struct Device { - std::unique_ptr source; + std::unique_ptr source; yati::container::Collections collections; }; @@ -43,7 +44,7 @@ int devoptab_open(struct _reent *r, void *fileStruct, const char *_path, int fla std::memset(file, 0, sizeof(*file)); char path[FS_MAX_PATH]; - if (!fix_path(_path, path)) { + if (!common::fix_path(_path, path)) { return set_errno(r, ENOENT); } @@ -111,7 +112,7 @@ DIR_ITER* devoptab_diropen(struct _reent *r, DIR_ITER *dirState, const char *_pa std::memset(dir, 0, sizeof(*dir)); char path[FS_MAX_PATH]; - if (!fix_path(_path, path)) { + if (!common::fix_path(_path, path)) { set_errno(r, ENOENT); return NULL; } @@ -166,7 +167,7 @@ int devoptab_lstat(struct _reent *r, const char *_path, struct stat *st) { log_write("[\t\tDEV] lstat\n"); char path[FS_MAX_PATH]; - if (!fix_path(_path, path)) { + if (!common::fix_path(_path, path)) { return set_errno(r, ENOENT); } @@ -236,7 +237,12 @@ Result MountNsp(fs::Fs* fs, const fs::FsPath& path, fs::FsPath& out_path) { } auto source = std::make_unique(fs, path); - yati::container::Nsp nsp{source.get()}; + + s64 size; + R_TRY(source->GetSize(&size)); + auto buffered = std::make_unique(std::move(source), size); + + yati::container::Nsp nsp{buffered.get()}; yati::container::Collections collections; R_TRY(nsp.GetCollections(collections)); @@ -245,7 +251,7 @@ Result MountNsp(fs::Fs* fs, const fs::FsPath& path, fs::FsPath& out_path) { entry.devoptab = DEVOPTAB; entry.devoptab.name = entry.name; entry.devoptab.deviceData = &entry.device; - entry.device.source = std::move(source); + entry.device.source = std::move(buffered); entry.device.collections = collections; std::snprintf(entry.name, sizeof(entry.name), "nsp_%u", g_mount_idx); std::snprintf(entry.mount, sizeof(entry.mount), "nsp_%u:/", g_mount_idx); diff --git a/sphaira/source/utils/devoptab_save.cpp b/sphaira/source/utils/devoptab_save.cpp index f4ec46a..f6d7b77 100644 --- a/sphaira/source/utils/devoptab_save.cpp +++ b/sphaira/source/utils/devoptab_save.cpp @@ -1,5 +1,6 @@ #include "utils/devoptab.hpp" +#include "utils/devoptab_common.hpp" #include "defines.hpp" #include "log.hpp" @@ -50,7 +51,7 @@ int devoptab_open(struct _reent *r, void *fileStruct, const char *_path, int fla std::memset(file, 0, sizeof(*file)); char path[FS_MAX_PATH]; - if (!fix_path(_path, path)) { + if (!common::fix_path(_path, path)) { return set_errno(r, ENOENT); } @@ -116,7 +117,7 @@ DIR_ITER* devoptab_diropen(struct _reent *r, DIR_ITER *dirState, const char *_pa std::memset(dir, 0, sizeof(*dir)); char path[FS_MAX_PATH]; - if (!fix_path(_path, path)) { + if (!common::fix_path(_path, path)) { set_errno(r, ENOENT); return NULL; } @@ -203,7 +204,7 @@ int devoptab_lstat(struct _reent *r, const char *_path, struct stat *st) { auto device = (Device*)r->deviceData; char path[FS_MAX_PATH]; - if (!fix_path(_path, path)) { + if (!common::fix_path(_path, path)) { return set_errno(r, ENOENT); } @@ -258,47 +259,6 @@ void MakeMountPath(u64 id, fs::FsPath& out_path) { } // namespace -bool fix_path(const char* str, char* out) { - // log_write("[SAVE] got path: %s\n", str); - - str = std::strrchr(str, ':'); - if (!str) { - return false; - } - - // skip over ':' - str++; - size_t len = 0; - - // todo: hanle utf8 paths. - for (size_t i = 0; str[i]; i++) { - // skip multiple slashes. - if (i && str[i] == '/' && str[i - 1] == '/') { - continue; - } - - // add leading slash. - if (!i && str[i] != '/') { - out[len++] = '/'; - } - - // save single char. - out[len++] = str[i]; - } - - // skip trailing slash. - if (len > 1 && out[len - 1] == '/') { - out[len - 1] = '\0'; - } - - // null the end. - out[len] = '\0'; - - // log_write("[SAVE] end path: %s\n", out); - - return true; -} - Result MountFromSavePath(u64 id, fs::FsPath& out_path) { SCOPED_MUTEX(&g_mutex); diff --git a/sphaira/source/utils/devoptab_xci.cpp b/sphaira/source/utils/devoptab_xci.cpp index 408be89..26e668d 100644 --- a/sphaira/source/utils/devoptab_xci.cpp +++ b/sphaira/source/utils/devoptab_xci.cpp @@ -1,5 +1,6 @@ #include "utils/devoptab.hpp" +#include "utils/devoptab_common.hpp" #include "defines.hpp" #include "log.hpp" @@ -16,7 +17,7 @@ namespace sphaira::devoptab { namespace { struct Device { - std::unique_ptr source; + std::unique_ptr source; yati::container::Xci::Partitions partitions; }; @@ -43,7 +44,7 @@ int devoptab_open(struct _reent *r, void *fileStruct, const char *_path, int fla std::memset(file, 0, sizeof(*file)); char path[FS_MAX_PATH]; - if (!fix_path(_path, path)) { + if (!common::fix_path(_path, path)) { return set_errno(r, ENOENT); } @@ -113,7 +114,7 @@ DIR_ITER* devoptab_diropen(struct _reent *r, DIR_ITER *dirState, const char *_pa std::memset(dir, 0, sizeof(*dir)); char path[FS_MAX_PATH]; - if (!fix_path(_path, path)) { + if (!common::fix_path(_path, path)) { set_errno(r, ENOENT); return NULL; } @@ -183,7 +184,7 @@ int devoptab_lstat(struct _reent *r, const char *_path, struct stat *st) { auto device = (Device*)r->deviceData; char path[FS_MAX_PATH]; - if (!fix_path(_path, path)) { + if (!common::fix_path(_path, path)) { return set_errno(r, ENOENT); } @@ -256,7 +257,12 @@ Result MountXci(fs::Fs* fs, const fs::FsPath& path, fs::FsPath& out_path) { } auto source = std::make_unique(fs, path); - yati::container::Xci xci{source.get()}; + + s64 size; + R_TRY(source->GetSize(&size)); + auto buffered = std::make_unique(std::move(source), size); + + yati::container::Xci xci{buffered.get()}; yati::container::Xci::Partitions partitions; R_TRY(xci.GetPartitions(partitions)); @@ -265,7 +271,7 @@ Result MountXci(fs::Fs* fs, const fs::FsPath& path, fs::FsPath& out_path) { entry.devoptab = DEVOPTAB; entry.devoptab.name = entry.name; entry.devoptab.deviceData = &entry.device; - entry.device.source = std::move(source); + entry.device.source = std::move(buffered); entry.device.partitions = partitions; std::snprintf(entry.name, sizeof(entry.name), "xci_%u", g_mount_idx); std::snprintf(entry.mount, sizeof(entry.mount), "xci_%u:/", g_mount_idx); diff --git a/sphaira/source/utils/devoptab_zip.cpp b/sphaira/source/utils/devoptab_zip.cpp index 43a8cad..fe2b662 100644 --- a/sphaira/source/utils/devoptab_zip.cpp +++ b/sphaira/source/utils/devoptab_zip.cpp @@ -1,5 +1,5 @@ - #include "utils/devoptab.hpp" +#include "utils/devoptab_common.hpp" #include "defines.hpp" #include "log.hpp" @@ -15,86 +15,6 @@ namespace sphaira::devoptab { namespace { -struct BufferedFileData { - BufferedFileData(std::unique_ptr&& _source, u64 _size) - : source{std::forward(_source)} - , capacity{_size} { - - } - - Result Read(void *_buffer, u64 file_off, u64 read_size); - -private: - std::unique_ptr source; - const u64 capacity; - - u64 m_off{}; - u64 m_size{}; - u8 m_data[1024 * 64]{}; -}; - -Result BufferedFileData::Read(void *_buffer, u64 file_off, u64 read_size) { - auto dst = static_cast(_buffer); - size_t amount = 0; - - R_UNLESS(file_off < capacity, FsError_UnsupportedOperateRangeForFileStorage); - read_size = std::min(read_size, capacity - file_off); - - if (m_size) { - // check if we can read this data into the beginning of dst. - if (file_off < m_off + m_size && file_off >= m_off) { - const auto off = file_off - m_off; - const auto size = std::min(read_size, m_size - off); - if (size) { - std::memcpy(dst, m_data + off, size); - - read_size -= size; - file_off += size; - amount += size; - dst += size; - } - } - } - - if (read_size) { - const auto alloc_size = sizeof(m_data); - m_off = 0; - m_size = 0; - u64 bytes_read; - - // if the dst is big enough, read data in place. - if (read_size > alloc_size) { - R_TRY(source->Read(dst, file_off, read_size, &bytes_read)); - - read_size -= bytes_read; - file_off += bytes_read; - amount += bytes_read; - dst += bytes_read; - - // save the last chunk of data to the m_buffered io. - const auto max_advance = std::min(amount, alloc_size); - m_off = file_off - max_advance; - m_size = max_advance; - std::memcpy(m_data, dst - max_advance, max_advance); - } else { - R_TRY(source->Read(m_data, file_off, alloc_size, &bytes_read)); - const auto bytes_read = alloc_size; - const auto max_advance = std::min(read_size, bytes_read); - std::memcpy(dst, m_data, max_advance); - - m_off = file_off; - m_size = bytes_read; - - read_size -= max_advance; - file_off += max_advance; - amount += max_advance; - dst += max_advance; - } - } - - R_SUCCEED(); -} - #define LOCAL_HEADER_SIG 0x4034B50 #define FILE_HEADER_SIG 0x2014B50 #define DATA_DESCRIPTOR_SIG 0x8074B50 @@ -193,7 +113,7 @@ struct DirectoryEntry { using FileTableEntries = std::vector; struct Device { - std::unique_ptr source; + std::unique_ptr source; DirectoryEntry root; }; @@ -284,7 +204,7 @@ int devoptab_open(struct _reent *r, void *fileStruct, const char *_path, int fla std::memset(file, 0, sizeof(*file)); char path[FS_MAX_PATH]{}; - if (!fix_path(_path, path)) { + if (!common::fix_path(_path, path)) { return set_errno(r, ENOENT); } @@ -462,7 +382,7 @@ DIR_ITER* devoptab_diropen(struct _reent *r, DIR_ITER *dirState, const char *_pa std::memset(dir, 0, sizeof(*dir)); char path[FS_MAX_PATH]; - if (!fix_path(_path, path)) { + if (!common::fix_path(_path, path)) { set_errno(r, ENOENT); return NULL; } @@ -531,7 +451,7 @@ int devoptab_lstat(struct _reent *r, const char *_path, struct stat *st) { } char path[FS_MAX_PATH]; - if (!fix_path(_path, path)) { + if (!common::fix_path(_path, path)) { return set_errno(r, ENOENT); } @@ -612,7 +532,7 @@ void Parse(const FileTableEntries& entries, DirectoryEntry& out) { Parse(entries, index, out); } -Result find_central_dir_offset(BufferedFileData* source, s64 size, mmz_EndRecord* record) { +Result find_central_dir_offset(common::BufferedData* source, s64 size, mmz_EndRecord* record) { // check if the record is at the end (no extra header). auto offset = size - sizeof(*record); R_TRY(source->Read(record, offset, sizeof(*record))); @@ -640,7 +560,7 @@ Result find_central_dir_offset(BufferedFileData* source, s64 size, mmz_EndRecord R_THROW(0x1); } -Result ParseZip(BufferedFileData* source, s64 size, FileTableEntries& out) { +Result ParseZip(common::BufferedData* source, s64 size, FileTableEntries& out) { mmz_EndRecord end_rec; R_TRY(find_central_dir_offset(source, size, &end_rec)); @@ -706,14 +626,12 @@ Result MountZip(fs::Fs* fs, const fs::FsPath& path, fs::FsPath& out_path) { } } - // create new entry. auto source = std::make_unique(fs, path); s64 size; R_TRY(source->GetSize(&size)); - log_write("[ZIP] got size\n"); - auto buffered = std::make_unique(std::move(source), size); + auto buffered = std::make_unique(std::move(source), size); FileTableEntries table_entries; R_TRY(ParseZip(buffered.get(), size, table_entries));