devoptab: cache all reads to max read throughput, now as fast as normal sd card reads, including compressed zips (80MiB/s).

sphaira/CMakeLists.txt

@@ -90,6 +90,7 @@ add_executable(sphaira
     source/minizip_helper.cpp
     source/fatfs.cpp
 
+    source/utils/devoptab_common.cpp
     source/utils/devoptab_save.cpp
     source/utils/devoptab_nsp.cpp
     source/utils/devoptab_xci.cpp

sphaira/include/utils/devoptab.hpp

@@ -19,6 +19,4 @@ void UmountNsp(const fs::FsPath& mount);
 Result MountXci(fs::Fs* fs, const fs::FsPath& path, fs::FsPath& out_path);
 void UmountXci(const fs::FsPath& mount);
 
-bool fix_path(const char* str, char* out);
-
 } // namespace sphaira::devoptab

sphaira/include/utils/devoptab_common.hpp

@@ -0,0 +1,34 @@
+#pragma once
+
+#include "yati/source/file.hpp"
+#include <memory>
+
+namespace sphaira::devoptab::common {
+
+// buffers data in 512k chunks to maximise throughput.
+// not suitable if random access >= 512k is common.
+// if that is needed, see the LRU cache varient used for fatfs.
+struct BufferedData final : yati::source::Base {
+    static constexpr inline u64 CHUNK_SIZE = 1024 * 512;
+
+    BufferedData(std::unique_ptr<yati::source::Base>&& _source, u64 _size)
+    : source{std::forward<decltype(_source)>(_source)}
+    , capacity{_size} {
+
+    }
+
+    Result Read(void *buf, s64 off, s64 size);
+    Result Read(void* buf, s64 off, s64 size, u64* bytes_read) override;
+
+private:
+    std::unique_ptr<yati::source::Base> source;
+    const u64 capacity;
+
+    u64 m_off{};
+    u64 m_size{};
+    u8 m_data[CHUNK_SIZE]{};
+};
+
+bool fix_path(const char* str, char* out);
+
+} // namespace sphaira::devoptab::common

sphaira/source/utils/devoptab_common.cpp

@@ -0,0 +1,121 @@
+#include "utils/devoptab_common.hpp"
+#include "defines.hpp"
+#include "log.hpp"
+
+#include <cstring>
+#include <algorithm>
+
+namespace sphaira::devoptab::common {
+
+Result BufferedData::Read(void* buf, s64 off, s64 size) {
+    u64 bytes_read;
+    return Read(buf, off, size, &bytes_read);
+}
+
+// todo: change above function to handle bytes read instead.
+Result BufferedData::Read(void *_buffer, s64 file_off, s64 read_size, u64* bytes_read) {
+    auto dst = static_cast<u8*>(_buffer);
+    size_t amount = 0;
+    *bytes_read = 0;
+
+    R_UNLESS(file_off < capacity, FsError_UnsupportedOperateRangeForFileStorage);
+    read_size = std::min<s64>(read_size, capacity - file_off);
+
+    if (m_size) {
+        // check if we can read this data into the beginning of dst.
+        if (file_off < m_off + m_size && file_off >= m_off) {
+            const auto off = file_off - m_off;
+            const auto size = std::min<s64>(read_size, m_size - off);
+            if (size) {
+                std::memcpy(dst, m_data + off, size);
+
+                read_size -= size;
+                file_off += size;
+                amount += size;
+                dst += size;
+            }
+        }
+    }
+
+    if (read_size) {
+        const auto alloc_size = sizeof(m_data);
+        m_off = 0;
+        m_size = 0;
+        u64 bytes_read;
+
+        // if the dst is big enough, read data in place.
+        if (read_size > alloc_size) {
+            R_TRY(source->Read(dst, file_off, read_size, &bytes_read));
+
+            read_size -= bytes_read;
+            file_off += bytes_read;
+            amount += bytes_read;
+            dst += bytes_read;
+
+            // save the last chunk of data to the m_buffered io.
+            const auto max_advance = std::min<u64>(amount, alloc_size);
+            m_off = file_off - max_advance;
+            m_size = max_advance;
+            std::memcpy(m_data, dst - max_advance, max_advance);
+        } else {
+            R_TRY(source->Read(m_data, file_off, alloc_size, &bytes_read));
+			const auto bytes_read = alloc_size;
+			const auto max_advance = std::min<u64>(read_size, bytes_read);
+            std::memcpy(dst, m_data, max_advance);
+
+            m_off = file_off;
+            m_size = bytes_read;
+
+            read_size -= max_advance;
+            file_off += max_advance;
+            amount += max_advance;
+            dst += max_advance;
+        }
+    }
+
+    *bytes_read = amount;
+    R_SUCCEED();
+}
+
+bool fix_path(const char* str, char* out) {
+    // log_write("[SAVE] got path: %s\n", str);
+
+    str = std::strrchr(str, ':');
+    if (!str) {
+        return false;
+    }
+
+    // skip over ':'
+    str++;
+    size_t len = 0;
+
+    // todo: hanle utf8 paths.
+    for (size_t i = 0; str[i]; i++) {
+        // skip multiple slashes.
+        if (i && str[i] == '/' && str[i - 1] == '/') {
+            continue;
+        }
+
+        // add leading slash.
+        if (!i && str[i] != '/') {
+            out[len++] = '/';
+        }
+
+        // save single char.
+        out[len++] = str[i];
+    }
+
+    // skip trailing slash.
+    if (len > 1 && out[len - 1] == '/') {
+        out[len - 1] = '\0';
+    }
+
+    // null the end.
+    out[len] = '\0';
+
+    // log_write("[SAVE] end path: %s\n", out);
+
+    return true;
+}
+
+} // sphaira::devoptab::common

sphaira/source/utils/devoptab_nsp.cpp

@@ -1,5 +1,6 @@
 
 #include "utils/devoptab.hpp"
+#include "utils/devoptab_common.hpp"
 #include "defines.hpp"
 #include "log.hpp"
 
@@ -17,7 +18,7 @@ namespace sphaira::devoptab {
 namespace {
 
 struct Device {
-    std::unique_ptr<yati::source::Base> source;
+    std::unique_ptr<common::BufferedData> source;
     yati::container::Collections collections;
 };
 
@@ -43,7 +44,7 @@ int devoptab_open(struct _reent *r, void *fileStruct, const char *_path, int fla
     std::memset(file, 0, sizeof(*file));
 
     char path[FS_MAX_PATH];
-    if (!fix_path(_path, path)) {
+    if (!common::fix_path(_path, path)) {
         return set_errno(r, ENOENT);
     }
 
@@ -111,7 +112,7 @@ DIR_ITER* devoptab_diropen(struct _reent *r, DIR_ITER *dirState, const char *_pa
     std::memset(dir, 0, sizeof(*dir));
 
     char path[FS_MAX_PATH];
-    if (!fix_path(_path, path)) {
+    if (!common::fix_path(_path, path)) {
         set_errno(r, ENOENT);
         return NULL;
     }
@@ -166,7 +167,7 @@ int devoptab_lstat(struct _reent *r, const char *_path, struct stat *st) {
     log_write("[\t\tDEV] lstat\n");
 
     char path[FS_MAX_PATH];
-    if (!fix_path(_path, path)) {
+    if (!common::fix_path(_path, path)) {
         return set_errno(r, ENOENT);
     }
 
@@ -236,7 +237,12 @@ Result MountNsp(fs::Fs* fs, const fs::FsPath& path, fs::FsPath& out_path) {
     }
 
     auto source = std::make_unique<yati::source::File>(fs, path);
-    yati::container::Nsp nsp{source.get()};
+
+    s64 size;
+    R_TRY(source->GetSize(&size));
+    auto buffered = std::make_unique<common::BufferedData>(std::move(source), size);
+
+    yati::container::Nsp nsp{buffered.get()};
     yati::container::Collections collections;
     R_TRY(nsp.GetCollections(collections));
 
@@ -245,7 +251,7 @@ Result MountNsp(fs::Fs* fs, const fs::FsPath& path, fs::FsPath& out_path) {
     entry.devoptab = DEVOPTAB;
     entry.devoptab.name = entry.name;
     entry.devoptab.deviceData = &entry.device;
-    entry.device.source = std::move(source);
+    entry.device.source = std::move(buffered);
     entry.device.collections = collections;
     std::snprintf(entry.name, sizeof(entry.name), "nsp_%u", g_mount_idx);
     std::snprintf(entry.mount, sizeof(entry.mount), "nsp_%u:/", g_mount_idx);

sphaira/source/utils/devoptab_save.cpp

@@ -1,5 +1,6 @@
 
 #include "utils/devoptab.hpp"
+#include "utils/devoptab_common.hpp"
 #include "defines.hpp"
 #include "log.hpp"
 
@@ -50,7 +51,7 @@ int devoptab_open(struct _reent *r, void *fileStruct, const char *_path, int fla
     std::memset(file, 0, sizeof(*file));
 
     char path[FS_MAX_PATH];
-    if (!fix_path(_path, path)) {
+    if (!common::fix_path(_path, path)) {
         return set_errno(r, ENOENT);
     }
 
@@ -116,7 +117,7 @@ DIR_ITER* devoptab_diropen(struct _reent *r, DIR_ITER *dirState, const char *_pa
     std::memset(dir, 0, sizeof(*dir));
 
     char path[FS_MAX_PATH];
-    if (!fix_path(_path, path)) {
+    if (!common::fix_path(_path, path)) {
         set_errno(r, ENOENT);
         return NULL;
     }
@@ -203,7 +204,7 @@ int devoptab_lstat(struct _reent *r, const char *_path, struct stat *st) {
     auto device = (Device*)r->deviceData;
 
     char path[FS_MAX_PATH];
-    if (!fix_path(_path, path)) {
+    if (!common::fix_path(_path, path)) {
         return set_errno(r, ENOENT);
     }
 
@@ -258,47 +259,6 @@ void MakeMountPath(u64 id, fs::FsPath& out_path) {
 
 } // namespace
 
-bool fix_path(const char* str, char* out) {
-    // log_write("[SAVE] got path: %s\n", str);
-
-    str = std::strrchr(str, ':');
-    if (!str) {
-        return false;
-    }
-
-    // skip over ':'
-    str++;
-    size_t len = 0;
-
-    // todo: hanle utf8 paths.
-    for (size_t i = 0; str[i]; i++) {
-        // skip multiple slashes.
-        if (i && str[i] == '/' && str[i - 1] == '/') {
-            continue;
-        }
-
-        // add leading slash.
-        if (!i && str[i] != '/') {
-            out[len++] = '/';
-        }
-
-        // save single char.
-        out[len++] = str[i];
-    }
-
-    // skip trailing slash.
-    if (len > 1 && out[len - 1] == '/') {
-        out[len - 1] = '\0';
-    }
-
-    // null the end.
-    out[len] = '\0';
-
-    // log_write("[SAVE] end path: %s\n", out);
-
-    return true;
-}
-
 Result MountFromSavePath(u64 id, fs::FsPath& out_path) {
     SCOPED_MUTEX(&g_mutex);
 

sphaira/source/utils/devoptab_xci.cpp

@@ -1,5 +1,6 @@
 
 #include "utils/devoptab.hpp"
+#include "utils/devoptab_common.hpp"
 #include "defines.hpp"
 #include "log.hpp"
 
@@ -16,7 +17,7 @@ namespace sphaira::devoptab {
 namespace {
 
 struct Device {
-    std::unique_ptr<yati::source::Base> source;
+    std::unique_ptr<common::BufferedData> source;
     yati::container::Xci::Partitions partitions;
 };
 
@@ -43,7 +44,7 @@ int devoptab_open(struct _reent *r, void *fileStruct, const char *_path, int fla
     std::memset(file, 0, sizeof(*file));
 
     char path[FS_MAX_PATH];
-    if (!fix_path(_path, path)) {
+    if (!common::fix_path(_path, path)) {
         return set_errno(r, ENOENT);
     }
 
@@ -113,7 +114,7 @@ DIR_ITER* devoptab_diropen(struct _reent *r, DIR_ITER *dirState, const char *_pa
     std::memset(dir, 0, sizeof(*dir));
 
     char path[FS_MAX_PATH];
-    if (!fix_path(_path, path)) {
+    if (!common::fix_path(_path, path)) {
         set_errno(r, ENOENT);
         return NULL;
     }
@@ -183,7 +184,7 @@ int devoptab_lstat(struct _reent *r, const char *_path, struct stat *st) {
     auto device = (Device*)r->deviceData;
 
     char path[FS_MAX_PATH];
-    if (!fix_path(_path, path)) {
+    if (!common::fix_path(_path, path)) {
         return set_errno(r, ENOENT);
     }
 
@@ -256,7 +257,12 @@ Result MountXci(fs::Fs* fs, const fs::FsPath& path, fs::FsPath& out_path) {
     }
 
     auto source = std::make_unique<yati::source::File>(fs, path);
-    yati::container::Xci xci{source.get()};
+
+    s64 size;
+    R_TRY(source->GetSize(&size));
+    auto buffered = std::make_unique<common::BufferedData>(std::move(source), size);
+
+    yati::container::Xci xci{buffered.get()};
     yati::container::Xci::Partitions partitions;
     R_TRY(xci.GetPartitions(partitions));
 
@@ -265,7 +271,7 @@ Result MountXci(fs::Fs* fs, const fs::FsPath& path, fs::FsPath& out_path) {
     entry.devoptab = DEVOPTAB;
     entry.devoptab.name = entry.name;
     entry.devoptab.deviceData = &entry.device;
-    entry.device.source = std::move(source);
+    entry.device.source = std::move(buffered);
     entry.device.partitions = partitions;
     std::snprintf(entry.name, sizeof(entry.name), "xci_%u", g_mount_idx);
     std::snprintf(entry.mount, sizeof(entry.mount), "xci_%u:/", g_mount_idx);

sphaira/source/utils/devoptab_zip.cpp

@@ -1,5 +1,5 @@
-
 #include "utils/devoptab.hpp"
+#include "utils/devoptab_common.hpp"
 #include "defines.hpp"
 #include "log.hpp"
 
@@ -15,86 +15,6 @@
 namespace sphaira::devoptab {
 namespace {
 
-struct BufferedFileData {
-    BufferedFileData(std::unique_ptr<yati::source::Base>&& _source, u64 _size)
-    : source{std::forward<decltype(_source)>(_source)}
-    , capacity{_size} {
-
-    }
-
-    Result Read(void *_buffer, u64 file_off, u64 read_size);
-
-private:
-    std::unique_ptr<yati::source::Base> source;
-    const u64 capacity;
-
-    u64 m_off{};
-    u64 m_size{};
-    u8 m_data[1024 * 64]{};
-};
-
-Result BufferedFileData::Read(void *_buffer, u64 file_off, u64 read_size) {
-    auto dst = static_cast<u8*>(_buffer);
-    size_t amount = 0;
-
-    R_UNLESS(file_off < capacity, FsError_UnsupportedOperateRangeForFileStorage);
-    read_size = std::min(read_size, capacity - file_off);
-
-    if (m_size) {
-        // check if we can read this data into the beginning of dst.
-        if (file_off < m_off + m_size && file_off >= m_off) {
-            const auto off = file_off - m_off;
-            const auto size = std::min<s64>(read_size, m_size - off);
-            if (size) {
-                std::memcpy(dst, m_data + off, size);
-
-                read_size -= size;
-                file_off += size;
-                amount += size;
-                dst += size;
-            }
-        }
-    }
-
-    if (read_size) {
-        const auto alloc_size = sizeof(m_data);
-        m_off = 0;
-        m_size = 0;
-        u64 bytes_read;
-
-        // if the dst is big enough, read data in place.
-        if (read_size > alloc_size) {
-            R_TRY(source->Read(dst, file_off, read_size, &bytes_read));
-
-            read_size -= bytes_read;
-            file_off += bytes_read;
-            amount += bytes_read;
-            dst += bytes_read;
-
-            // save the last chunk of data to the m_buffered io.
-            const auto max_advance = std::min<u64>(amount, alloc_size);
-            m_off = file_off - max_advance;
-            m_size = max_advance;
-            std::memcpy(m_data, dst - max_advance, max_advance);
-        } else {
-            R_TRY(source->Read(m_data, file_off, alloc_size, &bytes_read));
-			const auto bytes_read = alloc_size;
-			const auto max_advance = std::min<u64>(read_size, bytes_read);
-            std::memcpy(dst, m_data, max_advance);
-
-            m_off = file_off;
-            m_size = bytes_read;
-
-            read_size -= max_advance;
-            file_off += max_advance;
-            amount += max_advance;
-            dst += max_advance;
-        }
-    }
-
-    R_SUCCEED();
-}
-
 #define LOCAL_HEADER_SIG 0x4034B50
 #define FILE_HEADER_SIG 0x2014B50
 #define DATA_DESCRIPTOR_SIG 0x8074B50
@@ -193,7 +113,7 @@ struct DirectoryEntry {
 using FileTableEntries = std::vector<FileEntry>;
 
 struct Device {
-    std::unique_ptr<BufferedFileData> source;
+    std::unique_ptr<common::BufferedData> source;
     DirectoryEntry root;
 };
 
@@ -284,7 +204,7 @@ int devoptab_open(struct _reent *r, void *fileStruct, const char *_path, int fla
     std::memset(file, 0, sizeof(*file));
 
     char path[FS_MAX_PATH]{};
-    if (!fix_path(_path, path)) {
+    if (!common::fix_path(_path, path)) {
         return set_errno(r, ENOENT);
     }
 
@@ -462,7 +382,7 @@ DIR_ITER* devoptab_diropen(struct _reent *r, DIR_ITER *dirState, const char *_pa
     std::memset(dir, 0, sizeof(*dir));
 
     char path[FS_MAX_PATH];
-    if (!fix_path(_path, path)) {
+    if (!common::fix_path(_path, path)) {
         set_errno(r, ENOENT);
         return NULL;
     }
@@ -531,7 +451,7 @@ int devoptab_lstat(struct _reent *r, const char *_path, struct stat *st) {
     }
 
     char path[FS_MAX_PATH];
-    if (!fix_path(_path, path)) {
+    if (!common::fix_path(_path, path)) {
         return set_errno(r, ENOENT);
     }
 
@@ -612,7 +532,7 @@ void Parse(const FileTableEntries& entries, DirectoryEntry& out) {
     Parse(entries, index, out);
 }
 
-Result find_central_dir_offset(BufferedFileData* source, s64 size, mmz_EndRecord* record) {
+Result find_central_dir_offset(common::BufferedData* source, s64 size, mmz_EndRecord* record) {
     // check if the record is at the end (no extra header).
     auto offset = size - sizeof(*record);
     R_TRY(source->Read(record, offset, sizeof(*record)));
@@ -640,7 +560,7 @@ Result find_central_dir_offset(BufferedFileData* source, s64 size, mmz_EndRecord
     R_THROW(0x1);
 }
 
-Result ParseZip(BufferedFileData* source, s64 size, FileTableEntries& out) {
+Result ParseZip(common::BufferedData* source, s64 size, FileTableEntries& out) {
     mmz_EndRecord end_rec;
     R_TRY(find_central_dir_offset(source, size, &end_rec));
 
@@ -706,14 +626,12 @@ Result MountZip(fs::Fs* fs, const fs::FsPath& path, fs::FsPath& out_path) {
         }
     }
 
-    // create new entry.
     auto source = std::make_unique<yati::source::File>(fs, path);
 
     s64 size;
     R_TRY(source->GetSize(&size));
-    log_write("[ZIP] got size\n");
 
-    auto buffered = std::make_unique<BufferedFileData>(std::move(source), size);
+    auto buffered = std::make_unique<common::BufferedData>(std::move(source), size);
 
     FileTableEntries table_entries;
     R_TRY(ParseZip(buffered.get(), size, table_entries));