multi thread game dumps and file uploads.

previous uploads were all single threaded, which meant that it only uploaded as fast as the slowest source.
usb transfer is still single threaded due it being random access for both files and data, making it
hard for the read thread to run freely.

sphaira/CMakeLists.txt

@@ -79,6 +79,7 @@ add_executable(sphaira
     source/swkbd.cpp
     source/i18n.cpp
     source/ftpsrv_helper.cpp
+    source/threaded_file_transfer.cpp
 
     source/usb/base.cpp
     source/usb/usbds.cpp

sphaira/include/threaded_file_transfer.hpp

@@ -0,0 +1,17 @@
+#pragma once
+
+#include "ui/progress_box.hpp"
+#include <functional>
+#include <switch.h>
+
+namespace sphaira::thread {
+
+using ReadFunctionCallback = std::function<Result(void* data, s64 off, s64 size, u64* bytes_read)>;
+using WriteFunctionCallback = std::function<Result(const void* data, s64 off, s64 size)>;
+using PullFunctionCallback = std::function<Result(void* data, s64 size, u64* bytes_read)>;
+using StartFunctionCallback = std::function<Result(PullFunctionCallback pull)>;
+
+Result Transfer(ui::ProgressBox* pbox, s64 size, ReadFunctionCallback rfunc, WriteFunctionCallback wfunc);
+Result TransferPull(ui::ProgressBox* pbox, s64 size, ReadFunctionCallback rfunc, StartFunctionCallback sfunc);
+
+} // namespace sphaira::thread

sphaira/include/ui/menus/filebrowser.hpp

@@ -236,7 +236,7 @@ private:
     auto CheckIfUpdateFolder() -> Result;
 
     auto get_collection(const fs::FsPath& path, const fs::FsPath& parent_name, FsDirCollection& out, bool inc_file, bool inc_dir, bool inc_size) -> Result;
-    auto get_collections(const fs::FsPath& path, const fs::FsPath& parent_name, FsDirCollections& out) -> Result;
+    auto get_collections(const fs::FsPath& path, const fs::FsPath& parent_name, FsDirCollections& out, bool inc_size = false) -> Result;
 
     void SetFs(const fs::FsPath& new_path, u32 new_type);
 

sphaira/include/ui/progress_box.hpp

@@ -37,6 +37,10 @@ struct ProgressBox final : Widget {
     auto CopyFile(const fs::FsPath& src, const fs::FsPath& dst) -> Result;
     void Yield();
 
+    auto GetCpuId() const {
+        return m_cpuid;
+    }
+
     auto OnDownloadProgressCallback() {
         return [this](s64 dltotal, s64 dlnow, s64 ultotal, s64 ulnow){
             if (this->ShouldExit()) {
@@ -81,6 +85,7 @@ private:
     std::vector<u8> m_image_data{};
     // shared data end.
 
+    int m_cpuid{};
     int m_image{};
     bool m_own_image{};
 };

sphaira/source/threaded_file_transfer.cpp

@@ -0,0 +1,372 @@
+#include "threaded_file_transfer.hpp"
+#include "log.hpp"
+#include "defines.hpp"
+
+#include <vector>
+#include <algorithm>
+#include <cstring>
+
+namespace sphaira::thread {
+namespace {
+
+constexpr u64 READ_BUFFER_MAX = 1024*1024*4;
+
+struct ThreadBuffer {
+    ThreadBuffer() {
+        buf.reserve(READ_BUFFER_MAX);
+    }
+
+    std::vector<u8> buf;
+    s64 off;
+};
+
+template<std::size_t Size>
+struct RingBuf {
+private:
+    ThreadBuffer buf[Size]{};
+    unsigned r_index{};
+    unsigned w_index{};
+
+    static_assert((sizeof(RingBuf::buf) & (sizeof(RingBuf::buf) - 1)) == 0, "Must be power of 2!");
+
+public:
+    void ringbuf_reset() {
+        this->r_index = this->w_index;
+    }
+
+    unsigned ringbuf_capacity() const {
+        return sizeof(this->buf) / sizeof(this->buf[0]);
+    }
+
+    unsigned ringbuf_size() const {
+        return (this->w_index - this->r_index) % (ringbuf_capacity() * 2U);
+    }
+
+    unsigned ringbuf_free() const {
+        return ringbuf_capacity() - ringbuf_size();
+    }
+
+    void ringbuf_push(std::vector<u8>& buf_in, s64 off_in) {
+        auto& value = this->buf[this->w_index % ringbuf_capacity()];
+        value.off = off_in;
+        std::swap(value.buf, buf_in);
+
+        this->w_index = (this->w_index + 1U) % (ringbuf_capacity() * 2U);
+    }
+
+    void ringbuf_pop(std::vector<u8>& buf_out, s64& off_out) {
+        auto& value = this->buf[this->r_index % ringbuf_capacity()];
+        off_out = value.off;
+        std::swap(value.buf, buf_out);
+
+        this->r_index = (this->r_index + 1U) % (ringbuf_capacity() * 2U);
+    }
+};
+
+struct ThreadData {
+    ThreadData(ui::ProgressBox* _pbox, s64 size, ReadFunctionCallback _rfunc, WriteFunctionCallback _wfunc);
+
+    auto GetResults() -> Result;
+    void WakeAllThreads();
+
+    void SetReadResult(Result result) {
+        read_result = result;
+    }
+
+    void SetWriteResult(Result result) {
+        write_result = result;
+    }
+
+    void SetPullResult(Result result) {
+        pull_result = result;
+    }
+
+    auto GetWriteOffset() const {
+        return write_offset;
+    }
+
+    auto GetWriteSize() const {
+        return write_size;
+    }
+
+    Result Pull(void* data, s64 size, u64* bytes_read);
+    Result readFuncInternal();
+    Result writeFuncInternal();
+
+private:
+    Result SetWriteBuf(std::vector<u8>& buf, s64 size);
+    Result GetWriteBuf(std::vector<u8>& buf_out, s64& off_out);
+    Result SetPullBuf(std::vector<u8>& buf, s64 size);
+    Result GetPullBuf(void* data, s64 size, u64* bytes_read);
+
+    Result Read(void* buf, s64 size, u64* bytes_read);
+
+private:
+    // these need to be copied
+    ui::ProgressBox* pbox{};
+    ReadFunctionCallback rfunc{};
+    WriteFunctionCallback wfunc{};
+
+    // these need to be created
+    Mutex mutex{};
+    Mutex pull_mutex{};
+
+    CondVar can_read{};
+    CondVar can_write{};
+    CondVar can_pull{};
+    CondVar can_pull_write{};
+
+    RingBuf<2> write_buffers{};
+    std::vector<u8> pull_buffer{};
+    s64 pull_buffer_offset{};
+
+    u64 read_buffer_size{};
+    u64 max_buffer_size{};
+
+    // these are shared between threads
+    volatile s64 read_offset{};
+    volatile s64 write_offset{};
+    volatile s64 write_size{};
+
+    volatile Result read_result{};
+    volatile Result write_result{};
+    volatile Result pull_result{};
+};
+
+ThreadData::ThreadData(ui::ProgressBox* _pbox, s64 size, ReadFunctionCallback _rfunc, WriteFunctionCallback _wfunc)
+: pbox{_pbox}, rfunc{_rfunc}, wfunc{_wfunc} {
+    mutexInit(std::addressof(mutex));
+    mutexInit(std::addressof(pull_mutex));
+
+    condvarInit(std::addressof(can_read));
+    condvarInit(std::addressof(can_write));
+    condvarInit(std::addressof(can_pull));
+    condvarInit(std::addressof(can_pull_write));
+
+    write_size = size;
+    read_buffer_size = READ_BUFFER_MAX;
+    max_buffer_size = READ_BUFFER_MAX;
+}
+
+auto ThreadData::GetResults() -> Result {
+    R_UNLESS(!pbox->ShouldExit(), 0x1);
+    R_TRY(read_result);
+    R_TRY(write_result);
+    R_TRY(pull_result);
+    R_SUCCEED();
+}
+
+void ThreadData::WakeAllThreads() {
+    condvarWakeAll(std::addressof(can_read));
+    condvarWakeAll(std::addressof(can_write));
+    condvarWakeAll(std::addressof(can_pull));
+    condvarWakeAll(std::addressof(can_pull_write));
+
+    mutexUnlock(std::addressof(mutex));
+    mutexUnlock(std::addressof(pull_mutex));
+}
+
+Result ThreadData::SetWriteBuf(std::vector<u8>& buf, s64 size) {
+    buf.resize(size);
+
+    mutexLock(std::addressof(mutex));
+    if (!write_buffers.ringbuf_free()) {
+        R_TRY(condvarWait(std::addressof(can_read), std::addressof(mutex)));
+    }
+
+    ON_SCOPE_EXIT(mutexUnlock(std::addressof(mutex)));
+    R_TRY(GetResults());
+    write_buffers.ringbuf_push(buf, 0);
+    return condvarWakeOne(std::addressof(can_write));
+}
+
+Result ThreadData::GetWriteBuf(std::vector<u8>& buf_out, s64& off_out) {
+    mutexLock(std::addressof(mutex));
+    if (!write_buffers.ringbuf_size()) {
+        R_TRY(condvarWait(std::addressof(can_write), std::addressof(mutex)));
+    }
+
+    ON_SCOPE_EXIT(mutexUnlock(std::addressof(mutex)));
+    R_TRY(GetResults());
+    write_buffers.ringbuf_pop(buf_out, off_out);
+    return condvarWakeOne(std::addressof(can_read));
+}
+
+Result ThreadData::SetPullBuf(std::vector<u8>& buf, s64 size) {
+    buf.resize(size);
+
+    mutexLock(std::addressof(pull_mutex));
+    if (!pull_buffer.empty()) {
+        R_TRY(condvarWait(std::addressof(can_pull_write), std::addressof(pull_mutex)));
+    }
+
+    ON_SCOPE_EXIT(mutexUnlock(std::addressof(pull_mutex)));
+    R_TRY(GetResults());
+
+    pull_buffer.swap(buf);
+    return condvarWakeOne(std::addressof(can_pull));
+}
+
+Result ThreadData::GetPullBuf(void* data, s64 size, u64* bytes_read) {
+    mutexLock(std::addressof(pull_mutex));
+    if (pull_buffer.empty()) {
+        R_TRY(condvarWait(std::addressof(can_pull), std::addressof(pull_mutex)));
+    }
+
+    ON_SCOPE_EXIT(mutexUnlock(std::addressof(pull_mutex)));
+    R_TRY(GetResults());
+
+    *bytes_read = size = std::min<s64>(size, pull_buffer.size() - pull_buffer_offset);
+    std::memcpy(data, pull_buffer.data() + pull_buffer_offset, size);
+    pull_buffer_offset += size;
+
+    if (pull_buffer_offset == pull_buffer.size()) {
+        pull_buffer_offset = 0;
+        pull_buffer.clear();
+        return condvarWakeOne(std::addressof(can_pull_write));
+    } else {
+        R_SUCCEED();
+    }
+}
+
+Result ThreadData::Read(void* buf, s64 size, u64* bytes_read) {
+    size = std::min<s64>(size, write_size - read_offset);
+    const auto rc = rfunc(buf, read_offset, size, bytes_read);
+    read_offset += *bytes_read;
+    return rc;
+}
+
+Result ThreadData::Pull(void* data, s64 size, u64* bytes_read) {
+    return GetPullBuf(data, size, bytes_read);
+}
+
+// read thread reads all data from the source
+Result ThreadData::readFuncInternal() {
+    // the main buffer which data is read into.
+    std::vector<u8> buf;
+    buf.reserve(this->max_buffer_size);
+
+    while (this->read_offset < this->write_size && R_SUCCEEDED(this->GetResults())) {
+        // read more data
+        s64 read_size = this->read_buffer_size;
+
+        u64 bytes_read{};
+        buf.resize(read_size);
+        R_TRY(this->Read(buf.data(), read_size, std::addressof(bytes_read)));
+        auto buf_size = bytes_read;
+
+        R_TRY(this->SetWriteBuf(buf, buf_size));
+    }
+
+    log_write("read success\n");
+    R_SUCCEED();
+}
+
+// write thread writes data to the nca placeholder.
+Result ThreadData::writeFuncInternal() {
+    std::vector<u8> buf;
+    buf.reserve(this->max_buffer_size);
+
+    while (this->write_offset < this->write_size && R_SUCCEEDED(this->GetResults())) {
+        s64 dummy_off;
+        R_TRY(this->GetWriteBuf(buf, dummy_off));
+        const auto size = buf.size();
+
+        if (!this->wfunc) {
+            R_TRY(this->SetPullBuf(buf, buf.size()));
+        } else {
+            R_TRY(this->wfunc(buf.data(), this->write_offset, buf.size()));
+        }
+
+        this->write_offset += size;
+    }
+
+    log_write("finished write thread!\n");
+    R_SUCCEED();
+}
+
+void readFunc(void* d) {
+    auto t = static_cast<ThreadData*>(d);
+    t->SetReadResult(t->readFuncInternal());
+    log_write("read thread returned now\n");
+}
+
+void writeFunc(void* d) {
+    auto t = static_cast<ThreadData*>(d);
+    t->SetWriteResult(t->writeFuncInternal());
+    log_write("write thread returned now\n");
+}
+
+auto GetAlternateCore(int id) {
+    return id == 1 ? 2 : 1;
+}
+
+Result TransferInternal(ui::ProgressBox* pbox, s64 size, ReadFunctionCallback rfunc, WriteFunctionCallback wfunc, StartFunctionCallback sfunc) {
+    const auto WRITE_THREAD_CORE = sfunc ? pbox->GetCpuId() : GetAlternateCore(pbox->GetCpuId());
+    const auto READ_THREAD_CORE = GetAlternateCore(WRITE_THREAD_CORE);
+
+    ThreadData t_data{pbox, size, rfunc, wfunc};
+
+    Thread t_read{};
+    R_TRY(threadCreate(&t_read, readFunc, std::addressof(t_data), nullptr, 1024*64, 0x20, READ_THREAD_CORE));
+    ON_SCOPE_EXIT(threadClose(&t_read));
+
+    Thread t_write{};
+    R_TRY(threadCreate(&t_write, writeFunc, std::addressof(t_data), nullptr, 1024*64, 0x20, WRITE_THREAD_CORE));
+    ON_SCOPE_EXIT(threadClose(&t_write));
+
+    log_write("starting threads\n");
+    R_TRY(threadStart(std::addressof(t_read)));
+    ON_SCOPE_EXIT(threadWaitForExit(std::addressof(t_read)));
+
+    R_TRY(threadStart(std::addressof(t_write)));
+    ON_SCOPE_EXIT(threadWaitForExit(std::addressof(t_write)));
+
+    if (sfunc) {
+        t_data.SetPullResult(sfunc([&](void* data, s64 size, u64* bytes_read) -> Result {
+            R_TRY(t_data.GetResults());
+            return t_data.Pull(data, size, bytes_read);
+        }));
+    } else {
+        while (t_data.GetWriteOffset() != t_data.GetWriteSize() && R_SUCCEEDED(t_data.GetResults())) {
+            pbox->UpdateTransfer(t_data.GetWriteOffset(), t_data.GetWriteSize());
+            svcSleepThread(1e+6);
+        }
+    }
+
+    // wait for all threads to close.
+    log_write("waiting for threads to close\n");
+    for (;;) {
+        t_data.WakeAllThreads();
+        pbox->Yield();
+
+        if (R_FAILED(waitSingleHandle(t_read.handle, 1000))) {
+            continue;
+        } else if (R_FAILED(waitSingleHandle(t_write.handle, 1000))) {
+            continue;
+        }
+        break;
+    }
+    log_write("threads closed\n");
+
+    // if any of the threads failed, wake up all threads so they can exit.
+    if (R_FAILED(t_data.GetResults())) {
+        log_write("some reads failed, waking threads\n");
+        log_write("returning due to fail\n");
+        return t_data.GetResults();
+    }
+
+    return t_data.GetResults();
+}
+
+} // namespace
+
+Result Transfer(ui::ProgressBox* pbox, s64 size, ReadFunctionCallback rfunc, WriteFunctionCallback wfunc) {
+    return TransferInternal(pbox, size, rfunc, wfunc, nullptr);
+}
+
+Result TransferPull(ui::ProgressBox* pbox, s64 size, ReadFunctionCallback rfunc, StartFunctionCallback sfunc) {
+    return TransferInternal(pbox, size, rfunc, nullptr, sfunc);
+}
+
+} // namespace::thread

sphaira/source/ui/menus/filebrowser.cpp

@@ -19,6 +19,7 @@
 #include "swkbd.hpp"
 #include "i18n.hpp"
 #include "location.hpp"
+#include "threaded_file_transfer.hpp"
 
 #include "yati/yati.hpp"
 #include "yati/source/file.hpp"
@@ -1200,36 +1201,66 @@ void Menu::UploadFiles() {
             App::Push(std::make_shared<ProgressBox>(0, "Uploading"_i18n, "", [this, loc](auto pbox) -> bool {
                 auto targets = GetSelectedEntries();
 
-                const auto file_add = [&](const fs::FsPath& file_path, const char* name){
+                const auto file_add = [&](s64 file_size, const fs::FsPath& file_path, const char* name) -> Result {
                     // the file name needs to be relative to the current directory.
                     const auto relative_file_name = file_path.s + std::strlen(m_path);
                     pbox->SetTitle(name);
                     pbox->NewTransfer(relative_file_name);
 
-                    const auto result = curl::Api().FromFile(
+                    FsFile file;
-                        CURL_LOCATION_TO_API(loc),
+                    R_TRY(m_fs->OpenFile(file_path, FsOpenMode_Read, &file));
-                        curl::Path{file_path},
+                    ON_SCOPE_EXIT(fsFileClose(&file));
-                        curl::OnProgress{pbox->OnDownloadProgressCallback()},
-                        curl::UploadInfo{relative_file_name}
-                    );
 
-                    return result.success;
+                    return thread::TransferPull(pbox, file_size,
+                        [&](void* data, s64 off, s64 size, u64* bytes_read) -> Result {
+                            return fsFileRead(&file, off, data, size, FsReadOption_None, bytes_read);
+                        },
+                        [&](thread::PullFunctionCallback pull) -> Result {
+                            s64 offset{};
+                            const auto result = curl::Api().FromMemory(
+                                CURL_LOCATION_TO_API(loc),
+                                curl::OnProgress{pbox->OnDownloadProgressCallback()},
+                                curl::UploadInfo{
+                                    relative_file_name, file_size,
+                                    [&](void *ptr, size_t size) -> size_t {
+                                        // curl will request past the size of the file, causing an error.
+                                        if (offset >= file_size) {
+                                            log_write("finished file upload\n");
+                                            return 0;
+                                        }
+
+                                        u64 bytes_read{};
+                                        if (R_FAILED(pull(ptr, size, &bytes_read))) {
+                                            log_write("failed to read in custom callback: %zd size: %zd\n", offset, size);
+                                            return 0;
+                                        }
+
+                                        offset += bytes_read;
+                                        return bytes_read;
+                                    }
+                                }
+                            );
+
+                            R_UNLESS(result.success, 0x1);
+                            R_SUCCEED();
+                        }
+                    );
                 };
 
                 for (auto& e : targets) {
                     if (e.IsFile()) {
                         const auto file_path = GetNewPath(e);
-                        if (!file_add(file_path, e.GetName().c_str())) {
+                        if (R_FAILED(file_add(e.file_size, file_path, e.GetName().c_str()))) {
                             return false;
                         }
                     } else {
                         FsDirCollections collections;
-                        get_collections(GetNewPath(e), e.name, collections);
+                        get_collections(GetNewPath(e), e.name, collections, true);
 
                         for (const auto& collection : collections) {
                             for (const auto& file : collection.files) {
                                 const auto file_path = fs::AppendPath(collection.path, file.name);
-                                if (!file_add(file_path, file.name)) {
+                                if (R_FAILED(file_add(file.file_size, file_path, file.name))) {
                                     return false;
                                 }
                             }
@@ -1887,10 +1918,10 @@ auto Menu::get_collection(const fs::FsPath& path, const fs::FsPath& parent_name,
     R_SUCCEED();
 }
 
-auto Menu::get_collections(const fs::FsPath& path, const fs::FsPath& parent_name, FsDirCollections& out) -> Result {
+auto Menu::get_collections(const fs::FsPath& path, const fs::FsPath& parent_name, FsDirCollections& out, bool inc_size) -> Result {
     // get a list of all the files / dirs
     FsDirCollection collection;
-    R_TRY(get_collection(path, parent_name, collection, true, true, false));
+    R_TRY(get_collection(path, parent_name, collection, true, true, inc_size));
     log_write("got collection: %s parent_name: %s files: %zu dirs: %zu\n", path.s, parent_name.s, collection.files.size(), collection.dirs.size());
     out.emplace_back(collection);
 
@@ -1900,7 +1931,7 @@ auto Menu::get_collections(const fs::FsPath& path, const fs::FsPath& parent_name
         const auto new_path = std::make_unique<fs::FsPath>(Menu::GetNewPath(path, p.name));
         const auto new_parent_name = std::make_unique<fs::FsPath>(Menu::GetNewPath(parent_name, p.name));
         log_write("trying to get nested collection: %s parent_name: %s\n", new_path->s, new_parent_name->s);
-        R_TRY(get_collections(*new_path, *new_parent_name, out));
+        R_TRY(get_collections(*new_path, *new_parent_name, out, inc_size));
     }
 
     R_SUCCEED();

sphaira/source/ui/menus/game_menu.cpp

@@ -5,6 +5,7 @@
 #include "defines.hpp"
 #include "i18n.hpp"
 #include "location.hpp"
+#include "threaded_file_transfer.hpp"
 
 #include "ui/menus/game_menu.hpp"
 #include "ui/sidebar.hpp"
@@ -291,15 +292,17 @@ Result DumpNspToFile(ProgressBox* pbox, std::span<NspEntry> entries) {
 
     auto source = std::make_unique<NspSource>(entries);
     for (const auto& e : entries) {
+        auto path = e.path;
+        const auto file_size = e.nsp_size;
         pbox->SetTitle(e.application_name);
-        pbox->NewTransfer(e.path);
+        pbox->NewTransfer(path);
 
-        const auto temp_path = fs::AppendPath(DUMP_PATH, e.path + ".temp");
+        const auto temp_path = fs::AppendPath(DUMP_PATH, path + ".temp");
         fs.CreateDirectoryRecursivelyWithPath(temp_path);
         fs.DeleteFile(temp_path);
 
-        const auto flags = e.nsp_size >= BIG_FILE_SIZE ? FsCreateOption_BigFile : 0;
+        const auto flags = file_size >= BIG_FILE_SIZE ? FsCreateOption_BigFile : 0;
-        R_TRY(fs.CreateFile(temp_path, e.nsp_size, flags));
+        R_TRY(fs.CreateFile(temp_path, file_size, flags));
         ON_SCOPE_EXIT(fs.DeleteFile(temp_path));
 
         {
@@ -307,27 +310,17 @@ Result DumpNspToFile(ProgressBox* pbox, std::span<NspEntry> entries) {
             R_TRY(fs.OpenFile(temp_path, FsOpenMode_Write, &file));
             ON_SCOPE_EXIT(fsFileClose(&file));
 
-            s64 offset{};
+            R_TRY(thread::Transfer(pbox, file_size,
-            std::vector<u8> buf(1024*1024*4); // 4MiB
+                [&](void* data, s64 off, s64 size, u64* bytes_read) -> Result {
-
+                    return source->Read(path, data, off, size, bytes_read);
-            while (offset < e.nsp_size) {
+                },
-                if (pbox->ShouldExit()) {
+                [&](const void* data, s64 off, s64 size) -> Result {
-                    R_THROW(0xFFFF);
+                    return fsFileWrite(&file, off, data, size, FsWriteOption_None);
                 }
-
+            ));
-                u64 bytes_read;
-                R_TRY(source->Read(e.path, buf.data(), offset, buf.size(), &bytes_read));
-                pbox->Yield();
-
-                R_TRY(fsFileWrite(&file, offset, buf.data(), bytes_read, FsWriteOption_None));
-                pbox->Yield();
-
-                pbox->UpdateTransfer(offset, e.nsp_size);
-                offset += bytes_read;
-            }
         }
 
-        const auto path = fs::AppendPath(DUMP_PATH, e.path);
+        path = fs::AppendPath(DUMP_PATH, path);
         fs.DeleteFile(path);
         R_TRY(fs.RenameFile(temp_path, path));
     }
@@ -373,24 +366,19 @@ Result DumpNspToUsbS2S(ProgressBox* pbox, std::span<NspEntry> entries) {
 Result DumpNspToDevNull(ProgressBox* pbox, std::span<NspEntry> entries) {
     auto source = std::make_unique<NspSource>(entries);
     for (const auto& e : entries) {
+        const auto path = e.path;
+        const auto file_size = e.nsp_size;
         pbox->SetTitle(e.application_name);
-        pbox->NewTransfer(e.path);
+        pbox->NewTransfer(path);
 
-        s64 offset{};
+        R_TRY(thread::Transfer(pbox, file_size,
-        std::vector<u8> buf(1024*1024*4); // 4MiB
+            [&](void* data, s64 off, s64 size, u64* bytes_read) -> Result {
-
+                return source->Read(path, data, off, size, bytes_read);
-        while (offset < e.nsp_size) {
+            },
-            if (pbox->ShouldExit()) {
+            [&](const void* data, s64 off, s64 size) -> Result {
-                R_THROW(0xFFFF);
+                R_SUCCEED();
             }
-
+        ));
-            u64 bytes_read;
-            R_TRY(source->Read(e.path, buf.data(), offset, buf.size(), &bytes_read));
-            pbox->Yield();
-
-            pbox->UpdateTransfer(offset, e.nsp_size);
-            offset += bytes_read;
-        }
     }
 
     R_SUCCEED();
@@ -403,39 +391,45 @@ Result DumpNspToNetwork(ProgressBox* pbox, const location::Entry& loc, std::span
             R_THROW(0xFFFF);
         }
 
+        const auto path = e.path;
+        const auto file_size = e.nsp_size;
         pbox->SetTitle(e.application_name);
-        pbox->NewTransfer(e.path);
+        pbox->NewTransfer(path);
 
-        s64 offset{};
+        R_TRY(thread::TransferPull(pbox, file_size,
-        const auto result = curl::Api().FromMemory(
+            [&](void* data, s64 off, s64 size, u64* bytes_read) -> Result {
-            CURL_LOCATION_TO_API(loc),
+                return source->Read(path, data, off, size, bytes_read);
-            curl::OnProgress{pbox->OnDownloadProgressCallback()},
-            curl::UploadInfo{
-                e.path, e.nsp_size,
-                [&pbox, &e, &source, &offset](void *ptr, size_t size) -> size_t {
-                    u64 bytes_read{};
-                    if (R_FAILED(source->Read(e.path, ptr, offset, size, &bytes_read))) {
-                        // curl will request past the size of the file, causing an error.
-                        // only log the error if it failed in the middle of a transfer.
-                        if (offset != e.nsp_size) {
-                            log_write("failed to read in custom callback: %zd size: %zd\n", offset, e.nsp_size);
-                        }
-                        return 0;
-                    }
-
-                    offset += bytes_read;
-                    return bytes_read;
-                }
             },
-            curl::OnUploadSeek{
+            [&](thread::PullFunctionCallback pull) -> Result {
-                [&e, &offset](s64 new_offset){
+                s64 offset{};
-                    offset = new_offset;
+                const auto result = curl::Api().FromMemory(
-                    return true;
+                    CURL_LOCATION_TO_API(loc),
-                }
+                    curl::OnProgress{pbox->OnDownloadProgressCallback()},
-            }
+                    curl::UploadInfo{
-        );
+                        path, file_size,
+                        [&](void *ptr, size_t size) -> size_t {
+                            // curl will request past the size of the file, causing an error.
+                            if (offset >= file_size) {
+                                log_write("finished file upload\n");
+                                return 0;
+                            }
 
-        R_UNLESS(result.success, 0x1);
+                            u64 bytes_read{};
+                            if (R_FAILED(pull(ptr, size, &bytes_read))) {
+                                log_write("failed to read in custom callback: %zd size: %zd\n", offset, size);
+                                return 0;
+                            }
+
+                            offset += bytes_read;
+                            return bytes_read;
+                        }
+                    }
+                );
+
+                R_UNLESS(result.success, 0x1);
+                R_SUCCEED();
+            }
+        ));
     }
 
     R_SUCCEED();

sphaira/source/ui/menus/gc_menu.cpp

@@ -13,6 +13,7 @@
 #include "i18n.hpp"
 #include "download.hpp"
 #include "location.hpp"
+#include "threaded_file_transfer.hpp"
 
 #include <cstring>
 #include <algorithm>
@@ -285,6 +286,7 @@ Result DumpNspToFile(ProgressBox* pbox, std::span<const fs::FsPath> paths, XciEn
     R_TRY(fs.GetFsOpenResult());
 
     for (auto path : paths) {
+        const auto file_size = e.GetSize(path);
         pbox->SetTitle(e.application_name);
         pbox->NewTransfer(path);
 
@@ -292,9 +294,8 @@ Result DumpNspToFile(ProgressBox* pbox, std::span<const fs::FsPath> paths, XciEn
         fs.CreateDirectoryRecursivelyWithPath(temp_path);
         fs.DeleteFile(temp_path);
 
-        const auto size = e.GetSize(path);
+        const auto flags = file_size >= BIG_FILE_SIZE ? FsCreateOption_BigFile : 0;
-        const auto flags = size >= BIG_FILE_SIZE ? FsCreateOption_BigFile : 0;
+        R_TRY(fs.CreateFile(temp_path, file_size, flags));
-        R_TRY(fs.CreateFile(temp_path, size, flags));
         ON_SCOPE_EXIT(fs.DeleteFile(temp_path));
 
         {
@@ -302,24 +303,14 @@ Result DumpNspToFile(ProgressBox* pbox, std::span<const fs::FsPath> paths, XciEn
             R_TRY(fs.OpenFile(temp_path, FsOpenMode_Write, &file));
             ON_SCOPE_EXIT(fsFileClose(&file));
 
-            s64 offset{};
+            R_TRY(thread::Transfer(pbox, file_size,
-            std::vector<u8> buf(1024*1024*4); // 4MiB
+                [&](void* data, s64 off, s64 size, u64* bytes_read) -> Result {
-
+                    return e.Read(path, data, off, size, bytes_read);
-            while (offset < size) {
+                },
-                if (pbox->ShouldExit()) {
+                [&](const void* data, s64 off, s64 size) -> Result {
-                    R_THROW(0xFFFF);
+                    return fsFileWrite(&file, off, data, size, FsWriteOption_None);
                 }
-
+            ));
-                u64 bytes_read;
-                R_TRY(e.Read(path, buf.data(), offset, buf.size(), &bytes_read));
-                pbox->Yield();
-
-                R_TRY(fsFileWrite(&file, offset, buf.data(), bytes_read, FsWriteOption_None));
-                pbox->Yield();
-
-                pbox->UpdateTransfer(offset, size);
-                offset += bytes_read;
-            }
         }
 
         path = fs::AppendPath(DUMP_PATH, path);
@@ -367,25 +358,18 @@ Result DumpNspToUsbS2S(ProgressBox* pbox, std::span<const fs::FsPath> paths, Xci
 
 Result DumpNspToDevNull(ProgressBox* pbox, std::span<const fs::FsPath> paths, XciEntry& e) {
     for (const auto& path : paths) {
+        const auto file_size = e.GetSize(path);
         pbox->SetTitle(e.application_name);
         pbox->NewTransfer(path);
 
-        s64 offset{};
+        R_TRY(thread::Transfer(pbox, file_size,
-        const auto size = e.GetSize(path);
+            [&](void* data, s64 off, s64 size, u64* bytes_read) -> Result {
-        std::vector<u8> buf(1024*1024*4); // 4MiB
+                return e.Read(path, data, off, size, bytes_read);
-
+            },
-        while (offset < size) {
+            [&](const void* data, s64 off, s64 size) -> Result {
-            if (pbox->ShouldExit()) {
+                R_SUCCEED();
-                R_THROW(0xFFFF);
             }
-
+        ));
-            u64 bytes_read;
-            R_TRY(e.Read(path, buf.data(), offset, buf.size(), &bytes_read));
-            pbox->Yield();
-
-            pbox->UpdateTransfer(offset, size);
-            offset += bytes_read;
-        }
     }
 
     R_SUCCEED();
@@ -397,41 +381,45 @@ Result DumpNspToNetwork(ProgressBox* pbox, const location::Entry& loc, std::span
             R_THROW(0xFFFF);
         }
 
+        const auto file_size = e.GetSize(path);
         pbox->SetTitle(e.application_name);
         pbox->NewTransfer(path);
 
-        s64 offset{};
+        R_TRY(thread::TransferPull(pbox, file_size,
-        const auto size = e.GetSize(path);
+            [&](void* data, s64 off, s64 size, u64* bytes_read) -> Result {
-
+                return e.Read(path, data, off, size, bytes_read);
-        const auto result = curl::Api().FromMemory(
-            CURL_LOCATION_TO_API(loc),
-            curl::OnProgress{pbox->OnDownloadProgressCallback()},
-            curl::UploadInfo{
-                path, size,
-                [&pbox, &e, &offset, &path](void *ptr, size_t size) -> size_t {
-                    u64 bytes_read{};
-                    if (R_FAILED(e.Read(path, ptr, offset, size, &bytes_read))) {
-                        // curl will request past the size of the file, causing an error.
-                        // only log the error if it failed in the middle of a transfer.
-                        if (offset != size) {
-                            log_write("failed to read in custom callback: %zd size: %zd\n", offset, size);
-                        }
-                        return 0;
-                    }
-
-                    offset += bytes_read;
-                    return bytes_read;
-                }
             },
-            curl::OnUploadSeek{
+            [&](thread::PullFunctionCallback pull) -> Result {
-                [&offset](s64 new_offset){
+                s64 offset{};
-                    offset = new_offset;
+                const auto result = curl::Api().FromMemory(
-                    return true;
+                    CURL_LOCATION_TO_API(loc),
-                }
+                    curl::OnProgress{pbox->OnDownloadProgressCallback()},
-            }
+                    curl::UploadInfo{
-        );
+                        path, file_size,
+                        [&](void *ptr, size_t size) -> size_t {
+                            // curl will request past the size of the file, causing an error.
+                            if (offset >= file_size) {
+                                log_write("finished file upload\n");
+                                return 0;
+                            }
 
-        R_UNLESS(result.success, 0x1);
+                            u64 bytes_read{};
+                            if (R_FAILED(pull(ptr, size, &bytes_read))) {
+                                log_write("failed to read in custom callback: %zd size: %zd\n", offset, size);
+                                return 0;
+                            }
+
+                            offset += bytes_read;
+                            return bytes_read;
+                        }
+                    }
+                );
+
+
+                R_UNLESS(result.success, 0x1);
+                R_SUCCEED();
+            }
+        ));
     }
 
     R_SUCCEED();

sphaira/source/ui/progress_box.cpp

@@ -38,6 +38,7 @@ ProgressBox::ProgressBox(int image, const std::string& action, const std::string
     m_action = action;
     m_image = image;
 
+    m_cpuid = cpuid;
     m_thread_data.pbox = this;
     m_thread_data.callback = callback;
     if (R_FAILED(threadCreate(&m_thread, threadFunc, &m_thread_data, nullptr, stack_size, prio, cpuid))) {