make all threads preemptive. workaround file based emummc hanging due to fs r/w hanging the system.

the fix for file based emummc is to simply sleep between fs r/w to the sd card.
the performance impact is minimal, even with the reduced buffer size.

the above *only* applies for when using file based emummc. not affecting using partition or sysmmc.

sphaira/CMakeLists.txt

@@ -161,7 +161,7 @@ FetchContent_Declare(ftpsrv
 
 FetchContent_Declare(libhaze
     GIT_REPOSITORY https://github.com/ITotalJustice/libhaze.git
-    GIT_TAG 04f1526
+    GIT_TAG 8e16df2
 )
 
 FetchContent_Declare(libpulsar

sphaira/include/app.hpp

@@ -35,6 +35,11 @@ enum class LaunchType {
     Forwader_Sphaira,
 };
 
+struct AmsEmummcPaths {
+    char file_based_path[0x80];
+    char nintendo[0x80];
+};
+
 // todo: why is this global???
 void DrawElement(float x, float y, float w, float h, ThemeEntryID id);
 void DrawElement(const Vec4&, ThemeEntryID id);
@@ -159,20 +164,9 @@ public:
         return R_SUCCEEDED(pmdmntGetApplicationProcessId(&pid));
     }
 
-    static auto IsEmunand() -> bool {
+    static auto IsEmummc() -> bool;
-        alignas(0x1000) struct EmummcPaths {
+    static auto IsParitionBaseEmummc() -> bool;
-            char unk[0x80];
+    static auto IsFileBaseEmummc() -> bool;
-            char nintendo[0x80];
-        } paths{};
-
-        SecmonArgs args{};
-        args.X[0] = 0xF0000404; /* smcAmsGetEmunandConfig */
-        args.X[1] = 0; /* EXO_EMUMMC_MMC_NAND*/
-        args.X[2] = (u64)&paths; /* out path */
-        svcCallSecureMonitor(&args);
-
-        return (paths.unk[0] != '\0') || (paths.nintendo[0] != '\0');
-    }
 
     static void SetAutoSleepDisabled(bool enable) {
         static Mutex mutex{};
@@ -225,6 +219,7 @@ public:
     fs::FsPath theme_path{};
     s64 m_theme_index{};
 
+    AmsEmummcPaths m_emummc_paths{};
     bool m_quit{};
 
     // network

sphaira/include/defines.hpp

@@ -524,27 +524,15 @@ enum NcmError {
 #define ON_SCOPE_FAIL(_f) std::experimental::scope_exit ANONYMOUS_VARIABLE(SCOPE_EXIT_STATE_){[&] { if (R_FAILED(rc)) { _f; } }};
 #define ON_SCOPE_SUCCESS(_f) std::experimental::scope_exit ANONYMOUS_VARIABLE(SCOPE_EXIT_STATE_){[&] { if (R_SUCCEEDED(rc)) { _f; } }};
 
-#if 0
+// threading helpers.
-constexpr auto cexprHash(const char *str, std::size_t v = 0) noexcept -> std::size_t {
+#define PRIO_PREEMPTIVE 0x3B
-    return (*str == 0) ? v : 31 * cexprHash(str + 1) + *str;
-}
 
-constexpr auto cexprStrlen(const char *str) noexcept -> std::size_t {
+// threading affinity, use with svcSetThreadCoreMask().
-    return (*str != 0) ? 1 + cexprStrlen(str + 1) : 0;
+#define THREAD_AFFINITY_CORE0 BIT(0)
-}
+#define THREAD_AFFINITY_CORE1 BIT(1)
+#define THREAD_AFFINITY_CORE2 BIT(2)
+#define THREAD_AFFINITY_DEFAULT(core) (BIT(core)|THREAD_AFFINITY_CORE1|THREAD_AFFINITY_CORE2)
+#define THREAD_AFFINITY_ALL (THREAD_AFFINITY_CORE0|THREAD_AFFINITY_CORE1|THREAD_AFFINITY_CORE2)
 
-inline void showError(const char* title, const char* desc, Result rc) {
+// mutex helpers.
-    const auto type = appletGetAppletType();
+#define SCOPED_MUTEX(mutex) mutexLock(mutex); ON_SCOPE_EXIT(mutexUnlock(mutex))
-    if (type == AppletType_Application || type == AppletType_SystemApplication) {
-        ErrorApplicationConfig cfg;
-        errorApplicationCreate(&cfg, title, desc);
-        errorApplicationSetNumber(&cfg, rc);
-        errorApplicationShow(&cfg);
-    } else {
-        ErrorSystemConfig cfg;
-        errorSystemCreate(&cfg, title, desc);
-        errorSystemSetResult(&cfg, rc);
-        errorSystemShow(&cfg);
-    }
-}
-#endif

sphaira/include/ui/progress_box.hpp

@@ -18,7 +18,7 @@ struct ProgressBox final : Widget {
         const std::string& action,
         const std::string& title,
         ProgressBoxCallback callback, ProgressBoxDoneCallback done = [](Result rc){},
-        int cpuid = 1, int prio = 0x2C, int stack_size = 1024*128
+        int cpuid = 1, int prio = PRIO_PREEMPTIVE, int stack_size = 1024*128
     );
     ~ProgressBox();
 

sphaira/source/app.cpp

@@ -630,7 +630,7 @@ auto App::GetReplaceHbmenuEnable() -> bool {
 }
 
 auto App::GetInstallEnable() -> bool {
-    if (IsEmunand()) {
+    if (IsEmummc()) {
         return GetInstallEmummcEnable();
     } else {
         return GetInstallSysmmcEnable();
@@ -930,6 +930,21 @@ auto App::Install(ui::ProgressBox* pbox, OwoConfig& config) -> Result {
     return rc;
 }
 
+auto App::IsEmummc() -> bool {
+    const auto& paths = g_app->m_emummc_paths;
+    return (paths.file_based_path[0] != '\0') || (paths.nintendo[0] != '\0');
+}
+
+auto App::IsParitionBaseEmummc() -> bool {
+    const auto& paths = g_app->m_emummc_paths;
+    return (paths.file_based_path[0] == '\0') && (paths.nintendo[0] != '\0');
+}
+
+auto App::IsFileBaseEmummc() -> bool {
+    const auto& paths = g_app->m_emummc_paths;
+    return (paths.file_based_path[0] != '\0') && (paths.nintendo[0] != '\0');
+}
+
 void App::Exit() {
     g_app->m_quit = true;
 }
@@ -1288,6 +1303,21 @@ App::App(const char* argv0) {
         __nx_applet_exit_mode = 1;
     }
 
+    // get emummc config.
+    alignas(0x1000) AmsEmummcPaths paths{};
+    SecmonArgs args{};
+    args.X[0] = 0xF0000404; /* smcAmsGetEmunandConfig */
+    args.X[1] = 0; /* EXO_EMUMMC_MMC_NAND*/
+    args.X[2] = (u64)&paths; /* out path */
+    svcCallSecureMonitor(&args);
+    m_emummc_paths = paths;
+
+    log_write("emummc : %u\n", App::IsEmummc());
+    if (App::IsEmummc()) {
+        log_write("[emummc] file based path: %s\n", m_emummc_paths.file_based_path);
+        log_write("[emummc] nintendo path: %s\n", m_emummc_paths.nintendo);
+    }
+
     fs::FsNativeSd fs;
     fs.CreateDirectoryRecursively("/config/sphaira");
     fs.CreateDirectory("/config/sphaira/assoc");
@@ -1350,7 +1380,7 @@ App::App(const char* argv0) {
     }
 
     if (App::GetMtpEnable()) {
-        hazeInitialize(haze_callback, 0x2C, 2);
+        hazeInitialize(haze_callback, PRIO_PREEMPTIVE, 2);
     }
 
     if (App::GetFtpEnable()) {

sphaira/source/download.cpp

@@ -32,7 +32,7 @@ namespace {
 constexpr auto API_AGENT = "TotalJustice";
 constexpr u64 CHUNK_SIZE = 1024*1024;
 constexpr auto MAX_THREADS = 4;
-constexpr int THREAD_PRIO = 0x2C;
+constexpr int THREAD_PRIO = PRIO_PREEMPTIVE;
 constexpr int THREAD_CORE = 1;
 
 std::atomic_bool g_running{};
@@ -71,6 +71,10 @@ auto generate_key_from_path(const fs::FsPath& path) -> std::string {
     return std::to_string(key);
 }
 
+void Yield() {
+    svcSleepThread(YieldType_WithoutCoreMigration);
+}
+
 struct Cache {
     using Value = std::pair<std::string, std::string>;
 
@@ -259,6 +263,7 @@ struct ThreadEntry {
 
         ueventCreate(&m_uevent, true);
         R_TRY(threadCreate(&m_thread, ThreadFunc, this, nullptr, 1024*32, THREAD_PRIO, THREAD_CORE));
+        R_TRY(svcSetThreadCoreMask(m_thread.handle, THREAD_CORE, THREAD_AFFINITY_DEFAULT(THREAD_CORE)));
         R_TRY(threadStart(&m_thread));
         R_SUCCEED();
     }
@@ -371,7 +376,7 @@ auto ProgressCallbackFunc1(void *clientp, curl_off_t dltotal, curl_off_t dlnow,
         return 1;
     }
 
-    svcSleepThread(YieldType_WithoutCoreMigration);
+    Yield();
     return 0;
 }
 
@@ -386,7 +391,7 @@ auto ProgressCallbackFunc2(void *clientp, curl_off_t dltotal, curl_off_t dlnow,
         return 1;
     }
 
-    svcSleepThread(YieldType_WithoutCoreMigration);
+    Yield();
     return 0;
 }
 
@@ -445,7 +450,7 @@ auto ReadFileCallback(char *ptr, size_t size, size_t nmemb, void *userp) -> size
     }
 
     data_struct->offset += bytes_read;
-    svcSleepThread(YieldType_WithoutCoreMigration);
+    Yield();
     return bytes_read;
 }
 
@@ -461,7 +466,7 @@ auto ReadMemoryCallback(char *ptr, size_t size, size_t nmemb, void *userp) -> si
     std::memcpy(ptr, data_struct->data.data(), realsize);
     data_struct->offset += realsize;
 
-    svcSleepThread(YieldType_WithoutCoreMigration);
+    Yield();
     return realsize;
 }
 
@@ -474,7 +479,7 @@ auto ReadCustomCallback(char *ptr, size_t size, size_t nmemb, void *userp) -> si
     auto realsize = size * nmemb;
     const auto result = data_struct->m_callback(ptr, realsize);
 
-    svcSleepThread(YieldType_WithoutCoreMigration);
+    Yield();
     return result;
 }
 
@@ -495,7 +500,7 @@ auto WriteMemoryCallback(void *contents, size_t size, size_t num_files, void *us
     std::memcpy(data_struct->data.data() + data_struct->offset, contents, realsize);
     data_struct->offset += realsize;
 
-    svcSleepThread(YieldType_WithoutCoreMigration);
+    Yield();
     return realsize;
 }
 
@@ -530,7 +535,7 @@ auto WriteFileCallback(void *contents, size_t size, size_t num_files, void *user
         data_struct->offset += realsize;
     }
 
-    svcSleepThread(YieldType_WithoutCoreMigration);
+    Yield();
     return realsize;
 }
 

sphaira/source/ftpsrv_helper.cpp

@@ -30,6 +30,8 @@ struct InstallSharedData {
 };
 
 const char* INI_PATH = "/config/ftpsrv/config.ini";
+constexpr int THREAD_PRIO = PRIO_PREEMPTIVE;
+constexpr int THREAD_CORE = 2;
 FtpSrvConfig g_ftpsrv_config = {0};
 volatile bool g_should_exit = false;
 bool g_is_running{false};
@@ -357,11 +359,16 @@ bool Init() {
     vfs_nx_init(&custom, mount_devices, save_writable, mount_bis);
 
     Result rc;
-    if (R_FAILED(rc = threadCreate(&g_thread, loop, nullptr, nullptr, 1024*16, 0x2C, 2))) {
+    if (R_FAILED(rc = threadCreate(&g_thread, loop, nullptr, nullptr, 1024*16, THREAD_PRIO, THREAD_CORE))) {
         log_write("[FTP] failed to create nxlink thread: 0x%X\n", rc);
         return false;
     }
 
+    if (R_FAILED(rc = svcSetThreadCoreMask(g_thread.handle, THREAD_CORE, THREAD_AFFINITY_DEFAULT(THREAD_CORE)))) {
+        log_write("[FTP] failed to set core mask: 0x%X\n", rc);
+        return false;
+    }
+
     if (R_FAILED(rc = threadStart(&g_thread))) {
         log_write("[FTP] failed to start nxlink thread: 0x%X\n", rc);
         threadClose(&g_thread);

sphaira/source/nxlink.cpp

@@ -458,7 +458,7 @@ bool nxlinkInitialize(NxlinkCallback callback) {
     g_quit = false;
 
     Result rc;
-    if (R_FAILED(rc = threadCreate(&g_thread, loop, nullptr, nullptr, 1024*64, 0x2C, 2))) {
+    if (R_FAILED(rc = threadCreate(&g_thread, loop, nullptr, nullptr, 1024*64, PRIO_PREEMPTIVE, 2))) {
         log_write("failed to create nxlink thread: 0x%X\n", rc);
         return false;
     }

sphaira/source/threaded_file_transfer.cpp

@@ -145,8 +145,14 @@ ThreadData::ThreadData(ui::ProgressBox* _pbox, s64 size, ReadCallback _rfunc, Wr
     condvarInit(std::addressof(can_pull_write));
 
     write_size = size;
+
+    if (App::IsFileBaseEmummc()) {
+        read_buffer_size = 1024 * 512;
+        max_buffer_size = 1024 * 512;
+    } else {
         read_buffer_size = READ_BUFFER_MAX;
         max_buffer_size = READ_BUFFER_MAX;
+    }
 }
 
 auto ThreadData::GetResults() -> Result {
@@ -312,11 +318,11 @@ Result TransferInternal(ui::ProgressBox* pbox, s64 size, ReadCallback rfunc, Wri
     ThreadData t_data{pbox, size, rfunc, wfunc};
 
     Thread t_read{};
-    R_TRY(threadCreate(&t_read, readFunc, std::addressof(t_data), nullptr, 1024*64, 0x20, READ_THREAD_CORE));
+    R_TRY(threadCreate(&t_read, readFunc, std::addressof(t_data), nullptr, 1024*64, PRIO_PREEMPTIVE, READ_THREAD_CORE));
     ON_SCOPE_EXIT(threadClose(&t_read));
 
     Thread t_write{};
-    R_TRY(threadCreate(&t_write, writeFunc, std::addressof(t_data), nullptr, 1024*64, 0x20, WRITE_THREAD_CORE));
+    R_TRY(threadCreate(&t_write, writeFunc, std::addressof(t_data), nullptr, 1024*64, PRIO_PREEMPTIVE, WRITE_THREAD_CORE));
     ON_SCOPE_EXIT(threadClose(&t_write));
 
     const auto start_threads = [&]() -> Result {

sphaira/source/ui/menus/game_menu.cpp

@@ -27,6 +27,9 @@
 namespace sphaira::ui::menu::game {
 namespace {
 
+constexpr int THREAD_PRIO = PRIO_PREEMPTIVE;
+constexpr int THREAD_CORE = 1;
+
 constexpr u32 ContentMetaTypeToContentFlag(u8 meta_type) {
     if (meta_type & 0x80) {
         return 1 << (meta_type - 0x80);
@@ -649,9 +652,7 @@ void ThreadData::Run() {
             }
 
             // sleep after every other entry loaded.
-            if (i) {
+            svcSleepThread(2e+6); // 2ms
-                svcSleepThread(1e+6*2); // 2ms
-            }
 
             const auto result = LoadControlEntry(ids[i]);
             mutexLock(&m_mutex_result);
@@ -875,7 +876,8 @@ Menu::Menu(u32 flags) : grid::Menu{"Games"_i18n, flags} {
         e.Open();
     }
 
-    threadCreate(&m_thread, ThreadFunc, &m_thread_data, nullptr, 1024*32, 0x30, 1);
+    threadCreate(&m_thread, ThreadFunc, &m_thread_data, nullptr, 1024*32, THREAD_PRIO, THREAD_CORE);
+    svcSetThreadCoreMask(m_thread.handle, THREAD_CORE, THREAD_AFFINITY_DEFAULT(THREAD_CORE));
     threadStart(&m_thread);
 }
 

sphaira/source/ui/menus/gc_menu.cpp

@@ -646,6 +646,7 @@ Result Menu::GcMount() {
         if (m_option_index == 2) {
             SetPop();
         } else {
+            log_write("[GC] pressed A\n");
             if (!m_mounted) {
                 return;
             }
@@ -658,6 +659,7 @@ Result Menu::GcMount() {
                         "OK"_i18n
                     ));
                 } else {
+                    log_write("[GC] doing install A\n");
                     App::Push(std::make_shared<ui::ProgressBox>(m_icon, "Installing "_i18n, m_entries[m_entry_index].lang_entry.name, [this](auto pbox) -> Result {
                         auto source = std::make_shared<GcSource>(m_entries[m_entry_index], m_fs.get());
                         return yati::InstallFromCollections(pbox, source, source->m_collections, source->m_config);

sphaira/source/ui/menus/usb_menu.cpp

@@ -77,7 +77,7 @@ Menu::Menu(u32 flags) : MenuBase{"USB"_i18n, flags} {
     mutexInit(&m_mutex);
 
     if (m_state != State::Failed) {
-        threadCreate(&m_thread, thread_func, this, nullptr, 1024*32, 0x2C, 1);
+        threadCreate(&m_thread, thread_func, this, nullptr, 1024*32, PRIO_PREEMPTIVE, 1);
         threadStart(&m_thread);
     }
 }

sphaira/source/ui/progress_box.cpp

@@ -254,6 +254,9 @@ auto ProgressBox::ShouldExitResult() -> Result {
 }
 
 auto ProgressBox::CopyFile(fs::Fs* fs_src, fs::Fs* fs_dst, const fs::FsPath& src_path, const fs::FsPath& dst_path) -> Result {
+    const auto is_file_based_emummc = App::IsFileBaseEmummc();
+    const auto is_both_native = fs_src->IsNative() && fs_dst->IsNative();
+
     fs::File src_file;
     R_TRY(fs_src->OpenFile(src_path, FsOpenMode_Read, &src_file));
 
@@ -271,7 +274,13 @@ auto ProgressBox::CopyFile(fs::Fs* fs_src, fs::Fs* fs_dst, const fs::FsPath& src
     R_TRY(dst_file.SetSize(src_size));
 
     s64 offset{};
-    std::vector<u8> buf(1024*1024*4); // 4MiB
+    std::vector<u8> buf;
+
+    if (is_file_based_emummc) {
+        buf.resize(1024*512); // 512KiB
+    } else {
+        buf.resize(1024*1024*4); // 4MiB
+    }
 
     while (offset < src_size) {
         R_TRY(ShouldExitResult());
@@ -280,11 +289,19 @@ auto ProgressBox::CopyFile(fs::Fs* fs_src, fs::Fs* fs_dst, const fs::FsPath& src
         R_TRY(src_file.Read(offset, buf.data(), buf.size(), 0, &bytes_read));
         Yield();
 
+        if (is_both_native && is_file_based_emummc) {
+            svcSleepThread(2e+6); // 2ms
+        }
+
         R_TRY(dst_file.Write(offset, buf.data(), bytes_read, FsWriteOption_None));
         Yield();
 
         UpdateTransfer(offset, src_size);
         offset += bytes_read;
+
+        if (is_both_native && is_file_based_emummc) {
+            svcSleepThread(2e+6); // 2ms
+        }
     }
 
     R_SUCCEED();

sphaira/source/yati/yati.cpp

@@ -140,7 +140,13 @@ struct ThreadData {
         // this will be updated with the actual size from nca header.
         write_size = nca->size;
 
+        // reduce buffer size to preve
+        if (App::IsFileBaseEmummc()) {
+            read_buffer_size = 1024 * 512;
+        } else {
             read_buffer_size = 1024*1024*4;
+        }
+
         max_buffer_size = std::max(read_buffer_size, INFLATE_BUFFER_MAX);
     }
 
@@ -689,12 +695,27 @@ Result Yati::decompressFuncInternal(ThreadData* t) {
 Result Yati::writeFuncInternal(ThreadData* t) {
     std::vector<u8> buf;
     buf.reserve(t->max_buffer_size);
+    const auto is_file_based_emummc = App::IsFileBaseEmummc();
 
     while (t->write_offset < t->write_size && R_SUCCEEDED(t->GetResults())) {
         s64 dummy_off;
         R_TRY(t->GetWriteBuf(buf, dummy_off));
-        R_TRY(ncmContentStorageWritePlaceHolder(std::addressof(cs), std::addressof(t->nca->placeholder_id), t->write_offset, buf.data(), buf.size()));
+
-        t->write_offset += buf.size();
+        s64 off{};
+        while (off < buf.size() && t->write_offset < t->write_size && R_SUCCEEDED(t->GetResults())) {
+            const auto wsize = std::min<s64>(t->read_buffer_size, buf.size() - off);
+            R_TRY(ncmContentStorageWritePlaceHolder(std::addressof(cs), std::addressof(t->nca->placeholder_id), t->write_offset + off, buf.data() + off, wsize));
+
+            off += wsize;
+            t->write_offset += wsize;
+
+            // todo: check how much time elapsed and sleep the diff
+            // rather than always sleeping a fixed amount.
+            // ie, writing a small buffer (nca header) should not sleep the full 2 ms.
+            if (is_file_based_emummc) {
+                svcSleepThread(2e+6); // 2ms
+            }
+        }
     }
 
     log_write("finished write thread!\n");
@@ -834,15 +855,15 @@ Result Yati::InstallNcaInternal(std::span<TikCollection> tickets, NcaCollection&
     // #define WRITE_THREAD_CORE 2
 
     Thread t_read{};
-    R_TRY(threadCreate(&t_read, readFunc, std::addressof(t_data), nullptr, 1024*64, 0x20, READ_THREAD_CORE));
+    R_TRY(threadCreate(&t_read, readFunc, std::addressof(t_data), nullptr, 1024*64, PRIO_PREEMPTIVE, READ_THREAD_CORE));
     ON_SCOPE_EXIT(threadClose(&t_read));
 
     Thread t_decompress{};
-    R_TRY(threadCreate(&t_decompress, decompressFunc, std::addressof(t_data), nullptr, 1024*64, 0x20, DECOMPRESS_THREAD_CORE));
+    R_TRY(threadCreate(&t_decompress, decompressFunc, std::addressof(t_data), nullptr, 1024*64, PRIO_PREEMPTIVE, DECOMPRESS_THREAD_CORE));
     ON_SCOPE_EXIT(threadClose(&t_decompress));
 
     Thread t_write{};
-    R_TRY(threadCreate(&t_write, writeFunc, std::addressof(t_data), nullptr, 1024*64, 0x20, WRITE_THREAD_CORE));
+    R_TRY(threadCreate(&t_write, writeFunc, std::addressof(t_data), nullptr, 1024*64, PRIO_PREEMPTIVE, WRITE_THREAD_CORE));
     ON_SCOPE_EXIT(threadClose(&t_write));
 
     log_write("starting threads\n");