use nxtc (nx title cache) for caching titles in the game menu.

sphaira/CMakeLists.txt

@@ -197,6 +197,11 @@ FetchContent_Declare(libusbhsfs
     GIT_TAG db2bf2a
 )
 
+FetchContent_Declare(libnxtc
+    GIT_REPOSITORY https://github.com/DarkMatterCore/libnxtc.git
+    GIT_TAG v0.0.2
+)
+
 set(USE_NEW_ZSTD ON)
 
 set(ZSTD_BUILD_STATIC ON)
@@ -249,6 +254,7 @@ FetchContent_MakeAvailable(
     yyjson
     zstd
     libusbhsfs
+    libnxtc
 )
 
 set(FTPSRV_LIB_BUILD TRUE)
@@ -291,6 +297,13 @@ target_include_directories(ftpsrv_helper PUBLIC ${ftpsrv_SOURCE_DIR}/src/platfor
 add_library(stb INTERFACE)
 target_include_directories(stb INTERFACE ${stb_SOURCE_DIR})
 
+add_library(libnxtc
+    ${libnxtc_SOURCE_DIR}/source/nxtc.c
+    ${libnxtc_SOURCE_DIR}/source/nxtc_log.c
+    ${libnxtc_SOURCE_DIR}/source/nxtc_utils.c
+)
+target_include_directories(libnxtc PUBLIC ${libnxtc_SOURCE_DIR}/include)
+
 find_package(ZLIB REQUIRED)
 find_library(minizip_lib minizip REQUIRED)
 find_path(minizip_inc minizip REQUIRED)
@@ -320,6 +333,7 @@ target_link_libraries(sphaira PRIVATE
     stb
     yyjson
     # libusbhsfs
+    libnxtc
 
     ${minizip_lib}
     ZLIB::ZLIB

sphaira/include/ui/menus/game_menu.hpp

@@ -22,13 +22,12 @@ enum class NacpLoadStatus {
 
 struct Entry {
     u64 app_id{};
-    char display_version[0x10]{};
     NacpLanguageEntry lang{};
     int image{};
     bool selected{};
 
     std::shared_ptr<NsApplicationControlData> control{};
-    u64 control_size{};
+    u64 jpeg_size{};
     NacpLoadStatus status{NacpLoadStatus::None};
 
     auto GetName() const -> const char* {
@@ -38,35 +37,38 @@ struct Entry {
     auto GetAuthor() const -> const char* {
         return lang.author;
     }
-
-    auto GetDisplayVersion() const -> const char* {
-        return display_version;
-    }
 };
 
 struct ThreadResultData {
     u64 id{};
     std::shared_ptr<NsApplicationControlData> control{};
-    u64 control_size{};
+    u64 jpeg_size{};
-    char display_version[0x10]{};
     NacpLanguageEntry lang{};
     NacpLoadStatus status{NacpLoadStatus::None};
 };
 
 struct ThreadData {
-    ThreadData();
+    ThreadData(bool title_cache);
 
-    auto IsRunning() const -> bool;
     void Run();
     void Close();
     void Push(u64 id);
     void Push(std::span<const Entry> entries);
     void Pop(std::vector<ThreadResultData>& out);
 
+    auto IsRunning() const -> bool {
+        return m_running;
+    }
+
+    auto IsTitleCacheEnabled() const {
+        return m_title_cache;
+    }
+
 private:
     UEvent m_uevent{};
     Mutex m_mutex_id{};
     Mutex m_mutex_result{};
+    bool m_title_cache{};
 
     // app_ids pushed to the queue, signal uevent when pushed.
     std::vector<u64> m_ids{};
@@ -141,13 +143,14 @@ private:
     bool m_is_reversed{};
     bool m_dirty{};
 
-    ThreadData m_thread_data{};
+    std::unique_ptr<ThreadData> m_thread_data{};
     Thread m_thread{};
 
     option::OptionLong m_sort{INI_SECTION, "sort", SortType::SortType_Updated};
     option::OptionLong m_order{INI_SECTION, "order", OrderType::OrderType_Descending};
-    option::OptionLong m_layout{INI_SECTION, "layout", LayoutType::LayoutType_GridDetail};
+    option::OptionLong m_layout{INI_SECTION, "layout", LayoutType::LayoutType_Grid};
     option::OptionBool m_hide_forwarders{INI_SECTION, "hide_forwarders", false};
+    option::OptionBool m_title_cache{INI_SECTION, "title_cache", true};
 };
 
 } // namespace sphaira::ui::menu::game

sphaira/include/yati/nx/nca.hpp

@@ -221,7 +221,7 @@ Result VerifyFixedKey(const Header& header);
 
 // helpers that parse an nca.
 Result ParseCnmt(const fs::FsPath& path, u64 program_id, ncm::PackagedContentMeta& header, std::vector<u8>& extended_header, std::vector<NcmPackagedContentInfo>& infos);
-Result ParseControl(const fs::FsPath& path, u64 program_id, void* nacp_out = nullptr, s64 nacp_size = 0, std::vector<u8>* icon_out = nullptr);
+Result ParseControl(const fs::FsPath& path, u64 program_id, void* nacp_out = nullptr, s64 nacp_size = 0, std::vector<u8>* icon_out = nullptr, s64 nacp_off = 0);
 
 auto GetKeyGenStr(u8 key_gen) -> const char*;
 

sphaira/source/main.cpp

@@ -83,6 +83,10 @@ void userAppExit(void) {
     psmExit();
     plExit();
     socketExit();
+    // NOTE (DMC): prevents exfat corruption.
+    if (auto fs = fsdevGetDeviceFileSystem("sdmc:")) {
+        fsFsCommit(fs);
+    }
     appletUnlockExit();
 }
 

sphaira/source/ui/menus/game_menu.cpp

@@ -23,6 +23,7 @@
 #include <cstring>
 #include <algorithm>
 #include <minIni.h>
+#include <nxtc.h>
 
 namespace sphaira::ui::menu::game {
 namespace {
@@ -30,6 +31,34 @@ namespace {
 constexpr int THREAD_PRIO = PRIO_PREEMPTIVE;
 constexpr int THREAD_CORE = 1;
 
+// taken from nxtc
+constexpr u8 g_nacpLangTable[SetLanguage_Total] = {
+    [SetLanguage_JA]     =  2,
+    [SetLanguage_ENUS]   =  0,
+    [SetLanguage_FR]     =  3,
+    [SetLanguage_DE]     =  4,
+    [SetLanguage_IT]     =  7,
+    [SetLanguage_ES]     =  6,
+    [SetLanguage_ZHCN]   = 14,
+    [SetLanguage_KO]     = 12,
+    [SetLanguage_NL]     =  8,
+    [SetLanguage_PT]     = 10,
+    [SetLanguage_RU]     = 11,
+    [SetLanguage_ZHTW]   = 13,
+    [SetLanguage_ENGB]   =  1,
+    [SetLanguage_FRCA]   =  9,
+    [SetLanguage_ES419]  =  5,
+    [SetLanguage_ZHHANS] = 14,
+    [SetLanguage_ZHHANT] = 13,
+    [SetLanguage_PTBR]   = 15
+};
+
+auto GetNacpLangEntryIndex() -> u8 {
+    SetLanguage lang{SetLanguage_ENUS};
+    nxtcGetCacheLanguage(&lang);
+    return g_nacpLangTable[lang];
+}
+
 constexpr u32 ContentMetaTypeToContentFlag(u8 meta_type) {
     if (meta_type & 0x80) {
         return 1 << (meta_type - 0x80);
@@ -284,15 +313,13 @@ void FakeNacpEntry(ThreadResultData& e) {
     // fake the nacp entry
     std::strcpy(e.lang.name, "Corrupted");
     std::strcpy(e.lang.author, "Corrupted");
-    std::strcpy(e.display_version, "0.0.0");
     e.control.reset();
 }
 
 bool LoadControlImage(Entry& e) {
     if (!e.image && e.control) {
         TimeStamp ts;
-        const auto jpeg_size = e.control_size - sizeof(NacpStruct);
+        e.image = nvgCreateImageMem(App::GetVg(), 0, e.control->icon, e.jpeg_size);
-        e.image = nvgCreateImageMem(App::GetVg(), 0, e.control->icon, jpeg_size);
         e.control.reset();
         log_write("\t\t[image load] time taken: %.2fs %zums\n", ts.GetSecondsD(), ts.GetMs());
         return true;
@@ -301,14 +328,8 @@ bool LoadControlImage(Entry& e) {
     return false;
 }
 
-Result LoadControlManual(u64 id, ThreadResultData& data) {
+Result GetControlPathFromStatus(const NsApplicationContentMetaStatus& status, u64* out_program_id, fs::FsPath* out_path) {
-    TimeStamp ts;
+    const auto& ee = status;
-
-    MetaEntries entries;
-    R_TRY(GetMetaEntries(id, entries));
-    R_UNLESS(!entries.empty(), 0x1);
-
-    const auto& ee = entries.back();
     if (ee.storageID != NcmStorageId_SdCard && ee.storageID != NcmStorageId_BuiltInUser && ee.storageID != NcmStorageId_GameCard) {
         return 0x1;
     }
@@ -322,17 +343,28 @@ Result LoadControlManual(u64 id, ThreadResultData& data) {
     NcmContentId content_id;
     R_TRY(ncmContentMetaDatabaseGetContentIdByType(&db, &content_id, &key, NcmContentType_Control));
 
-    u64 program_id;
+    R_TRY(ncmContentStorageGetProgramId(&cs, out_program_id, &content_id, FsContentAttributes_All));
-    R_TRY(ncmContentStorageGetProgramId(&cs, &program_id, &content_id, FsContentAttributes_All));
 
+    R_TRY(ncmContentStorageGetPath(&cs, out_path->s, sizeof(*out_path), &content_id));
+    R_SUCCEED();
+}
+
+Result LoadControlManual(u64 id, ThreadResultData& data) {
+    TimeStamp ts;
+
+    MetaEntries entries;
+    R_TRY(GetMetaEntries(id, entries));
+    R_UNLESS(!entries.empty(), 0x1);
+
+    u64 program_id;
     fs::FsPath path;
-    R_TRY(ncmContentStorageGetPath(&cs, path, sizeof(path), &content_id));
+    R_TRY(GetControlPathFromStatus(entries.back(), &program_id, &path));
 
     std::vector<u8> icon;
-    R_TRY(nca::ParseControl(path, program_id, &data.control->nacp, sizeof(data.control->nacp), &icon));
+    R_TRY(nca::ParseControl(path, program_id, &data.control->nacp.lang[GetNacpLangEntryIndex()], sizeof(NacpLanguageEntry), &icon));
     std::memcpy(data.control->icon, icon.data(), icon.size());
 
-    data.control_size = sizeof(data.control->nacp) + icon.size();
+    data.jpeg_size = icon.size();
     log_write("\t\t[manual control] time taken: %.2fs %zums\n", ts.GetSecondsD(), ts.GetMs());
 
     R_SUCCEED();
@@ -347,8 +379,10 @@ auto LoadControlEntry(u64 id) -> ThreadResultData {
     bool manual_load = true;
     if (hosversionBefore(20,0,0)) {
         TimeStamp ts;
-        if (R_SUCCEEDED(nsGetApplicationControlData(NsApplicationControlSource_CacheOnly, id, data.control.get(), sizeof(NsApplicationControlData), &data.control_size))) {
+        u64 actual_size;
+        if (R_SUCCEEDED(nsGetApplicationControlData(NsApplicationControlSource_CacheOnly, id, data.control.get(), sizeof(NsApplicationControlData), &actual_size))) {
             manual_load = false;
+            data.jpeg_size = actual_size - sizeof(NacpStruct);
             log_write("\t\t[ns control cache] time taken: %.2fs %zums\n", ts.GetSecondsD(), ts.GetMs());
         }
     }
@@ -360,18 +394,17 @@ auto LoadControlEntry(u64 id) -> ThreadResultData {
     Result rc{};
     if (!manual_load) {
         TimeStamp ts;
-        rc = nsGetApplicationControlData(NsApplicationControlSource_Storage, id, data.control.get(), sizeof(NsApplicationControlData), &data.control_size);
+        u64 actual_size;
+        if (R_SUCCEEDED(rc = nsGetApplicationControlData(NsApplicationControlSource_Storage, id, data.control.get(), sizeof(NsApplicationControlData), &actual_size))) {
+            data.jpeg_size = actual_size - sizeof(NacpStruct);
             log_write("\t\t[ns control storage] time taken: %.2fs %zums\n", ts.GetSecondsD(), ts.GetMs());
         }
+    }
 
     if (R_SUCCEEDED(rc)) {
-        NacpLanguageEntry* lang{};
+        data.lang = data.control->nacp.lang[GetNacpLangEntryIndex()];
-        if (R_SUCCEEDED(rc = nsGetApplicationDesiredLanguage(&data.control->nacp, &lang)) && lang) {
-            data.lang = *lang;
-            std::memcpy(data.display_version, data.control->nacp.display_version, sizeof(data.display_version));
         data.status = NacpLoadStatus::Loaded;
     }
-    }
 
     if (R_FAILED(rc)) {
         FakeNacpEntry(data);
@@ -383,8 +416,7 @@ auto LoadControlEntry(u64 id) -> ThreadResultData {
 void LoadResultIntoEntry(Entry& e, const ThreadResultData& result) {
     e.status = result.status;
     e.control = result.control;
-    e.control_size = result.control_size;
+    e.jpeg_size= result.jpeg_size;
-    std::memcpy(e.display_version, result.display_version, sizeof(result.display_version));
     e.lang = result.lang;
     e.status = result.status;
 }
@@ -462,8 +494,16 @@ auto BuildNspPath(const Entry& e, const NsApplicationContentMetaStatus& status)
     utilsReplaceIllegalCharacters(name_buf, true);
 
     char version[sizeof(NacpStruct::display_version) + 1]{};
+    // status.storageID
     if (status.meta_type == NcmContentMetaType_Patch) {
-        std::snprintf(version, sizeof(version), "%s ", e.GetDisplayVersion());
+        u64 program_id;
+        fs::FsPath path;
+        if (R_SUCCEEDED(GetControlPathFromStatus(status, &program_id, &path))) {
+            char display_version[0x10];
+            if (R_SUCCEEDED(nca::ParseControl(path, program_id, display_version, sizeof(display_version), nullptr, offsetof(NacpStruct, display_version)))) {
+                std::snprintf(version, sizeof(version), "%s ", display_version);
+            }
+        }
     }
 
     fs::FsPath path;
@@ -619,6 +659,11 @@ void LaunchEntry(const Entry& e) {
 void ThreadFunc(void* user) {
     auto data = static_cast<ThreadData*>(user);
 
+    if (data->IsTitleCacheEnabled() && !nxtcInitialize()) {
+        log_write("[NXTC] failed to init cache\n");
+    }
+    ON_SCOPE_EXIT(nxtcExit());
+
     while (data->IsRunning()) {
         data->Run();
     }
@@ -626,21 +671,23 @@ void ThreadFunc(void* user) {
 
 } // namespace
 
-ThreadData::ThreadData() {
+ThreadData::ThreadData(bool title_cache) : m_title_cache{title_cache} {
     ueventCreate(&m_uevent, true);
     mutexInit(&m_mutex_id);
     mutexInit(&m_mutex_result);
     m_running = true;
 }
 
-auto ThreadData::IsRunning() const -> bool {
-    return m_running;
-}
-
 void ThreadData::Run() {
-    while (IsRunning()) {
     const auto waiter = waiterForUEvent(&m_uevent);
-        waitSingle(waiter, UINT64_MAX);
+    while (IsRunning()) {
+        const auto rc = waitSingle(waiter, 3e+9);
+
+        // if we timed out, flush the cache and poll again.
+        if (R_FAILED(rc)) {
+            nxtcFlushCacheFile();
+            continue;
+        }
 
         if (!IsRunning()) {
             return;
@@ -658,10 +705,28 @@ void ThreadData::Run() {
                 return;
             }
 
+            ThreadResultData result{ids[i]};
+            TimeStamp ts;
+            if (auto data = nxtcGetApplicationMetadataEntryById(ids[i])) {
+                log_write("[NXTC] loaded from cache time taken: %.2fs %zums %zuns\n", ts.GetSecondsD(), ts.GetMs(), ts.GetNs());
+                ON_SCOPE_EXIT(nxtcFreeApplicationMetadata(&data));
+
+                result.control = std::make_unique<NsApplicationControlData>();
+                result.status = NacpLoadStatus::Loaded;
+                std::strcpy(result.lang.name, data->name);
+                std::strcpy(result.lang.author, data->publisher);
+                std::memcpy(result.control->icon, data->icon_data, data->icon_size);
+                result.jpeg_size = data->icon_size;
+            } else {
                 // sleep after every other entry loaded.
                 svcSleepThread(2e+6); // 2ms
 
-            const auto result = LoadControlEntry(ids[i]);
+                result = LoadControlEntry(ids[i]);
+                if (result.status == NacpLoadStatus::Loaded) {
+                    nxtcAddEntry(ids[i], &result.control->nacp, result.jpeg_size, result.control->icon, true);
+                }
+            }
+
             mutexLock(&m_mutex_result);
             ON_SCOPE_EXIT(mutexUnlock(&m_mutex_result));
             m_result.emplace_back(result);
@@ -871,6 +936,10 @@ Menu::Menu(u32 flags) : grid::Menu{"Games"_i18n, flags} {
                     ));
                 }, true));
             }
+
+            options->Add(std::make_shared<SidebarEntryBool>("Title cache"_i18n, m_title_cache.Get(), [this](bool& v_out){
+                m_title_cache.Set(v_out);
+            }));
         }})
     );
 
@@ -883,13 +952,14 @@ Menu::Menu(u32 flags) : grid::Menu{"Games"_i18n, flags} {
         e.Open();
     }
 
-    threadCreate(&m_thread, ThreadFunc, &m_thread_data, nullptr, 1024*32, THREAD_PRIO, THREAD_CORE);
+    m_thread_data = std::make_unique<ThreadData>(m_title_cache.Get());
+    threadCreate(&m_thread, ThreadFunc, m_thread_data.get(), nullptr, 1024*32, THREAD_PRIO, THREAD_CORE);
     svcSetThreadCoreMask(m_thread.handle, THREAD_CORE, THREAD_AFFINITY_DEFAULT(THREAD_CORE));
     threadStart(&m_thread);
 }
 
 Menu::~Menu() {
-    m_thread_data.Close();
+    m_thread_data->Close();
 
     for (auto& e : ncm_entries) {
         e.Close();
@@ -928,7 +998,7 @@ void Menu::Draw(NVGcontext* vg, Theme* theme) {
     int image_load_count = 0;
 
     std::vector<ThreadResultData> data;
-    m_thread_data.Pop(data);
+    m_thread_data->Pop(data);
 
     for (const auto& d : data) {
         const auto it = std::ranges::find_if(m_entries, [&d](auto& e) {
@@ -945,7 +1015,7 @@ void Menu::Draw(NVGcontext* vg, Theme* theme) {
         auto& e = m_entries[pos];
 
         if (e.status == NacpLoadStatus::None) {
-            m_thread_data.Push(e.app_id);
+            m_thread_data->Push(e.app_id);
             e.status = NacpLoadStatus::Progress;
         }
 
@@ -956,8 +1026,11 @@ void Menu::Draw(NVGcontext* vg, Theme* theme) {
             }
         }
 
+        char title_id[33];
+        std::snprintf(title_id, sizeof(title_id), "%016lX", e.app_id);
+
         const auto selected = pos == m_index;
-        DrawEntry(vg, theme, m_layout.Get(), v, selected, e.image, e.GetName(), e.GetAuthor(), e.GetDisplayVersion());
+        DrawEntry(vg, theme, m_layout.Get(), v, selected, e.image, e.GetName(), e.GetAuthor(), title_id);
 
         if (e.selected) {
             gfx::drawRect(vg, v, theme->GetColour(ThemeEntryID_FOCUS), 5);

sphaira/source/yati/nx/nca.cpp

@@ -186,7 +186,7 @@ Result ParseCnmt(const fs::FsPath& path, u64 program_id, ncm::PackagedContentMet
     R_SUCCEED();
 }
 
-Result ParseControl(const fs::FsPath& path, u64 program_id, void* nacp_out, s64 nacp_size, std::vector<u8>* icon_out) {
+Result ParseControl(const fs::FsPath& path, u64 program_id, void* nacp_out, s64 nacp_size, std::vector<u8>* icon_out, s64 nacp_off) {
     FsFileSystem fs;
     R_TRY(fsOpenFileSystemWithId(std::addressof(fs), program_id, FsFileSystemType_ContentControl, path, FsContentAttributes_All));
     ON_SCOPE_EXIT(fsFsClose(std::addressof(fs)));
@@ -198,7 +198,7 @@ Result ParseControl(const fs::FsPath& path, u64 program_id, void* nacp_out, s64
         ON_SCOPE_EXIT(fsFileClose(std::addressof(file)));
 
         u64 bytes_read;
-        R_TRY(fsFileRead(&file, 0, nacp_out, nacp_size, 0, &bytes_read));
+        R_TRY(fsFileRead(&file, nacp_off, nacp_out, nacp_size, 0, &bytes_read));
     }
 
     // read icon.