From 73886c28ae060804d1a060df14141627a1245b5b Mon Sep 17 00:00:00 2001 From: ITotalJustice <47043333+ITotalJustice@users.noreply.github.com> Date: Tue, 29 Apr 2025 12:47:38 +0100 Subject: [PATCH] add gc event waiting, fix control nca mounting, better skip nca support. - gamecards now wait for an event to change, rather than polling each frame. this reduces cpu load on core 3 slightly (3-4% less). - my understanding of fsOpenFileSystemWithId() was wrong, i thought it used the app_id for the id param. turns out it needs the program id (found in the nca header), this is why mounting some control ncas would fail. fs (and ncm) have a call in 17+ to get the program id, it does so by parsing the nca header. in yati, we already have the header so we can avoid the call. for the gamecard menu, we don't. so we can parse the nca header, or use the id offset (which we already have) to form the program id. - std::find_if in yati now takes args by ref rather than by value, avoid quite large copies. - stream installs can now parse the control nca. - if an nca is already installed, it is now skipped. this is regardless to whether it is not in ncm db. - nca skipping is technically supported for stream installs, however it is disabled for now as there needs to be a way to allow for the stream to continue reading and discarding data until the stream has finished. currently, if a ftp (stream) install is skipped, it will close the progress box and cause spahira to hang. this is because sphaira expects the stream to only be closed upon all data being read, so there's nothing more to process. - renamed the title_id field in nca header to program_id. --- sphaira/include/ui/menus/gc_menu.hpp | 7 +- sphaira/include/yati/nx/nca.hpp | 2 +- sphaira/include/yati/yati.hpp | 4 +- sphaira/source/owo.cpp | 2 +- sphaira/source/ui/menus/gc_menu.cpp | 67 ++++++++++---- sphaira/source/yati/yati.cpp | 134 ++++++++++++++++----------- 6 files changed, 139 insertions(+), 77 deletions(-) diff --git a/sphaira/include/ui/menus/gc_menu.hpp b/sphaira/include/ui/menus/gc_menu.hpp index 279dd00..9170d39 100644 --- a/sphaira/include/ui/menus/gc_menu.hpp +++ b/sphaira/include/ui/menus/gc_menu.hpp @@ -10,14 +10,16 @@ namespace sphaira::ui::menu::gc { struct GcCollection : yati::container::CollectionEntry { - GcCollection(const char* _name, s64 _size, u8 _type) { + GcCollection(const char* _name, s64 _size, u8 _type, u8 _id_offset) { name = _name; size = _size; type = _type; + id_offset = _id_offset; } // NcmContentType u8 type{}; + u8 id_offset{}; }; using GcCollections = std::vector; @@ -48,6 +50,7 @@ private: Result GcMount(); void GcUnmount(); Result GcPoll(bool* inserted); + Result GcOnEvent(); Result UpdateStorageSize(); void FreeImage(); @@ -57,6 +60,8 @@ private: FsDeviceOperator m_dev_op{}; FsGameCardHandle m_handle{}; std::unique_ptr m_fs{}; + FsEventNotifier m_event_notifier{}; + Event m_event{}; std::vector m_entries{}; std::unique_ptr m_list{}; diff --git a/sphaira/include/yati/nx/nca.hpp b/sphaira/include/yati/nx/nca.hpp index cc4e086..80d0a04 100644 --- a/sphaira/include/yati/nx/nca.hpp +++ b/sphaira/include/yati/nx/nca.hpp @@ -175,7 +175,7 @@ struct Header { u8 old_key_gen; // see KeyGenerationOld. u8 kaek_index; // see KeyAreaEncryptionKeyIndex. u64 size; - u64 title_id; + u64 program_id; u32 context_id; u32 sdk_version; u8 key_gen; // see KeyGeneration. diff --git a/sphaira/include/yati/yati.hpp b/sphaira/include/yati/yati.hpp index d448fb4..c582d8a 100644 --- a/sphaira/include/yati/yati.hpp +++ b/sphaira/include/yati/yati.hpp @@ -132,7 +132,7 @@ Result InstallFromSource(ui::ProgressBox* pbox, std::shared_ptr so Result InstallFromContainer(ui::ProgressBox* pbox, std::shared_ptr container, const ConfigOverride& override = {}); Result InstallFromCollections(ui::ProgressBox* pbox, std::shared_ptr source, const container::Collections& collections, const ConfigOverride& override = {}); -Result ParseCnmtNca(const fs::FsPath& path, ncm::PackagedContentMeta& header, std::vector& extended_header, std::vector& infos); -Result ParseControlNca(const fs::FsPath& path, u64 id, void* nacp_out = nullptr, s64 nacp_size = 0, std::vector* icon_out = nullptr); +Result ParseCnmtNca(const fs::FsPath& path, u64 program_id, ncm::PackagedContentMeta& header, std::vector& extended_header, std::vector& infos); +Result ParseControlNca(const fs::FsPath& path, u64 program_id, void* nacp_out = nullptr, s64 nacp_size = 0, std::vector* icon_out = nullptr); } // namespace sphaira::yati diff --git a/sphaira/source/owo.cpp b/sphaira/source/owo.cpp index 1a4c012..3e0dcfc 100644 --- a/sphaira/source/owo.cpp +++ b/sphaira/source/owo.cpp @@ -715,7 +715,7 @@ void write_nca_header_encypted(nca::Header& nca_header, u64 tid, const keys::Key nca_header.magic = NCA3_MAGIC; nca_header.distribution_type = nca::DistributionType_System; nca_header.content_type = type; - nca_header.title_id = tid; + nca_header.program_id = tid; nca_header.sdk_version = 0x000C1100; nca_header.size = buf.tell(); diff --git a/sphaira/source/ui/menus/gc_menu.cpp b/sphaira/source/ui/menus/gc_menu.cpp index 53dc2ea..5e6ec65 100644 --- a/sphaira/source/ui/menus/gc_menu.cpp +++ b/sphaira/source/ui/menus/gc_menu.cpp @@ -44,6 +44,13 @@ auto BuildGcPath(const char* name, const FsGameCardHandle* handle, FsGameCardPar return path; } +Result fsOpenGameCardDetectionEventNotifier(FsEventNotifier* out) { + return serviceDispatch(fsGetServiceSession(), 501, + .out_num_objects = 1, + .out_objects = &out->s + ); +} + auto InRange(u64 off, u64 offset, u64 size) -> bool { return off < offset + size && off >= offset; } @@ -175,29 +182,24 @@ Menu::Menu() : MenuBase{"GameCard"_i18n} { m_list = std::make_unique(1, 3, m_pos, v, pad); fsOpenDeviceOperator(std::addressof(m_dev_op)); + fsOpenGameCardDetectionEventNotifier(std::addressof(m_event_notifier)); + fsEventNotifierGetEventHandle(std::addressof(m_event_notifier), std::addressof(m_event), true); + GcOnEvent(); UpdateStorageSize(); } Menu::~Menu() { GcUnmount(); + eventClose(std::addressof(m_event)); + fsEventNotifierClose(std::addressof(m_event_notifier)); fsDeviceOperatorClose(std::addressof(m_dev_op)); } void Menu::Update(Controller* controller, TouchInfo* touch) { // poll for the gamecard first before handling inputs as the gamecard // may have been removed, thus pressing A would fail. - bool inserted{}; - GcPoll(&inserted); - if (m_mounted != inserted) { - log_write("gc state changed\n"); - m_mounted = inserted; - if (m_mounted) { - log_write("trying to mount\n"); - m_mounted = R_SUCCEEDED(GcMount()); - } else { - log_write("trying to unmount\n"); - GcUnmount(); - } + if (R_SUCCEEDED(eventWait(std::addressof(m_event), 0))) { + GcOnEvent(); } MenuBase::Update(controller, touch); @@ -312,7 +314,7 @@ Result Menu::GcMount() { std::vector extended_header; std::vector infos; const auto path = BuildGcPath(e.name, &m_handle); - R_TRY(yati::ParseCnmtNca(path, header, extended_header, infos)); + R_TRY(yati::ParseCnmtNca(path, 0, header, extended_header, infos)); u8 key_gen; FsRightsId rights_id; @@ -321,23 +323,24 @@ Result Menu::GcMount() { // always add tickets, yati will ignore them if not needed. GcCollections collections; // add cnmt file. - collections.emplace_back(e.name, e.file_size, NcmContentType_Meta); + collections.emplace_back(e.name, e.file_size, NcmContentType_Meta, 0); - for (const auto& info : infos) { + for (const auto& packed_info : infos) { + const auto& info = packed_info.info; // these don't exist for gamecards, however i may copy/paste this code // somewhere so i'm future proofing against myself. - if (info.info.content_type == NcmContentType_DeltaFragment) { + if (info.content_type == NcmContentType_DeltaFragment) { continue; } // find the nca file, this will never fail for gamecards, see above comment. - const auto str = hexIdToStr(info.info.content_id); + const auto str = hexIdToStr(info.content_id); const auto it = std::find_if(buf.cbegin(), buf.cend(), [str](auto& e){ return !std::strncmp(str.str, e.name, std::strlen(str.str)); }); R_UNLESS(it != buf.cend(), yati::Result_NcaNotFound); - collections.emplace_back(it->name, it->file_size, info.info.content_type); + collections.emplace_back(it->name, it->file_size, info.content_type, info.id_offset); } const auto app_id = ncm::GetAppId(header); @@ -409,6 +412,7 @@ Result Menu::GcMount() { } OnChangeIndex(0); + m_mounted = true; R_SUCCEED(); } @@ -439,6 +443,25 @@ Result Menu::GcPoll(bool* inserted) { R_SUCCEED(); } +Result Menu::GcOnEvent() { + bool inserted{}; + R_TRY(GcPoll(&inserted)); + + if (m_mounted != inserted) { + log_write("gc state changed\n"); + m_mounted = inserted; + if (m_mounted) { + log_write("trying to mount\n"); + m_mounted = R_SUCCEEDED(GcMount()); + } else { + log_write("trying to unmount\n"); + GcUnmount(); + } + } + + R_SUCCEED(); +} + Result Menu::UpdateStorageSize() { fs::FsNativeContentStorage fs_nand{FsContentStorageId_User}; fs::FsNativeContentStorage fs_sd{FsContentStorageId_SdCard}; @@ -475,7 +498,13 @@ void Menu::OnChangeIndex(s64 new_index) { NacpStruct nacp; std::vector icon; const auto path = BuildGcPath(collection.name.c_str(), &m_handle); - if (R_SUCCEEDED(yati::ParseControlNca(path, m_entries[m_entry_index].app_id, &nacp, sizeof(nacp), &icon))) { + + u64 program_id = m_entries[m_entry_index].app_id | collection.id_offset; + if (hosversionAtLeast(17, 0, 0)) { + fsGetProgramId(&program_id, path, FsContentAttributes_All); + } + + if (R_SUCCEEDED(yati::ParseControlNca(path, program_id, &nacp, sizeof(nacp), &icon))) { log_write("managed to parse control nca %s\n", path.s); NacpLanguageEntry* lang_entry{}; nacpGetLanguageEntry(&nacp, &lang_entry); diff --git a/sphaira/source/yati/yati.cpp b/sphaira/source/yati/yati.cpp index 4ea3dba..1a8e892 100644 --- a/sphaira/source/yati/yati.cpp +++ b/sphaira/source/yati/yati.cpp @@ -64,6 +64,7 @@ using PageAlignedVector = std::vector>; constexpr u32 KEYGEN_LIMIT = 0x20; struct NcaCollection : container::CollectionEntry { + nca::Header header{}; // NcmContentType u8 type{}; NcmContentId content_id{}; @@ -72,6 +73,8 @@ struct NcaCollection : container::CollectionEntry { u8 hash[SHA256_HASH_SIZE]{}; // set true if nca has been modified. bool modified{}; + // set if the nca was not installed. + bool skipped{}; }; struct CnmtCollection : NcaCollection { @@ -81,7 +84,7 @@ struct CnmtCollection : NcaCollection { // if set, the ticket / cert will be installed once all nca's have installed. std::vector rights_id{}; - NcmContentMetaHeader header{}; + NcmContentMetaHeader meta_header{}; NcmContentMetaKey key{}; NcmContentInfo content_info{}; std::vector extended_header{}; @@ -276,8 +279,8 @@ struct Yati { Result Setup(const ConfigOverride& override); Result InstallNca(std::span tickets, NcaCollection& nca); + Result InstallNcaInternal(std::span tickets, NcaCollection& nca); Result InstallCnmtNca(std::span tickets, CnmtCollection& cnmt, const container::Collections& collections); - Result InstallControlNca(std::span tickets, const CnmtCollection& cnmt, NcaCollection& nca); Result readFuncInternal(ThreadData* t); Result decompressFuncInternal(ThreadData* t); @@ -538,6 +541,9 @@ Result Yati::decompressFuncInternal(ThreadData* t) { R_UNLESS(header.magic == 0x3341434E, Result_InvalidNcaMagic); log_write("nca magic is ok! type: %u\n", header.content_type); + // store the unmodified header. + t->nca->header = header; + if (!config.skip_rsa_header_fixed_key_verify) { log_write("verifying nca fixed key\n"); R_TRY(nca::VerifyFixedKey(header)); @@ -556,7 +562,7 @@ Result Yati::decompressFuncInternal(ThreadData* t) { TikCollection* ticket = nullptr; if (isRightsIdValid(header.rights_id)) { - auto it = std::find_if(t->tik.begin(), t->tik.end(), [header](auto& e){ + auto it = std::find_if(t->tik.begin(), t->tik.end(), [&header](auto& e){ return !std::memcmp(&header.rights_id, &e.rights_id, sizeof(e.rights_id)); }); @@ -829,10 +835,17 @@ Result Yati::Setup(const ConfigOverride& override) { R_SUCCEED(); } -Result Yati::InstallNca(std::span tickets, NcaCollection& nca) { - log_write("in install nca\n"); - pbox->NewTransfer(nca.name); - keys::parse_hex_key(std::addressof(nca.content_id), nca.name.c_str()); +Result Yati::InstallNcaInternal(std::span tickets, NcaCollection& nca) { + if (config.skip_if_already_installed) { + R_TRY(ncmContentStorageHas(std::addressof(cs), std::addressof(nca.skipped), std::addressof(nca.content_id))); + if (nca.skipped) { + log_write("\tskipped nca as it's already installed ncmContentStorageHas()\n"); + R_TRY(ncmContentStorageReadContentIdFile(std::addressof(cs), std::addressof(nca.header), sizeof(nca.header), std::addressof(nca.content_id), 0)); + crypto::cryptoAes128Xts(std::addressof(nca.header), std::addressof(nca.header), keys.header_key, 0, 0x200, sizeof(nca.header), false); + R_SUCCEED(); + } + } + log_write("generateing placeholder\n"); R_TRY(ncmContentStorageGeneratePlaceHolderId(std::addressof(cs), std::addressof(nca.placeholder_id))); log_write("creating placeholder\n"); @@ -918,24 +931,56 @@ Result Yati::InstallNca(std::span tickets, NcaCollection& nca) { R_SUCCEED(); } +Result Yati::InstallNca(std::span tickets, NcaCollection& nca) { + log_write("in install nca\n"); + pbox->NewTransfer(nca.name); + keys::parse_hex_key(std::addressof(nca.content_id), nca.name.c_str()); + + R_TRY(InstallNcaInternal(tickets, nca)); + + fs::FsPath path; + if (nca.skipped) { + R_TRY(ncmContentStorageGetPath(std::addressof(cs), path, sizeof(path), std::addressof(nca.content_id))); + } else { + R_TRY(ncmContentStorageFlushPlaceHolder(std::addressof(cs))); + R_TRY(ncmContentStorageGetPlaceHolderPath(std::addressof(cs), path, sizeof(path), std::addressof(nca.placeholder_id))); + } + + if (nca.header.content_type == nca::ContentType_Program) { + // todo: verify npdm key. + } else if (nca.header.content_type == nca::ContentType_Control) { + NacpLanguageEntry entry; + std::vector icon; + R_TRY(yati::ParseControlNca(path, nca.header.program_id, &entry, sizeof(entry), &icon)); + pbox->SetTitle(entry.name).SetImageData(icon); + } + + R_SUCCEED(); +} + Result Yati::InstallCnmtNca(std::span tickets, CnmtCollection& cnmt, const container::Collections& collections) { R_TRY(InstallNca(tickets, cnmt)); fs::FsPath path; - R_TRY(ncmContentStorageFlushPlaceHolder(std::addressof(cs))); - R_TRY(ncmContentStorageGetPlaceHolderPath(std::addressof(cs), path, sizeof(path), std::addressof(cnmt.placeholder_id))); + if (cnmt.skipped) { + R_TRY(ncmContentStorageGetPath(std::addressof(cs), path, sizeof(path), std::addressof(cnmt.content_id))); + } else { + R_TRY(ncmContentStorageFlushPlaceHolder(std::addressof(cs))); + R_TRY(ncmContentStorageGetPlaceHolderPath(std::addressof(cs), path, sizeof(path), std::addressof(cnmt.placeholder_id))); + } ncm::PackagedContentMeta header; std::vector infos; - R_TRY(ParseCnmtNca(path, header, cnmt.extended_header, infos)); + R_TRY(ParseCnmtNca(path, cnmt.header.program_id, header, cnmt.extended_header, infos)); - for (const auto& info : infos) { - if (info.info.content_type == NcmContentType_DeltaFragment) { + for (const auto& packed_info : infos) { + const auto& info = packed_info.info; + if (info.content_type == NcmContentType_DeltaFragment) { continue; } - const auto str = hexIdToStr(info.info.content_id); - const auto it = std::find_if(collections.cbegin(), collections.cend(), [str](auto& e){ + const auto str = hexIdToStr(info.content_id); + const auto it = std::find_if(collections.cbegin(), collections.cend(), [&str](auto& e){ return e.name.find(str.str) != e.name.npos; }); @@ -944,13 +989,13 @@ Result Yati::InstallCnmtNca(std::span tickets, CnmtCollection& cn log_write("found: %s\n", str.str); cnmt.infos.emplace_back(info); auto& nca = cnmt.ncas.emplace_back(*it); - nca.type = info.info.content_type; + nca.type = info.content_type; } // update header - cnmt.header = header.meta_header; - cnmt.header.content_count = cnmt.infos.size() + 1; - cnmt.header.storage_id = 0; + cnmt.meta_header = header.meta_header; + cnmt.meta_header.content_count = cnmt.infos.size() + 1; + cnmt.meta_header.storage_id = 0; cnmt.key.id = header.title_id; cnmt.key.version = header.title_version; @@ -985,26 +1030,6 @@ Result Yati::InstallCnmtNca(std::span tickets, CnmtCollection& cn R_SUCCEED(); } -Result Yati::InstallControlNca(std::span tickets, const CnmtCollection& cnmt, NcaCollection& nca) { - R_TRY(InstallNca(tickets, nca)); - - fs::FsPath path; - R_TRY(ncmContentStorageFlushPlaceHolder(std::addressof(cs))); - R_TRY(ncmContentStorageGetPlaceHolderPath(std::addressof(cs), path, sizeof(path), std::addressof(nca.placeholder_id))); - - // this can fail if it's not a valid control nca, examples are mario 3d all stars. - // there are 4 control ncas, only 1 is valid (InvalidNcaId 0x235E02). - NacpLanguageEntry entry; - std::vector icon; - if (R_SUCCEEDED(yati::ParseControlNca(path, ncm::GetAppId(cnmt.key), &entry, sizeof(entry), &icon))) { - pbox->SetTitle(entry.name).SetImageData(icon); - } else { - log_write("\tWARNING: failed to parse control nca!\n"); - } - - R_SUCCEED(); -} - Result Yati::ParseTicketsIntoCollection(std::vector& tickets, const container::Collections& collections, bool read_data) { for (const auto& collection : collections) { if (collection.name.ends_with(".tik")) { @@ -1012,7 +1037,7 @@ Result Yati::ParseTicketsIntoCollection(std::vector& tickets, con keys::parse_hex_key(entry.rights_id.c, collection.name.c_str()); const auto str = collection.name.substr(0, collection.name.length() - 4) + ".cert"; - const auto cert = std::find_if(collections.cbegin(), collections.cend(), [str](auto& e){ + const auto cert = std::find_if(collections.cbegin(), collections.cend(), [&str](auto& e){ return e.name.find(str) != e.name.npos; }); @@ -1091,6 +1116,13 @@ Result Yati::ShouldSkip(const CnmtCollection& cnmt, bool& skip) { } else if (config.skip_data_patch && cnmt.key.type == NcmContentMetaType_DataPatch) { log_write("\tskipping: [NcmContentMetaType_DataPatch]\n"); skip = true; + } else if (config.skip_if_already_installed) { + bool has; + R_TRY(ncmContentMetaDatabaseHas(std::addressof(db), std::addressof(has), std::addressof(cnmt.key))); + if (has) { + log_write("\tskipping: [ncmContentMetaDatabaseHas()]\n"); + skip = true; + } } R_SUCCEED(); @@ -1183,7 +1215,7 @@ Result Yati::RegisterNcasAndPushRecord(const CnmtCollection& cnmt, u32 latest_ve log_write("registered cnmt nca\n"); for (auto& nca : cnmt.ncas) { - if (nca.type != NcmContentType_DeltaFragment) { + if (!nca.skipped && nca.type != NcmContentType_DeltaFragment) { log_write("registering nca: %s\n", nca.name.c_str()); R_TRY(ncm::Register(std::addressof(cs), std::addressof(nca.content_id), std::addressof(nca.placeholder_id))); log_write("registered nca: %s\n", nca.name.c_str()); @@ -1194,7 +1226,7 @@ Result Yati::RegisterNcasAndPushRecord(const CnmtCollection& cnmt, u32 latest_ve // build ncm meta and push to the database. BufHelper buf{}; - buf.write(std::addressof(cnmt.header), sizeof(cnmt.header)); + buf.write(std::addressof(cnmt.meta_header), sizeof(cnmt.meta_header)); buf.write(cnmt.extended_header.data(), cnmt.extended_header.size()); buf.write(std::addressof(cnmt.content_info), sizeof(cnmt.content_info)); @@ -1262,12 +1294,7 @@ Result InstallInternal(ui::ProgressBox* pbox, std::shared_ptr sour log_write("installing nca's\n"); for (auto& nca : cnmt.ncas) { - if (nca.type == NcmContentType_Control) { - log_write("installing control nca\n"); - R_TRY(yati->InstallControlNca(tickets, cnmt, nca)); - } else { - R_TRY(yati->InstallNca(tickets, nca)); - } + R_TRY(yati->InstallNca(tickets, nca)); } R_TRY(yati->ImportTickets(tickets)); @@ -1284,6 +1311,7 @@ Result InstallInternalStream(ui::ProgressBox* pbox, std::shared_ptrSetup(override)); // not supported with stream installs (yet). + yati->config.skip_if_already_installed = false; yati->config.convert_to_standard_crypto = false; yati->config.lower_master_key = false; @@ -1326,7 +1354,7 @@ Result InstallInternalStream(ui::ProgressBox* pbox, std::shared_ptr& extended_header, std::vector& infos) { +Result ParseCnmtNca(const fs::FsPath& path, u64 program_id, ncm::PackagedContentMeta& header, std::vector& extended_header, std::vector& infos) { FsFileSystem fs; - R_TRY(fsOpenFileSystem(std::addressof(fs), FsFileSystemType_ContentMeta, path)); + R_TRY(fsOpenFileSystemWithId(std::addressof(fs), program_id, FsFileSystemType_ContentMeta, path, FsContentAttributes_All)); ON_SCOPE_EXIT(fsFsClose(std::addressof(fs))); FsDir dir; @@ -1447,9 +1475,9 @@ Result ParseCnmtNca(const fs::FsPath& path, ncm::PackagedContentMeta& header, st R_SUCCEED(); } -Result ParseControlNca(const fs::FsPath& path, u64 id, void* nacp_out, s64 nacp_size, std::vector* icon_out) { +Result ParseControlNca(const fs::FsPath& path, u64 program_id, void* nacp_out, s64 nacp_size, std::vector* icon_out) { FsFileSystem fs; - R_TRY(fsOpenFileSystemWithId(std::addressof(fs), id, FsFileSystemType_ContentControl, path, FsContentAttributes_All)); + R_TRY(fsOpenFileSystemWithId(std::addressof(fs), program_id, FsFileSystemType_ContentControl, path, FsContentAttributes_All)); ON_SCOPE_EXIT(fsFsClose(std::addressof(fs))); // read nacp.