diff --git a/sphaira/include/defines.hpp b/sphaira/include/defines.hpp index 335eb8c..125a0ea 100644 --- a/sphaira/include/defines.hpp +++ b/sphaira/include/defines.hpp @@ -537,6 +537,9 @@ enum class SphairaResult : Result { ZipOpenNewFileInZip, ZipWriteInFileInZip, + MmzBadLocalHeaderSig, + MmzBadLocalHeaderRead, + FileBrowserFailedUpload, FileBrowserDirNotDaybreak, @@ -685,6 +688,8 @@ enum : Result { MAKE_SPHAIRA_RESULT_ENUM(ZipOpen2_64), MAKE_SPHAIRA_RESULT_ENUM(ZipOpenNewFileInZip), MAKE_SPHAIRA_RESULT_ENUM(ZipWriteInFileInZip), + MAKE_SPHAIRA_RESULT_ENUM(MmzBadLocalHeaderSig), + MAKE_SPHAIRA_RESULT_ENUM(MmzBadLocalHeaderRead), MAKE_SPHAIRA_RESULT_ENUM(FileBrowserFailedUpload), MAKE_SPHAIRA_RESULT_ENUM(FileBrowserDirNotDaybreak), MAKE_SPHAIRA_RESULT_ENUM(AppstoreFailedZipDownload), diff --git a/sphaira/include/minizip_helper.hpp b/sphaira/include/minizip_helper.hpp index 0b4fc2e..ebf2219 100644 --- a/sphaira/include/minizip_helper.hpp +++ b/sphaira/include/minizip_helper.hpp @@ -4,6 +4,7 @@ #include #include #include +#include "fs.hpp" namespace sphaira::mz { @@ -20,5 +21,12 @@ struct MzSpan { void FileFuncMem(MzMem* mem, zlib_filefunc64_def* funcs); void FileFuncSpan(MzSpan* span, zlib_filefunc64_def* funcs); void FileFuncStdio(zlib_filefunc64_def* funcs); +void FileFuncNative(zlib_filefunc64_def* funcs); + +// minizip takes 18ms to open a zip and 4ms to parse the first file entry. +// this results in a dropped frame. +// this version simply reads the local header + file name in 2 reads, +// which takes 1-2ms. +Result PeekFirstFileName(fs::Fs* fs, const fs::FsPath& path, fs::FsPath& name); } // namespace sphaira::mz diff --git a/sphaira/include/ui/menus/file_picker.hpp b/sphaira/include/ui/menus/file_picker.hpp index 9909ca2..66ed3c6 100644 --- a/sphaira/include/ui/menus/file_picker.hpp +++ b/sphaira/include/ui/menus/file_picker.hpp @@ -1,5 +1,6 @@ #pragma once +#include "ui/menus/filebrowser.hpp" #include "ui/menus/menu_base.hpp" #include "ui/scrolling_text.hpp" #include "ui/list.hpp" @@ -17,13 +18,6 @@ enum FsEntryFlag { FsEntryFlag_Assoc = 1 << 1, }; -enum class FsType { - Sd, - ImageNand, - ImageSd, - Stdio, -}; - enum SortType { SortType_Size, SortType_Alphabetical, @@ -34,77 +28,10 @@ enum OrderType { OrderType_Ascending, }; -struct FsEntry { - fs::FsPath name{}; - fs::FsPath root{}; - FsType type{}; - u32 flags{FsEntryFlag_None}; - - auto IsReadOnly() const -> bool { - return flags & FsEntryFlag_ReadOnly; - } - - auto IsAssoc() const -> bool { - return flags & FsEntryFlag_Assoc; - } - - auto IsSame(const FsEntry& e) const { - return root == e.root && type == e.type; - } -}; - -// roughly 1kib in size per entry -struct FileEntry : FsDirectoryEntry { - std::string extension{}; // if any - std::string internal_name{}; // if any - std::string internal_extension{}; // if any - s64 file_count{-1}; // number of files in a folder, non-recursive - s64 dir_count{-1}; // number folders in a folder, non-recursive - FsTimeStampRaw time_stamp{}; - bool checked_extension{}; // did we already search for an ext? - bool checked_internal_extension{}; // did we already search for an ext? - - auto IsFile() const -> bool { - return type == FsDirEntryType_File; - } - - auto IsDir() const -> bool { - return !IsFile(); - } - - auto IsHidden() const -> bool { - return name[0] == '.'; - } - - auto GetName() const -> std::string { - return name; - } - - auto GetExtension() const -> std::string { - return extension; - } - - auto GetInternalName() const -> std::string { - if (!internal_name.empty()) { - return internal_name; - } - return GetName(); - } - - auto GetInternalExtension() const -> std::string { - if (!internal_extension.empty()) { - return internal_extension; - } - return GetExtension(); - } -}; - -struct LastFile { - fs::FsPath name{}; - s64 index{}; - float offset{}; - s64 entries_count{}; -}; +using FsType = filebrowser::FsType; +using FsEntry = filebrowser::FsEntry; +using FileEntry = filebrowser::FileEntry; +using LastFile = filebrowser::LastFile; using Callback = std::function; diff --git a/sphaira/include/ui/menus/filebrowser.hpp b/sphaira/include/ui/menus/filebrowser.hpp index 19ba180..7912356 100644 --- a/sphaira/include/ui/menus/filebrowser.hpp +++ b/sphaira/include/ui/menus/filebrowser.hpp @@ -7,7 +7,6 @@ #include "fs.hpp" #include "option.hpp" #include "hasher.hpp" -// #include #include namespace sphaira::ui::menu::filebrowser { @@ -97,6 +96,11 @@ struct FileEntry : FsDirectoryEntry { } auto GetExtension() const -> std::string { + if (!checked_extension) { + if (auto ext = std::strrchr(name, '.')) { + return ext+1; + } + } return extension; } @@ -396,14 +400,6 @@ private: std::vector m_assoc_entries{}; SelectedStash m_selected{}; - // this keeps track of the highlighted file before opening a folder - // if the user presses B to go back to the previous dir - // this vector is popped, then, that entry is checked if it still exists - // if it does, the index becomes that file. - std::vector m_previous_highlighted_file{}; - s64 m_index{}; - s64 m_selected_count{}; - option::OptionLong m_sort{INI_SECTION, "sort", SortType::SortType_Alphabetical}; option::OptionLong m_order{INI_SECTION, "order", OrderType::OrderType_Descending}; option::OptionBool m_show_hidden{INI_SECTION, "show_hidden", false}; @@ -412,7 +408,6 @@ private: option::OptionBool m_ignore_read_only{INI_SECTION, "ignore_read_only", false}; bool m_loaded_assoc_entries{}; - bool m_is_update_folder{}; bool m_split_screen{}; }; diff --git a/sphaira/source/minizip_helper.cpp b/sphaira/source/minizip_helper.cpp index 0c0fac3..2d8c119 100644 --- a/sphaira/source/minizip_helper.cpp +++ b/sphaira/source/minizip_helper.cpp @@ -7,6 +7,68 @@ namespace sphaira::mz { namespace { +// mmz is part of ftpsrv code. +#define LOCAL_HEADER_SIG 0x4034B50 +#define FILE_HEADER_SIG 0x2014B50 +#define END_RECORD_SIG 0x6054B50 + +// 30 bytes (0x1E) +#pragma pack(push,1) +typedef struct mmz_LocalHeader { + uint32_t sig; + uint16_t version; + uint16_t flags; + uint16_t compression; + uint16_t modtime; + uint16_t moddate; + uint32_t crc32; + uint32_t compressed_size; + uint32_t uncompressed_size; + uint16_t filename_len; + uint16_t extrafield_len; +} mmz_LocalHeader; +#pragma pack(pop) + +// 46 bytes (0x2E) +#pragma pack(push,1) +typedef struct mmz_FileHeader { + uint32_t sig; + uint16_t version; + uint16_t version_needed; + uint16_t flags; + uint16_t compression; + uint16_t modtime; + uint16_t moddate; + uint32_t crc32; + uint32_t compressed_size; + uint32_t uncompressed_size; + uint16_t filename_len; + uint16_t extrafield_len; + uint16_t filecomment_len; + uint16_t disk_start; // wat + uint16_t internal_attr; // wat + uint32_t external_attr; // wat + uint32_t local_hdr_off; +} mmz_FileHeader; +#pragma pack(pop) + +#pragma pack(push,1) +typedef struct mmz_EndRecord { + uint32_t sig; + uint16_t disk_number; + uint16_t disk_wcd; + uint16_t disk_entries; + uint16_t total_entries; + uint32_t central_directory_size; + uint32_t file_hdr_off; + uint16_t comment_len; +} mmz_EndRecord; +#pragma pack(pop) + +static_assert(sizeof(mmz_LocalHeader) == 0x1E); +static_assert(sizeof(mmz_FileHeader) == 0x2E); +static_assert(sizeof(mmz_EndRecord) == 0x16); + voidpf minizip_open_file_func_mem(voidpf opaque, const void* filename, int mode) { return opaque; } @@ -191,6 +253,99 @@ constexpr zlib_filefunc64_def zlib_filefunc_stdio = { .zerror_file = minizip_error_file_func_stdio, }; + + + + + +struct Internal { + FsFile file; + s64 offset; + s64 size; + Result rc; +}; + +static void* zopen64_file(void* opaque, const void* filename, int mode) +{ + struct Internal* fs = (struct Internal*)calloc(1, sizeof(*fs)); + if (R_FAILED(fs->rc = fsFsOpenFile(fsdevGetDeviceFileSystem("sdmc:"), (const char*)filename, FsOpenMode_Read, &fs->file))) { + free(fs); + return NULL; + } + + if (R_FAILED(fs->rc = fsFileGetSize(&fs->file, &fs->size))) { + free(fs); + return NULL; + } + + return fs; +} + +static uLong zread_file(void* opaque, void* stream, void* buf, unsigned long size) +{ + struct Internal* fs = (struct Internal*)stream; + + u64 bytes_read; + if (R_FAILED(fs->rc = fsFileRead(&fs->file, fs->offset, buf, size, 0, &bytes_read))) { + return 0; + } + + fs->offset += bytes_read; + return bytes_read; +} + +static ZPOS64_T ztell64_file(void* opaque, void* stream) +{ + struct Internal* fs = (struct Internal*)stream; + return fs->offset; +} + +static long zseek64_file(void* opaque, void* stream, ZPOS64_T offset, int origin) +{ + struct Internal* fs = (struct Internal*)stream; + switch (origin) { + case SEEK_SET: { + fs->offset = offset; + } break; + case SEEK_CUR: { + fs->offset += offset; + } break; + case SEEK_END: { + fs->offset = fs->size + offset; + } break; + } + return 0; +} + +static int zclose_file(void* opaque, void* stream) +{ + if (stream) { + struct Internal* fs = (struct Internal*)stream; + fsFileClose(&fs->file); + memset(fs, 0, sizeof(*fs)); + free(fs); + } + return 0; +} + +static int zerror_file(void* opaque, void* stream) +{ + struct Internal* fs = (struct Internal*)stream; + if (R_FAILED(fs->rc)) { + return -1; + } + return 0; +} + +static const zlib_filefunc64_def zlib_filefunc_native = { + .zopen64_file = zopen64_file, + .zread_file = zread_file, + .ztell64_file = ztell64_file, + .zseek64_file = zseek64_file, + .zclose_file = zclose_file, + .zerror_file = zerror_file, +}; + } // namespace void FileFuncMem(MzMem* mem, zlib_filefunc64_def* funcs) { @@ -207,4 +362,26 @@ void FileFuncStdio(zlib_filefunc64_def* funcs) { *funcs = zlib_filefunc_stdio; } +void FileFuncNative(zlib_filefunc64_def* funcs) { + *funcs = zlib_filefunc_native; +} + +Result PeekFirstFileName(fs::Fs* fs, const fs::FsPath& path, fs::FsPath& name) { + fs::File file; + R_TRY(fs->OpenFile(path, FsOpenMode_Read, &file)); + + mmz_LocalHeader local_hdr; + u64 bytes_read; + R_TRY(file.Read(0, &local_hdr, sizeof(local_hdr), 0, &bytes_read)); + + R_UNLESS(bytes_read == sizeof(local_hdr), Result_MmzBadLocalHeaderRead); + R_UNLESS(local_hdr.sig == LOCAL_HEADER_SIG, Result_MmzBadLocalHeaderSig); + + const auto name_len = std::min(local_hdr.filename_len, sizeof(name) - 1); + R_TRY(file.Read(bytes_read, name, name_len, 0, &bytes_read)); + name[name_len] = '\0'; + + R_SUCCEED(); +} + } // namespace sphaira::mz diff --git a/sphaira/source/ui/error_box.cpp b/sphaira/source/ui/error_box.cpp index 2ce28a7..43f5fe4 100644 --- a/sphaira/source/ui/error_box.cpp +++ b/sphaira/source/ui/error_box.cpp @@ -71,6 +71,8 @@ auto GetCodeMessage(Result rc) -> const char* { case Result_ZipOpen2_64: return "SphairaError_ZipOpen2_64"; case Result_ZipOpenNewFileInZip: return "SphairaError_ZipOpenNewFileInZip"; case Result_ZipWriteInFileInZip: return "SphairaError_ZipWriteInFileInZip"; + case Result_MmzBadLocalHeaderSig: return "SphairaError_MmzBadLocalHeaderSig"; + case Result_MmzBadLocalHeaderRead: return "SphairaError_MmzBadLocalHeaderRead"; case Result_FileBrowserFailedUpload: return "SphairaError_FileBrowserFailedUpload"; case Result_FileBrowserDirNotDaybreak: return "SphairaError_FileBrowserDirNotDaybreak"; case Result_AppstoreFailedZipDownload: return "SphairaError_AppstoreFailedZipDownload"; diff --git a/sphaira/source/ui/menus/file_picker.cpp b/sphaira/source/ui/menus/file_picker.cpp index e929b15..9daabf4 100644 --- a/sphaira/source/ui/menus/file_picker.cpp +++ b/sphaira/source/ui/menus/file_picker.cpp @@ -14,7 +14,6 @@ #include "minizip_helper.hpp" #include -#include #include #include #include @@ -82,21 +81,14 @@ void Menu::SetIndex(s64 index) { if (IsSd() && !m_entries_current.empty() && !GetEntry().checked_internal_extension && IsSamePath(GetEntry().extension, "zip")) { GetEntry().checked_internal_extension = true; - if (auto zfile = unzOpen64(GetNewPathCurrent())) { - ON_SCOPE_EXIT(unzClose(zfile)); - - // only check first entry (i think RA does the same) - fs::FsPath filename_inzip{}; - unz_file_info64 file_info{}; - if (UNZ_OK == unzOpenCurrentFile(zfile)) { - ON_SCOPE_EXIT(unzCloseCurrentFile(zfile)); - if (UNZ_OK == unzGetCurrentFileInfo64(zfile, &file_info, filename_inzip, sizeof(filename_inzip), NULL, 0, NULL, 0)) { - if (auto ext = std::strrchr(filename_inzip, '.')) { - GetEntry().internal_name = filename_inzip.toString(); - GetEntry().internal_extension = ext+1; - } - } + TimeStamp ts; + fs::FsPath filename_inzip{}; + if (R_SUCCEEDED(mz::PeekFirstFileName(GetFs(), GetNewPathCurrent(), filename_inzip))) { + if (auto ext = std::strrchr(filename_inzip, '.')) { + GetEntry().internal_name = filename_inzip.toString(); + GetEntry().internal_extension = ext+1; } + log_write("\tzip, time taken: %.2fs %zums\n", ts.GetSecondsD(), ts.GetMs()); } } @@ -450,24 +442,12 @@ void Menu::Draw(NVGcontext* vg, Theme* theme) { } constexpr float text_xoffset{15.f}; + bool got_dir_count = false; - m_list->Draw(vg, theme, m_entries_current.size(), [this, text_col](auto* vg, auto* theme, auto v, auto i) { + m_list->Draw(vg, theme, m_entries_current.size(), [this, text_col, &got_dir_count](auto* vg, auto* theme, auto v, auto i) { const auto& [x, y, w, h] = v; auto& e = GetEntry(i); - if (e.IsDir()) { - // NOTE: make this native only if hdd dir scan is too slow. - // if (m_fs->IsNative() && e.file_count == -1 && e.dir_count == -1) { - if (e.file_count == -1 && e.dir_count == -1) { - m_fs->DirGetEntryCount(GetNewPath(e), &e.file_count, &e.dir_count); - } - } else if (!e.checked_extension) { - e.checked_extension = true; - if (auto ext = std::strrchr(e.name, '.')) { - e.extension = ext+1; - } - } - auto text_id = ThemeEntryID_TEXT; const auto selected = m_index == i; if (selected) { @@ -506,8 +486,19 @@ void Menu::Draw(NVGcontext* vg, Theme* theme) { // NOTE: make this native only if i disable dir scan from above. if (e.IsDir()) { - gfx::drawTextArgs(vg, x + w - text_xoffset, y + (h / 2.f) - 3, 16.f, NVG_ALIGN_RIGHT | NVG_ALIGN_BOTTOM, theme->GetColour(text_id), "%zd files"_i18n.c_str(), e.file_count); - gfx::drawTextArgs(vg, x + w - text_xoffset, y + (h / 2.f) + 3, 16.f, NVG_ALIGN_RIGHT | NVG_ALIGN_TOP, theme->GetColour(text_id), "%zd dirs"_i18n.c_str(), e.dir_count); + // NOTE: this takes longer than 16ms when opening a new folder due to it + // checking all 9 folders at once. + if (!got_dir_count && e.file_count == -1 && e.dir_count == -1) { + got_dir_count = true; + m_fs->DirGetEntryCount(GetNewPath(e), &e.file_count, &e.dir_count); + } + + if (e.file_count != -1) { + gfx::drawTextArgs(vg, x + w - text_xoffset, y + (h / 2.f) - 3, 16.f, NVG_ALIGN_RIGHT | NVG_ALIGN_BOTTOM, theme->GetColour(text_id), "%zd files"_i18n.c_str(), e.file_count); + } + if (e.dir_count != -1) { + gfx::drawTextArgs(vg, x + w - text_xoffset, y + (h / 2.f) + 3, 16.f, NVG_ALIGN_RIGHT | NVG_ALIGN_TOP, theme->GetColour(text_id), "%zd dirs"_i18n.c_str(), e.dir_count); + } } else if (e.IsFile()) { if (!e.time_stamp.is_valid) { const auto path = GetNewPath(e); diff --git a/sphaira/source/ui/menus/filebrowser.cpp b/sphaira/source/ui/menus/filebrowser.cpp index 1edc5c0..34a59a6 100644 --- a/sphaira/source/ui/menus/filebrowser.cpp +++ b/sphaira/source/ui/menus/filebrowser.cpp @@ -566,24 +566,12 @@ void FsView::Draw(NVGcontext* vg, Theme* theme) { } constexpr float text_xoffset{15.f}; + bool got_dir_count = false; - m_list->Draw(vg, theme, m_entries_current.size(), [this, text_col](auto* vg, auto* theme, auto v, auto i) { + m_list->Draw(vg, theme, m_entries_current.size(), [this, text_col, &got_dir_count](auto* vg, auto* theme, auto v, auto i) { const auto& [x, y, w, h] = v; auto& e = GetEntry(i); - if (e.IsDir()) { - // NOTE: make this native only if hdd dir scan is too slow. - // if (m_fs->IsNative() && e.file_count == -1 && e.dir_count == -1) { - if (e.file_count == -1 && e.dir_count == -1) { - m_fs->DirGetEntryCount(GetNewPath(e), &e.file_count, &e.dir_count); - } - } else if (!e.checked_extension) { - e.checked_extension = true; - if (auto ext = std::strrchr(e.name, '.')) { - e.extension = ext+1; - } - } - auto text_id = ThemeEntryID_TEXT; const auto selected = m_index == i; if (selected) { @@ -626,8 +614,19 @@ void FsView::Draw(NVGcontext* vg, Theme* theme) { // NOTE: make this native only if i disable dir scan from above. if (e.IsDir()) { - gfx::drawTextArgs(vg, x + w - text_xoffset, y + (h / 2.f) - 3, 16.f, NVG_ALIGN_RIGHT | NVG_ALIGN_BOTTOM, theme->GetColour(text_id), "%zd files"_i18n.c_str(), e.file_count); - gfx::drawTextArgs(vg, x + w - text_xoffset, y + (h / 2.f) + 3, 16.f, NVG_ALIGN_RIGHT | NVG_ALIGN_TOP, theme->GetColour(text_id), "%zd dirs"_i18n.c_str(), e.dir_count); + // NOTE: this takes longer than 16ms when opening a new folder due to it + // checking all 9 folders at once. + if (!got_dir_count && e.file_count == -1 && e.dir_count == -1) { + got_dir_count = true; + m_fs->DirGetEntryCount(GetNewPath(e), &e.file_count, &e.dir_count); + } + + if (e.file_count != -1) { + gfx::drawTextArgs(vg, x + w - text_xoffset, y + (h / 2.f) - 3, 16.f, NVG_ALIGN_RIGHT | NVG_ALIGN_BOTTOM, theme->GetColour(text_id), "%zd files"_i18n.c_str(), e.file_count); + } + if (e.dir_count != -1) { + gfx::drawTextArgs(vg, x + w - text_xoffset, y + (h / 2.f) + 3, 16.f, NVG_ALIGN_RIGHT | NVG_ALIGN_TOP, theme->GetColour(text_id), "%zd dirs"_i18n.c_str(), e.dir_count); + } } else if (e.IsFile()) { if (!e.time_stamp.is_valid) { const auto path = GetNewPath(e); @@ -703,24 +702,17 @@ void FsView::SetIndex(s64 index) { m_list->SetYoff(); } - if (IsSd() && !m_entries_current.empty() && !GetEntry().checked_internal_extension && IsSamePath(GetEntry().extension, "zip")) { + if (IsSd() && !m_entries_current.empty() && !GetEntry().checked_internal_extension && IsSamePath(GetEntry().GetExtension(), "zip")) { GetEntry().checked_internal_extension = true; - if (auto zfile = unzOpen64(GetNewPathCurrent())) { - ON_SCOPE_EXIT(unzClose(zfile)); - - // only check first entry (i think RA does the same) - fs::FsPath filename_inzip{}; - unz_file_info64 file_info{}; - if (UNZ_OK == unzOpenCurrentFile(zfile)) { - ON_SCOPE_EXIT(unzCloseCurrentFile(zfile)); - if (UNZ_OK == unzGetCurrentFileInfo64(zfile, &file_info, filename_inzip, sizeof(filename_inzip), NULL, 0, NULL, 0)) { - if (auto ext = std::strrchr(filename_inzip, '.')) { - GetEntry().internal_name = filename_inzip.toString(); - GetEntry().internal_extension = ext+1; - } - } + TimeStamp ts; + fs::FsPath filename_inzip{}; + if (R_SUCCEEDED(mz::PeekFirstFileName(GetFs(), GetNewPathCurrent(), filename_inzip))) { + if (auto ext = std::strrchr(filename_inzip, '.')) { + GetEntry().internal_name = filename_inzip.toString(); + GetEntry().internal_extension = ext+1; } + log_write("\tzip, time taken: %.2fs %zums\n", ts.GetSecondsD(), ts.GetMs()); } } @@ -737,7 +729,7 @@ void FsView::InstallForwarder() { const auto assoc_list = m_menu->FindFileAssocFor(); if (assoc_list.empty()) { - log_write("failed to find assoc for: %s ext: %s\n", GetEntry().name, GetEntry().extension.c_str()); + log_write("failed to find assoc for: %s ext: %s\n", GetEntry().name, GetEntry().GetExtension().c_str()); return; } @@ -1081,9 +1073,6 @@ auto FsView::Scan(const fs::FsPath& new_path, bool is_walk_up) -> Result { i++; } - m_entries.shrink_to_fit(); - m_entries_index.shrink_to_fit(); - m_entries_index_hidden.shrink_to_fit(); Sort(); // quick check to see if this is an update folder @@ -2000,8 +1989,8 @@ auto Menu::FindFileAssocFor() -> std::vector { // only support roms in correctly named folders, sorry! const auto db_indexs = GetRomDatabaseFromPath(view->m_path); const auto& entry = view->GetEntry(); - const auto extension = entry.extension; - const auto internal_extension = entry.internal_extension.empty() ? entry.extension : entry.internal_extension; + const auto extension = entry.GetExtension(); + const auto internal_extension = entry.GetInternalExtension(); if (extension.empty() && internal_extension.empty()) { // log_write("failed to get extension for db: %s path: %s\n", database_entry.c_str(), m_path); return {};