From 6ce566aea54c7a0735fbff6a0222d3ccf263b214 Mon Sep 17 00:00:00 2001 From: ITotalJustice <47043333+ITotalJustice@users.noreply.github.com> Date: Fri, 5 Sep 2025 14:10:06 +0100 Subject: [PATCH] http: optimise the dir_list parsing, only parse tables. filebrowser: option to disable stat per fs. this improves network fs speed by disabling stat for http entriely, and only enabling file stat for everything else. this can be overriden in the config. --- sphaira/include/location.hpp | 9 +- sphaira/include/ui/menus/filebrowser.hpp | 23 +++- sphaira/source/location.cpp | 11 +- sphaira/source/ui/menus/filebrowser.cpp | 19 +-- sphaira/source/usbdvd.cpp | 2 +- sphaira/source/utils/devoptab_ftp.cpp | 23 +++- sphaira/source/utils/devoptab_http.cpp | 149 +++++++++++++---------- sphaira/source/utils/devoptab_nfs.cpp | 29 ++++- sphaira/source/utils/devoptab_smb2.cpp | 23 +++- 9 files changed, 191 insertions(+), 97 deletions(-) diff --git a/sphaira/include/location.hpp b/sphaira/include/location.hpp index 3cb7919..3411ff1 100644 --- a/sphaira/include/location.hpp +++ b/sphaira/include/location.hpp @@ -3,9 +3,14 @@ #include #include #include +// to import FsEntryFlags. +// todo: this should be part of a smaller header, such as filesystem_types.hpp +#include "ui/menus/filebrowser.hpp" namespace sphaira::location { +using FsEntryFlag = ui::menu::filebrowser::FsEntryFlag; + struct Entry { std::string name{}; std::string url{}; @@ -29,8 +34,8 @@ struct StdioEntry { std::string mount{}; // ums0: (USB Flash Disk) std::string name{}; - // set if read-only. - bool write_protect; + // FsEntryFlag + u32 flags{}; }; using StdioEntries = std::vector; diff --git a/sphaira/include/ui/menus/filebrowser.hpp b/sphaira/include/ui/menus/filebrowser.hpp index b240563..2437013 100644 --- a/sphaira/include/ui/menus/filebrowser.hpp +++ b/sphaira/include/ui/menus/filebrowser.hpp @@ -39,7 +39,12 @@ enum FsEntryFlag { // supports file assoc. FsEntryFlag_Assoc = 1 << 1, // this is an sd card, files can be launched from here. - FsEntryFlag_IsSd = 1 << 2, + FsEntryFlag_IsSd = 1 << 2, // todo: remove this. + // do not stat files in this entry (faster for network mount). + FsEntryFlag_NoStatFile = 1 << 3, + FsEntryFlag_NoStatDir = 1 << 4, + FsEntryFlag_NoRandomReads = 1 << 5, + FsEntryFlag_NoRandomWrites = 1 << 6, }; enum class FsType { @@ -90,6 +95,22 @@ struct FsEntry { return flags & FsEntryFlag_IsSd; } + auto IsNoStatFile() const -> bool { + return flags & FsEntryFlag_NoStatFile; + } + + auto IsNoStatDir() const -> bool { + return flags & FsEntryFlag_NoStatDir; + } + + auto IsNoRandomReads() const -> bool { + return flags & FsEntryFlag_NoRandomReads; + } + + auto IsNoRandomWrites() const -> bool { + return flags & FsEntryFlag_NoRandomWrites; + } + auto IsSame(const FsEntry& e) const { return root == e.root && type == e.type; } diff --git a/sphaira/source/location.cpp b/sphaira/source/location.cpp index ccbadbc..76ce52e 100644 --- a/sphaira/source/location.cpp +++ b/sphaira/source/location.cpp @@ -82,7 +82,7 @@ auto GetStdio(bool write) -> StdioEntries { const auto add_from_entries = [](const StdioEntries& entries, StdioEntries& out, bool write) { for (const auto& e : entries) { - if (write && e.write_protect) { + if (write && (e.flags & FsEntryFlag::FsEntryFlag_ReadOnly)) { log_write("[STDIO] skipping read only mount: %s\n", e.name.c_str()); continue; } @@ -155,7 +155,12 @@ auto GetStdio(bool write) -> StdioEntries { char display_name[0x100]; std::snprintf(display_name, sizeof(display_name), "%s (%s - %s - %zu GB)", e.name, LIBUSBHSFS_FS_TYPE_STR(e.fs_type), e.product_name, e.capacity / 1024 / 1024 / 1024); - out.emplace_back(e.name, display_name, e.write_protect); + u32 flags = 0; + if (e.write_protect || (e.flags & UsbHsFsMountFlags_ReadOnly)) { + flags |= FsEntryFlag::FsEntryFlag_ReadOnly; + } + + out.emplace_back(e.name, display_name, flags); log_write("\t[USBHSFS] %s name: %s serial: %s man: %s\n", e.name, e.product_name, e.serial_number, e.manufacturer); } @@ -172,7 +177,7 @@ auto GetFat() -> StdioEntries { char display_name[0x100]; std::snprintf(display_name, sizeof(display_name), "%s (Read Only)", path); - out.emplace_back(path, display_name, true); + out.emplace_back(path, display_name, FsEntryFlag::FsEntryFlag_ReadOnly); } return out; diff --git a/sphaira/source/ui/menus/filebrowser.cpp b/sphaira/source/ui/menus/filebrowser.cpp index 23d85cf..372b2ea 100644 --- a/sphaira/source/ui/menus/filebrowser.cpp +++ b/sphaira/source/ui/menus/filebrowser.cpp @@ -591,8 +591,7 @@ void FsView::Draw(NVGcontext* vg, Theme* theme) { m_scroll_name.Draw(vg, selected, x + text_xoffset+65, y + (h / 2.f), w-(75+text_xoffset+65+50), 20, NVG_ALIGN_LEFT | NVG_ALIGN_MIDDLE, theme->GetColour(text_id), e.name); - // NOTE: make this native only if i disable dir scan from above. - if (e.IsDir()) { + if (e.IsDir() && !m_fs_entry.IsNoStatDir() && (e.dir_count != -1 || !e.done_stat)) { // NOTE: this takes longer than 16ms when opening a new folder due to it // checking all 9 folders at once. if (!got_dir_count && !e.done_stat && e.file_count == -1 && e.dir_count == -1) { @@ -607,7 +606,7 @@ void FsView::Draw(NVGcontext* vg, Theme* theme) { if (e.dir_count != -1) { gfx::drawTextArgs(vg, x + w - text_xoffset, y + (h / 2.f) + 3, 16.f, NVG_ALIGN_RIGHT | NVG_ALIGN_TOP, theme->GetColour(text_id), "%zd dirs"_i18n.c_str(), e.dir_count); } - } else if (e.IsFile()) { + } else if (e.IsFile() && !m_fs_entry.IsNoStatFile() && (e.file_size != -1 || !e.time_stamp.is_valid)) { if (!e.time_stamp.is_valid && !e.done_stat) { e.done_stat = true; const auto path = GetNewPath(e); @@ -1706,12 +1705,7 @@ void FsView::DisplayOptions() { const auto stdio_locations = location::GetStdio(false); for (const auto& e: stdio_locations) { - u32 flags{}; - if (e.write_protect) { - flags |= FsEntryFlag_ReadOnly; - } - - fs_entries.emplace_back(e.name, e.mount, FsType::Stdio, flags); + fs_entries.emplace_back(e.name, e.mount, FsType::Stdio, e.flags); mount_items.push_back(e.name); } @@ -1727,12 +1721,7 @@ void FsView::DisplayOptions() { const auto fat_entries = location::GetFat(); for (const auto& e: fat_entries) { - u32 flags{}; - if (e.write_protect) { - flags |= FsEntryFlag_ReadOnly; - } - - fs_entries.emplace_back(e.name, e.mount, FsType::Stdio, flags); + fs_entries.emplace_back(e.name, e.mount, FsType::Stdio, e.flags); mount_items.push_back(e.name); } diff --git a/sphaira/source/usbdvd.cpp b/sphaira/source/usbdvd.cpp index aa09d0f..b145371 100644 --- a/sphaira/source/usbdvd.cpp +++ b/sphaira/source/usbdvd.cpp @@ -65,7 +65,7 @@ bool GetMountPoint(location::StdioEntry& out) { out.mount = fs.mountpoint; out.name = display_name; - out.write_protect = true; + out.flags = location::FsEntryFlag::FsEntryFlag_ReadOnly; return true; } diff --git a/sphaira/source/utils/devoptab_ftp.cpp b/sphaira/source/utils/devoptab_ftp.cpp index 9b48272..22f2fe5 100644 --- a/sphaira/source/utils/devoptab_ftp.cpp +++ b/sphaira/source/utils/devoptab_ftp.cpp @@ -42,6 +42,8 @@ struct FtpMountConfig { std::optional port{}; long timeout{DEFAULT_FTP_TIMEOUT}; bool read_only{}; + bool no_stat_file{false}; + bool no_stat_dir{true}; }; using FtpMountConfigs = std::vector; @@ -914,9 +916,13 @@ Result MountFtpAll() { } else if (!std::strcmp(Key, "port")) { e->back().port = ini_parse_getl(Value, DEFAULT_FTP_PORT); } else if (!std::strcmp(Key, "timeout")) { - e->back().timeout = ini_parse_getl(Value, DEFAULT_FTP_TIMEOUT); + e->back().timeout = ini_parse_getl(Value, e->back().timeout); } else if (!std::strcmp(Key, "read_only")) { - e->back().read_only = ini_parse_getbool(Value, false); + e->back().read_only = ini_parse_getbool(Value, e->back().read_only); + } else if (!std::strcmp(Key, "no_stat_file")) { + e->back().no_stat_file = ini_parse_getbool(Value, e->back().no_stat_file); + } else if (!std::strcmp(Key, "no_stat_dir")) { + e->back().no_stat_dir = ini_parse_getbool(Value, e->back().no_stat_dir); } else { log_write("[FTP] INI: Unknown key %s=%s\n", Key, Value); } @@ -989,7 +995,18 @@ Result GetFtpMounts(location::StdioEntries& out) { for (const auto& entry : g_entries) { if (entry) { - out.emplace_back(entry->mount, entry->name, entry->device.config.read_only); + u32 flags = 0; + if (entry->device.config.read_only) { + flags |= location::FsEntryFlag::FsEntryFlag_ReadOnly; + } + if (entry->device.config.no_stat_file) { + flags |= location::FsEntryFlag::FsEntryFlag_NoStatFile; + } + if (entry->device.config.no_stat_dir) { + flags |= location::FsEntryFlag::FsEntryFlag_NoStatDir; + } + + out.emplace_back(entry->mount, entry->name, flags); } } diff --git a/sphaira/source/utils/devoptab_http.cpp b/sphaira/source/utils/devoptab_http.cpp index cf62a4e..31f7884 100644 --- a/sphaira/source/utils/devoptab_http.cpp +++ b/sphaira/source/utils/devoptab_http.cpp @@ -19,7 +19,7 @@ namespace sphaira::devoptab { namespace { -constexpr int DEFAULT_HTTP_TIMEOUT = 3000; // 3 seconds. +constexpr long DEFAULT_HTTP_TIMEOUT = 3000; // 3 seconds. #define CURL_EASY_SETOPT_LOG(handle, opt, v) \ if (auto r = curl_easy_setopt(handle, opt, v); r != CURLE_OK) { \ @@ -36,8 +36,10 @@ struct HttpMountConfig { std::string url{}; std::string user{}; std::string pass{}; - std::optional port{}; - int timeout{DEFAULT_HTTP_TIMEOUT}; + std::optional port{}; + long timeout{DEFAULT_HTTP_TIMEOUT}; + bool no_stat_file{true}; + bool no_stat_dir{true}; }; using HttpMountConfigs = std::vector; @@ -127,8 +129,8 @@ std::string build_url(const std::string& base, const std::string& path, bool is_ void http_set_common_options(Device& client, const std::string& url) { curl_easy_reset(client.curl); CURL_EASY_SETOPT_LOG(client.curl, CURLOPT_URL, url.c_str()); - CURL_EASY_SETOPT_LOG(client.curl, CURLOPT_TIMEOUT, (long)client.config.timeout); - CURL_EASY_SETOPT_LOG(client.curl, CURLOPT_CONNECTTIMEOUT, (long)client.config.timeout); + CURL_EASY_SETOPT_LOG(client.curl, CURLOPT_TIMEOUT, client.config.timeout); + CURL_EASY_SETOPT_LOG(client.curl, CURLOPT_CONNECTTIMEOUT, client.config.timeout); CURL_EASY_SETOPT_LOG(client.curl, CURLOPT_AUTOREFERER, 1L); CURL_EASY_SETOPT_LOG(client.curl, CURLOPT_FOLLOWLOCATION, 1L); CURL_EASY_SETOPT_LOG(client.curl, CURLOPT_SSL_VERIFYPEER, 0L); @@ -139,7 +141,7 @@ void http_set_common_options(Device& client, const std::string& url) { CURL_EASY_SETOPT_LOG(client.curl, CURLOPT_BUFFERSIZE, 1024L * 512L); if (client.config.port) { - CURL_EASY_SETOPT_LOG(client.curl, CURLOPT_PORT, (long)client.config.port.value()); + CURL_EASY_SETOPT_LOG(client.curl, CURLOPT_PORT, client.config.port.value()); } // enable all forms of compression supported by libcurl. @@ -175,66 +177,74 @@ bool http_dirlist(Device& client, const std::string& path, DirEntries& out) { return false; } - chunk.emplace_back('\0'); // null-terminate the chunk - const char* dilim = ""); - if (!href_end) { - continue; + // very fast/basic html parsing. + // takes 17ms to parse 3MB html with 7641 entries. + // todo: if i ever add an xml parser to sphaira, use that instead. + // todo: for the above, benchmark the parser to ensure its faster than the my code. + std::string_view chunk_view{chunk.data(), chunk.size()}; + const auto table_start = chunk_view.find(""); + + if (table_start != std::string_view::npos && table_end != std::string_view::npos && table_end > table_start) { + const std::string_view href_tag_start = ""; + const std::string_view anchor_tag_end = ""; + const auto table_view = chunk_view.substr(table_start, table_end - table_start); + + size_t pos = 0; + out.reserve(10000); + + for (;;) { + const auto href_pos = table_view.find(href_tag_start, pos); + if (href_pos == std::string_view::npos) { + break; // no more href. + } + pos = href_pos + href_tag_start.length(); + + const auto href_begin = pos; + const auto href_end = table_view.find(href_tag_end, href_begin); + if (href_end == std::string_view::npos) { + break; // no more href. + } + + const auto name_begin = href_end + href_tag_end.length(); + const auto name_end = table_view.find(anchor_tag_end, name_begin); + if (name_end == std::string_view::npos) { + break; // no more names. + } + + pos = name_end + anchor_tag_end.length(); + const auto href_view = table_view.substr(href_begin, href_end - href_begin); + const auto name_view = table_view.substr(name_begin, name_end - name_begin); + + // skip empty names/links, root dir entry and links that are not actual files/dirs (e.g. sorting/filter controls). + if (name_view.empty() || href_view.empty() || name_view == "/" || href_view.starts_with('?') || href_view.starts_with('#')) { + continue; + } + + // skip parent directory entry and external links. + if (href_view == ".." || name_view == ".." || href_view.starts_with("../") || name_view.starts_with("../") || href_view.find("://") != std::string::npos) { + continue; + } + + std::string name{name_view}; + const std::string href{href_view}; + + const auto is_dir = name.ends_with('/'); + if (is_dir) { + name.pop_back(); // remove the trailing '/' + } + + out.emplace_back(name, href, is_dir); } - const auto href_len = href_end - href_begin; - - const auto name_begin = href_end + std::strlen("\">"); - const auto name_end = std::strstr(name_begin, ""); - if (!name_end) { - continue; - } - const auto name_len = name_end - name_begin; - - if (href_len <= 0 || name_len <= 0) { - continue; - } - - // skip if inside