http: optimise the dir_list parsing, only parse tables. filebrowser: option to disable stat per fs.

this improves network fs speed by disabling stat for http entriely, and only enabling file stat for everything else.
this can be overriden in the config.

sphaira/include/location.hpp

@@ -3,9 +3,14 @@
 #include <string>
 #include <vector>
 #include <switch.h>
+// to import FsEntryFlags.
+// todo: this should be part of a smaller header, such as filesystem_types.hpp
+#include "ui/menus/filebrowser.hpp"
 
 namespace sphaira::location {
 
+using FsEntryFlag = ui::menu::filebrowser::FsEntryFlag;
+
 struct Entry {
     std::string name{};
     std::string url{};
@@ -29,8 +34,8 @@ struct StdioEntry {
     std::string mount{};
     // ums0: (USB Flash Disk)
     std::string name{};
-    // set if read-only.
+    // FsEntryFlag
-    bool write_protect;
+    u32 flags{};
 };
 
 using StdioEntries = std::vector<StdioEntry>;

sphaira/include/ui/menus/filebrowser.hpp

@@ -39,7 +39,12 @@ enum FsEntryFlag {
     // supports file assoc.
     FsEntryFlag_Assoc = 1 << 1,
     // this is an sd card, files can be launched from here.
-    FsEntryFlag_IsSd = 1 << 2,
+    FsEntryFlag_IsSd = 1 << 2, // todo: remove this.
+    // do not stat files in this entry (faster for network mount).
+    FsEntryFlag_NoStatFile = 1 << 3,
+    FsEntryFlag_NoStatDir = 1 << 4,
+    FsEntryFlag_NoRandomReads = 1 << 5,
+    FsEntryFlag_NoRandomWrites = 1 << 6,
 };
 
 enum class FsType {
@@ -90,6 +95,22 @@ struct FsEntry {
         return flags & FsEntryFlag_IsSd;
     }
 
+    auto IsNoStatFile() const -> bool {
+        return flags & FsEntryFlag_NoStatFile;
+    }
+
+    auto IsNoStatDir() const -> bool {
+        return flags & FsEntryFlag_NoStatDir;
+    }
+
+    auto IsNoRandomReads() const -> bool {
+        return flags & FsEntryFlag_NoRandomReads;
+    }
+
+    auto IsNoRandomWrites() const -> bool {
+        return flags & FsEntryFlag_NoRandomWrites;
+    }
+
     auto IsSame(const FsEntry& e) const {
         return root == e.root && type == e.type;
     }

sphaira/source/location.cpp

@@ -82,7 +82,7 @@ auto GetStdio(bool write) -> StdioEntries {
 
     const auto add_from_entries = [](const StdioEntries& entries, StdioEntries& out, bool write) {
         for (const auto& e : entries) {
-            if (write && e.write_protect) {
+            if (write && (e.flags & FsEntryFlag::FsEntryFlag_ReadOnly)) {
                 log_write("[STDIO] skipping read only mount: %s\n", e.name.c_str());
                 continue;
             }
@@ -155,7 +155,12 @@ auto GetStdio(bool write) -> StdioEntries {
         char display_name[0x100];
         std::snprintf(display_name, sizeof(display_name), "%s (%s - %s - %zu GB)", e.name, LIBUSBHSFS_FS_TYPE_STR(e.fs_type), e.product_name, e.capacity / 1024 / 1024 / 1024);
 
-        out.emplace_back(e.name, display_name, e.write_protect);
+        u32 flags = 0;
+        if (e.write_protect || (e.flags & UsbHsFsMountFlags_ReadOnly)) {
+            flags |= FsEntryFlag::FsEntryFlag_ReadOnly;
+        }
+
+        out.emplace_back(e.name, display_name, flags);
         log_write("\t[USBHSFS] %s name: %s serial: %s man: %s\n", e.name, e.product_name, e.serial_number, e.manufacturer);
     }
 
@@ -172,7 +177,7 @@ auto GetFat() -> StdioEntries {
         char display_name[0x100];
         std::snprintf(display_name, sizeof(display_name), "%s (Read Only)", path);
 
-        out.emplace_back(path, display_name, true);
+        out.emplace_back(path, display_name, FsEntryFlag::FsEntryFlag_ReadOnly);
     }
 
     return out;

sphaira/source/ui/menus/filebrowser.cpp

@@ -591,8 +591,7 @@ void FsView::Draw(NVGcontext* vg, Theme* theme) {
 
         m_scroll_name.Draw(vg, selected, x + text_xoffset+65, y + (h / 2.f), w-(75+text_xoffset+65+50), 20, NVG_ALIGN_LEFT | NVG_ALIGN_MIDDLE, theme->GetColour(text_id), e.name);
 
-        // NOTE: make this native only if i disable dir scan from above.
+        if (e.IsDir() && !m_fs_entry.IsNoStatDir() && (e.dir_count != -1 || !e.done_stat)) {
-        if (e.IsDir()) {
             // NOTE: this takes longer than 16ms when opening a new folder due to it
             // checking all 9 folders at once.
             if (!got_dir_count && !e.done_stat && e.file_count == -1 && e.dir_count == -1) {
@@ -607,7 +606,7 @@ void FsView::Draw(NVGcontext* vg, Theme* theme) {
             if (e.dir_count != -1) {
                 gfx::drawTextArgs(vg, x + w - text_xoffset, y + (h / 2.f) + 3, 16.f, NVG_ALIGN_RIGHT | NVG_ALIGN_TOP, theme->GetColour(text_id), "%zd dirs"_i18n.c_str(), e.dir_count);
             }
-        } else if (e.IsFile()) {
+        } else if (e.IsFile() && !m_fs_entry.IsNoStatFile() && (e.file_size != -1 || !e.time_stamp.is_valid)) {
             if (!e.time_stamp.is_valid && !e.done_stat) {
                 e.done_stat = true;
                 const auto path = GetNewPath(e);
@@ -1706,12 +1705,7 @@ void FsView::DisplayOptions() {
 
     const auto stdio_locations = location::GetStdio(false);
     for (const auto& e: stdio_locations) {
-        u32 flags{};
+        fs_entries.emplace_back(e.name, e.mount, FsType::Stdio, e.flags);
-        if (e.write_protect) {
-            flags |= FsEntryFlag_ReadOnly;
-        }
-
-        fs_entries.emplace_back(e.name, e.mount, FsType::Stdio, flags);
         mount_items.push_back(e.name);
     }
 
@@ -1727,12 +1721,7 @@ void FsView::DisplayOptions() {
 
     const auto fat_entries = location::GetFat();
     for (const auto& e: fat_entries) {
-        u32 flags{};
+        fs_entries.emplace_back(e.name, e.mount, FsType::Stdio, e.flags);
-        if (e.write_protect) {
-            flags |= FsEntryFlag_ReadOnly;
-        }
-
-        fs_entries.emplace_back(e.name, e.mount, FsType::Stdio, flags);
         mount_items.push_back(e.name);
     }
 

sphaira/source/usbdvd.cpp

@@ -65,7 +65,7 @@ bool GetMountPoint(location::StdioEntry& out) {
 
     out.mount = fs.mountpoint;
     out.name = display_name;
-    out.write_protect = true;
+    out.flags = location::FsEntryFlag::FsEntryFlag_ReadOnly;
 
     return true;
 }

sphaira/source/utils/devoptab_ftp.cpp

@@ -42,6 +42,8 @@ struct FtpMountConfig {
     std::optional<long> port{};
     long timeout{DEFAULT_FTP_TIMEOUT};
     bool read_only{};
+    bool no_stat_file{false};
+    bool no_stat_dir{true};
 };
 using FtpMountConfigs = std::vector<FtpMountConfig>;
 
@@ -914,9 +916,13 @@ Result MountFtpAll() {
         } else if (!std::strcmp(Key, "port")) {
             e->back().port = ini_parse_getl(Value, DEFAULT_FTP_PORT);
         } else if (!std::strcmp(Key, "timeout")) {
-            e->back().timeout = ini_parse_getl(Value, DEFAULT_FTP_TIMEOUT);
+            e->back().timeout = ini_parse_getl(Value, e->back().timeout);
         } else if (!std::strcmp(Key, "read_only")) {
-            e->back().read_only = ini_parse_getbool(Value, false);
+            e->back().read_only = ini_parse_getbool(Value, e->back().read_only);
+        } else if (!std::strcmp(Key, "no_stat_file")) {
+            e->back().no_stat_file = ini_parse_getbool(Value, e->back().no_stat_file);
+        } else if (!std::strcmp(Key, "no_stat_dir")) {
+            e->back().no_stat_dir = ini_parse_getbool(Value, e->back().no_stat_dir);
         } else {
             log_write("[FTP] INI: Unknown key %s=%s\n", Key, Value);
         }
@@ -989,7 +995,18 @@ Result GetFtpMounts(location::StdioEntries& out) {
 
     for (const auto& entry : g_entries) {
         if (entry) {
-            out.emplace_back(entry->mount, entry->name, entry->device.config.read_only);
+            u32 flags = 0;
+            if (entry->device.config.read_only) {
+                flags |= location::FsEntryFlag::FsEntryFlag_ReadOnly;
+            }
+            if (entry->device.config.no_stat_file) {
+                flags |= location::FsEntryFlag::FsEntryFlag_NoStatFile;
+            }
+            if (entry->device.config.no_stat_dir) {
+                flags |= location::FsEntryFlag::FsEntryFlag_NoStatDir;
+            }
+
+            out.emplace_back(entry->mount, entry->name, flags);
         }
     }
 

sphaira/source/utils/devoptab_http.cpp

@@ -19,7 +19,7 @@
 namespace sphaira::devoptab {
 namespace {
 
-constexpr int DEFAULT_HTTP_TIMEOUT = 3000; // 3 seconds.
+constexpr long DEFAULT_HTTP_TIMEOUT = 3000; // 3 seconds.
 
 #define CURL_EASY_SETOPT_LOG(handle, opt, v) \
     if (auto r = curl_easy_setopt(handle, opt, v); r != CURLE_OK) { \
@@ -36,8 +36,10 @@ struct HttpMountConfig {
     std::string url{};
     std::string user{};
     std::string pass{};
-    std::optional<int> port{};
+    std::optional<long> port{};
-    int timeout{DEFAULT_HTTP_TIMEOUT};
+    long timeout{DEFAULT_HTTP_TIMEOUT};
+    bool no_stat_file{true};
+    bool no_stat_dir{true};
 };
 using HttpMountConfigs = std::vector<HttpMountConfig>;
 
@@ -127,8 +129,8 @@ std::string build_url(const std::string& base, const std::string& path, bool is_
 void http_set_common_options(Device& client, const std::string& url) {
     curl_easy_reset(client.curl);
     CURL_EASY_SETOPT_LOG(client.curl, CURLOPT_URL, url.c_str());
-    CURL_EASY_SETOPT_LOG(client.curl, CURLOPT_TIMEOUT, (long)client.config.timeout);
+    CURL_EASY_SETOPT_LOG(client.curl, CURLOPT_TIMEOUT, client.config.timeout);
-    CURL_EASY_SETOPT_LOG(client.curl, CURLOPT_CONNECTTIMEOUT, (long)client.config.timeout);
+    CURL_EASY_SETOPT_LOG(client.curl, CURLOPT_CONNECTTIMEOUT, client.config.timeout);
     CURL_EASY_SETOPT_LOG(client.curl, CURLOPT_AUTOREFERER, 1L);
     CURL_EASY_SETOPT_LOG(client.curl, CURLOPT_FOLLOWLOCATION, 1L);
     CURL_EASY_SETOPT_LOG(client.curl, CURLOPT_SSL_VERIFYPEER, 0L);
@@ -139,7 +141,7 @@ void http_set_common_options(Device& client, const std::string& url) {
     CURL_EASY_SETOPT_LOG(client.curl, CURLOPT_BUFFERSIZE, 1024L * 512L);
 
     if (client.config.port) {
-        CURL_EASY_SETOPT_LOG(client.curl, CURLOPT_PORT, (long)client.config.port.value());
+        CURL_EASY_SETOPT_LOG(client.curl, CURLOPT_PORT, client.config.port.value());
     }
 
     // enable all forms of compression supported by libcurl.
@@ -175,66 +177,74 @@ bool http_dirlist(Device& client, const std::string& path, DirEntries& out) {
         return false;
     }
 
-    chunk.emplace_back('\0'); // null-terminate the chunk
+    log_write("[HTTP] Received %zu bytes for directory listing\n", chunk.size());
-    const char* dilim = "<a href=\"";
-    const char* ptr = chunk.data();
 
-    // try and parse the href links.
+    SCOPED_TIMESTAMP("http_dirlist parse");
-    // it works with python http.serve, npm http-server and rclone http server.
-    while ((ptr = std::strstr(ptr, dilim))) {
-        // skip the delimiter.
-        ptr += std::strlen(dilim);
 
-        const auto href_begin = ptr;
+    // very fast/basic html parsing.
-        const auto href_end = std::strstr(href_begin, "\">");
+    // takes 17ms to parse 3MB html with 7641 entries.
-        if (!href_end) {
+    // todo: if i ever add an xml parser to sphaira, use that instead.
-            continue;
+    // todo: for the above, benchmark the parser to ensure its faster than the my code.
+    std::string_view chunk_view{chunk.data(), chunk.size()};
+    const auto table_start = chunk_view.find("<table");
+    const auto table_end = chunk_view.rfind("</table>");
+
+    if (table_start != std::string_view::npos && table_end != std::string_view::npos && table_end > table_start) {
+        const std::string_view href_tag_start = "<a href=\"";
+        const std::string_view href_tag_end = "\">";
+        const std::string_view anchor_tag_end = "</a>";
+        const auto table_view = chunk_view.substr(table_start, table_end - table_start);
+
+        size_t pos = 0;
+        out.reserve(10000);
+
+        for (;;) {
+            const auto href_pos = table_view.find(href_tag_start, pos);
+            if (href_pos == std::string_view::npos) {
+                break; // no more href.
+            }
+            pos = href_pos + href_tag_start.length();
+
+            const auto href_begin = pos;
+            const auto href_end = table_view.find(href_tag_end, href_begin);
+            if (href_end == std::string_view::npos) {
+                break; // no more href.
+            }
+
+            const auto name_begin = href_end + href_tag_end.length();
+            const auto name_end = table_view.find(anchor_tag_end, name_begin);
+            if (name_end == std::string_view::npos) {
+                break; // no more names.
+            }
+
+            pos = name_end + anchor_tag_end.length();
+            const auto href_view = table_view.substr(href_begin, href_end - href_begin);
+            const auto name_view = table_view.substr(name_begin, name_end - name_begin);
+
+            // skip empty names/links, root dir entry and links that are not actual files/dirs (e.g. sorting/filter controls).
+            if (name_view.empty() || href_view.empty() || name_view == "/" || href_view.starts_with('?') || href_view.starts_with('#')) {
+                continue;
+            }
+
+            // skip parent directory entry and external links.
+            if (href_view == ".." || name_view == ".." || href_view.starts_with("../") || name_view.starts_with("../") || href_view.find("://") != std::string::npos) {
+                continue;
+            }
+
+            std::string name{name_view};
+            const std::string href{href_view};
+
+            const auto is_dir = name.ends_with('/');
+            if (is_dir) {
+                name.pop_back(); // remove the trailing '/'
+            }
+
+            out.emplace_back(name, href, is_dir);
         }
-        const auto href_len = href_end - href_begin;
-
-        const auto name_begin = href_end + std::strlen("\">");
-        const auto name_end = std::strstr(name_begin, "</a>");
-        if (!name_end) {
-            continue;
-        }
-        const auto name_len = name_end - name_begin;
-
-        if (href_len <= 0 || name_len <= 0) {
-            continue;
-        }
-
-        // skip if inside <script> or <style> tags (simple check)
-        const auto script_tag = std::strstr(href_begin - 32, "<script");
-        const auto style_tag = std::strstr(href_begin - 32, "<style");
-        if ((script_tag && script_tag < href_begin) || (style_tag && style_tag < href_begin)) {
-            continue;
-        }
-
-        std::string href(href_begin, href_len);
-        std::string name(name_begin, name_len);
-
-        // skip parent directory entry and external links.
-        if (href == ".." || name == ".." || href.starts_with("../") || name.starts_with("../") || href.find("://") != std::string::npos) {
-            continue;
-        }
-
-        // skip links that are not actual files/dirs (e.g. sorting/filter controls)
-        if (href.starts_with('?')) {
-            continue;
-        }
-
-        if (name.empty() || href.empty() || name == "/" || href.starts_with('?') || href.starts_with('#')) {
-            continue;
-        }
-
-        const auto is_dir = name.ends_with('/');
-        if (is_dir) {
-            name.pop_back(); // remove the trailing '/'
-        }
-
-        out.emplace_back(name, href, is_dir);
     }
 
+    log_write("[HTTP] Parsed %zu entries from directory listing\n", out.size());
+
     return true;
 }
 
@@ -574,7 +584,11 @@ Result MountHttpAll() {
             // todo: idk what the default should be.
             e->back().port = ini_parse_getl(Value, 8000);
         } else if (!std::strcmp(Key, "timeout")) {
-            e->back().timeout = ini_parse_getl(Value, DEFAULT_HTTP_TIMEOUT);
+            e->back().timeout = ini_parse_getl(Value, e->back().timeout);
+        } else if (!std::strcmp(Key, "no_stat_file")) {
+            e->back().no_stat_file = ini_parse_getbool(Value, e->back().no_stat_file);
+        } else if (!std::strcmp(Key, "no_stat_dir")) {
+            e->back().no_stat_dir = ini_parse_getbool(Value, e->back().no_stat_dir);
         } else {
             log_write("[HTTP] INI: Unknown key %s=%s\n", Key, Value);
         }
@@ -646,7 +660,16 @@ Result GetHttpMounts(location::StdioEntries& out) {
 
     for (const auto& entry : g_entries) {
         if (entry) {
-            out.emplace_back(entry->mount, entry->name, true);
+            u32 flags = 0;
+            flags |= location::FsEntryFlag::FsEntryFlag_ReadOnly;
+            if (entry->device.config.no_stat_file) {
+                flags |= location::FsEntryFlag::FsEntryFlag_NoStatFile;
+            }
+            if (entry->device.config.no_stat_dir) {
+                flags |= location::FsEntryFlag::FsEntryFlag_NoStatDir;
+            }
+
+            out.emplace_back(entry->mount, entry->name, flags);
         }
     }
 

sphaira/source/utils/devoptab_nfs.cpp

@@ -28,6 +28,8 @@ struct NfsMountConfig {
     int version{DEFAULT_NFS_VERSION};
     int timeout{DEFAULT_NFS_TIMEOUT};
     bool read_only{};
+    bool no_stat_file{false};
+    bool no_stat_dir{true};
 };
 using NfsMountConfigs = std::vector<NfsMountConfig>;
 
@@ -581,15 +583,19 @@ Result MountNfsAll() {
         } else if (!std::strcmp(Key, "name")) {
             e->back().name = Value;
         } else if (!std::strcmp(Key, "uid")) {
-            e->back().uid = ini_parse_getl(Value, DEFAULT_NFS_UID);
+            e->back().uid = ini_parse_getl(Value, e->back().uid);
         } else if (!std::strcmp(Key, "gid")) {
-            e->back().gid = ini_parse_getl(Value, DEFAULT_NFS_GID);
+            e->back().gid = ini_parse_getl(Value, e->back().gid);
         } else if (!std::strcmp(Key, "version")) {
-            e->back().version = ini_parse_getl(Value, DEFAULT_NFS_VERSION);
+            e->back().version = ini_parse_getl(Value, e->back().version);
         } else if (!std::strcmp(Key, "timeout")) {
-            e->back().timeout = ini_parse_getl(Value, DEFAULT_NFS_TIMEOUT);
+            e->back().timeout = ini_parse_getl(Value, e->back().timeout);
         } else if (!std::strcmp(Key, "read_only")) {
-            e->back().read_only = ini_parse_getbool(Value, false);
+            e->back().read_only = ini_parse_getbool(Value, e->back().read_only);
+        } else if (!std::strcmp(Key, "no_stat_file")) {
+            e->back().no_stat_file = ini_parse_getbool(Value, e->back().no_stat_file);
+        } else if (!std::strcmp(Key, "no_stat_dir")) {
+            e->back().no_stat_dir = ini_parse_getbool(Value, e->back().no_stat_dir);
         } else {
             log_write("[NFS] INI: Unknown key %s=%s\n", Key, Value);
         }
@@ -663,7 +669,18 @@ Result GetNfsMounts(location::StdioEntries& out) {
 
     for (const auto& entry : g_entries) {
         if (entry) {
-            out.emplace_back(entry->mount, entry->name, entry->device.config.read_only);
+            u32 flags = 0;
+            if (entry->device.config.read_only) {
+                flags |= location::FsEntryFlag::FsEntryFlag_ReadOnly;
+            }
+            if (entry->device.config.no_stat_file) {
+                flags |= location::FsEntryFlag::FsEntryFlag_NoStatFile;
+            }
+            if (entry->device.config.no_stat_dir) {
+                flags |= location::FsEntryFlag::FsEntryFlag_NoStatDir;
+            }
+
+            out.emplace_back(entry->mount, entry->name, flags);
         }
     }
 

sphaira/source/utils/devoptab_smb2.cpp

@@ -27,6 +27,8 @@ struct Smb2MountConfig {
     std::string workstation{};
     int timeout{DEFAULT_SMB2_TIMEOUT};
     bool read_only{};
+    bool no_stat_file{false};
+    bool no_stat_dir{true};
 };
 using Smb2MountConfigs = std::vector<Smb2MountConfig>;
 
@@ -548,9 +550,13 @@ Result MountSmb2All() {
         } else if (!std::strcmp(Key, "workstation")) {
             e->back().workstation = Value;
         } else if (!std::strcmp(Key, "timeout")) {
-            e->back().timeout = ini_parse_getl(Value, DEFAULT_SMB2_TIMEOUT);
+            e->back().timeout = ini_parse_getl(Value, e->back().timeout);
         } else if (!std::strcmp(Key, "read_only")) {
-            e->back().read_only = ini_parse_getbool(Value, false);
+            e->back().read_only = ini_parse_getbool(Value, e->back().read_only);
+        } else if (!std::strcmp(Key, "no_stat_file")) {
+            e->back().no_stat_file = ini_parse_getbool(Value, e->back().no_stat_file);
+        } else if (!std::strcmp(Key, "no_stat_dir")) {
+            e->back().no_stat_dir = ini_parse_getbool(Value, e->back().no_stat_dir);
         } else {
             log_write("[SMB2] INI: Unknown key %s=%s\n", Key, Value);
         }
@@ -624,7 +630,18 @@ Result GetSmb2Mounts(location::StdioEntries& out) {
 
     for (const auto& entry : g_entries) {
         if (entry) {
-            out.emplace_back(entry->mount, entry->name, entry->device.config.read_only);
+            u32 flags = 0;
+            if (entry->device.config.read_only) {
+                flags |= location::FsEntryFlag::FsEntryFlag_ReadOnly;
+            }
+            if (entry->device.config.no_stat_file) {
+                flags |= location::FsEntryFlag::FsEntryFlag_NoStatFile;
+            }
+            if (entry->device.config.no_stat_dir) {
+                flags |= location::FsEntryFlag::FsEntryFlag_NoStatDir;
+            }
+
+            out.emplace_back(entry->mount, entry->name, flags);
         }
     }