OmniNX/sphaira
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

http: optimise the dir_list parsing, only parse tables. filebrowser: option to disable stat per fs.

Browse Source 
this improves network fs speed by disabling stat for http entriely, and only enabling file stat for everything else.
this can be overriden in the config.
This commit is contained in:
ITotalJustice
2025-09-05 14:10:06 +01:00
parent a4209961e2
commit 6ce566aea5
9 changed files with 191 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
sphaira/include/location.hpp
View File

@@ -3,9 +3,14 @@
#include <string>
#include <vector>
#include <switch.h>
// to import FsEntryFlags.
// todo: this should be part of a smaller header, such as filesystem_types.hpp
#include "ui/menus/filebrowser.hpp"
namespace sphaira::location {
using FsEntryFlag = ui::menu::filebrowser::FsEntryFlag;
struct Entry {
std::string name{};
std::string url{};
@@ -29,8 +34,8 @@ struct StdioEntry {
std::string mount{};
// ums0: (USB Flash Disk)
std::string name{};
// set if read-only.
bool write_protect;
// FsEntryFlag
u32 flags{};
};
using StdioEntries = std::vector<StdioEntry>;

23
sphaira/include/ui/menus/filebrowser.hpp
View File

@@ -39,7 +39,12 @@ enum FsEntryFlag {
// supports file assoc.
FsEntryFlag_Assoc = 1 << 1,
// this is an sd card, files can be launched from here.
FsEntryFlag_IsSd = 1 << 2,
FsEntryFlag_IsSd = 1 << 2, // todo: remove this.
// do not stat files in this entry (faster for network mount).
FsEntryFlag_NoStatFile = 1 << 3,
FsEntryFlag_NoStatDir = 1 << 4,
FsEntryFlag_NoRandomReads = 1 << 5,
FsEntryFlag_NoRandomWrites = 1 << 6,
};
enum class FsType {
@@ -90,6 +95,22 @@ struct FsEntry {
return flags & FsEntryFlag_IsSd;
}
auto IsNoStatFile() const -> bool {
return flags & FsEntryFlag_NoStatFile;
}
auto IsNoStatDir() const -> bool {
return flags & FsEntryFlag_NoStatDir;
}
auto IsNoRandomReads() const -> bool {
return flags & FsEntryFlag_NoRandomReads;
}
auto IsNoRandomWrites() const -> bool {
return flags & FsEntryFlag_NoRandomWrites;
}
auto IsSame(const FsEntry& e) const {
return root == e.root && type == e.type;
}

11
sphaira/source/location.cpp
View File

@@ -82,7 +82,7 @@ auto GetStdio(bool write) -> StdioEntries {
const auto add_from_entries = [](const StdioEntries& entries, StdioEntries& out, bool write) {
for (const auto& e : entries) {
if (write && e.write_protect) {
if (write && (e.flags & FsEntryFlag::FsEntryFlag_ReadOnly)) {
log_write("[STDIO] skipping read only mount: %s\n", e.name.c_str());
continue;
}
@@ -155,7 +155,12 @@ auto GetStdio(bool write) -> StdioEntries {
char display_name[0x100];
std::snprintf(display_name, sizeof(display_name), "%s (%s - %s - %zu GB)", e.name, LIBUSBHSFS_FS_TYPE_STR(e.fs_type), e.product_name, e.capacity / 1024 / 1024 / 1024);
out.emplace_back(e.name, display_name, e.write_protect);
u32 flags = 0;
if (e.write_protect || (e.flags & UsbHsFsMountFlags_ReadOnly)) {
flags |= FsEntryFlag::FsEntryFlag_ReadOnly;
}
out.emplace_back(e.name, display_name, flags);
log_write("\t[USBHSFS] %s name: %s serial: %s man: %s\n", e.name, e.product_name, e.serial_number, e.manufacturer);
}
@@ -172,7 +177,7 @@ auto GetFat() -> StdioEntries {
char display_name[0x100];
std::snprintf(display_name, sizeof(display_name), "%s (Read Only)", path);
out.emplace_back(path, display_name, true);
out.emplace_back(path, display_name, FsEntryFlag::FsEntryFlag_ReadOnly);
}
return out;

19
sphaira/source/ui/menus/filebrowser.cpp
View File

@@ -591,8 +591,7 @@ void FsView::Draw(NVGcontext* vg, Theme* theme) {
m_scroll_name.Draw(vg, selected, x + text_xoffset+65, y + (h / 2.f), w-(75+text_xoffset+65+50), 20, NVG_ALIGN_LEFT | NVG_ALIGN_MIDDLE, theme->GetColour(text_id), e.name);
// NOTE: make this native only if i disable dir scan from above.
if (e.IsDir()) {
if (e.IsDir() && !m_fs_entry.IsNoStatDir() && (e.dir_count != -1 || !e.done_stat)) {
// NOTE: this takes longer than 16ms when opening a new folder due to it
// checking all 9 folders at once.
if (!got_dir_count && !e.done_stat && e.file_count == -1 && e.dir_count == -1) {
@@ -607,7 +606,7 @@ void FsView::Draw(NVGcontext* vg, Theme* theme) {
if (e.dir_count != -1) {
gfx::drawTextArgs(vg, x + w - text_xoffset, y + (h / 2.f) + 3, 16.f, NVG_ALIGN_RIGHT | NVG_ALIGN_TOP, theme->GetColour(text_id), "%zd dirs"_i18n.c_str(), e.dir_count);
}
} else if (e.IsFile()) {
} else if (e.IsFile() && !m_fs_entry.IsNoStatFile() && (e.file_size != -1 || !e.time_stamp.is_valid)) {
if (!e.time_stamp.is_valid && !e.done_stat) {
e.done_stat = true;
const auto path = GetNewPath(e);
@@ -1706,12 +1705,7 @@ void FsView::DisplayOptions() {
const auto stdio_locations = location::GetStdio(false);
for (const auto& e: stdio_locations) {
u32 flags{};
if (e.write_protect) {
flags |= FsEntryFlag_ReadOnly;
}
fs_entries.emplace_back(e.name, e.mount, FsType::Stdio, flags);
fs_entries.emplace_back(e.name, e.mount, FsType::Stdio, e.flags);
mount_items.push_back(e.name);
}
@@ -1727,12 +1721,7 @@ void FsView::DisplayOptions() {
const auto fat_entries = location::GetFat();
for (const auto& e: fat_entries) {
u32 flags{};
if (e.write_protect) {
flags |= FsEntryFlag_ReadOnly;
}
fs_entries.emplace_back(e.name, e.mount, FsType::Stdio, flags);
fs_entries.emplace_back(e.name, e.mount, FsType::Stdio, e.flags);
mount_items.push_back(e.name);
}

2
sphaira/source/usbdvd.cpp
View File

@@ -65,7 +65,7 @@ bool GetMountPoint(location::StdioEntry& out) {
out.mount = fs.mountpoint;
out.name = display_name;
out.write_protect = true;
out.flags = location::FsEntryFlag::FsEntryFlag_ReadOnly;
return true;
}

23
sphaira/source/utils/devoptab_ftp.cpp
View File

@@ -42,6 +42,8 @@ struct FtpMountConfig {
std::optional<long> port{};
long timeout{DEFAULT_FTP_TIMEOUT};
bool read_only{};
bool no_stat_file{false};
bool no_stat_dir{true};
};
using FtpMountConfigs = std::vector<FtpMountConfig>;
@@ -914,9 +916,13 @@ Result MountFtpAll() {
} else if (!std::strcmp(Key, "port")) {
e->back().port = ini_parse_getl(Value, DEFAULT_FTP_PORT);
} else if (!std::strcmp(Key, "timeout")) {
e->back().timeout = ini_parse_getl(Value, DEFAULT_FTP_TIMEOUT);
e->back().timeout = ini_parse_getl(Value, e->back().timeout);
} else if (!std::strcmp(Key, "read_only")) {
e->back().read_only = ini_parse_getbool(Value, false);
e->back().read_only = ini_parse_getbool(Value, e->back().read_only);
} else if (!std::strcmp(Key, "no_stat_file")) {
e->back().no_stat_file = ini_parse_getbool(Value, e->back().no_stat_file);
} else if (!std::strcmp(Key, "no_stat_dir")) {
e->back().no_stat_dir = ini_parse_getbool(Value, e->back().no_stat_dir);
} else {
log_write("[FTP] INI: Unknown key %s=%s\n", Key, Value);
}
@@ -989,7 +995,18 @@ Result GetFtpMounts(location::StdioEntries& out) {
for (const auto& entry : g_entries) {
if (entry) {
out.emplace_back(entry->mount, entry->name, entry->device.config.read_only);
u32 flags = 0;
if (entry->device.config.read_only) {
flags |= location::FsEntryFlag::FsEntryFlag_ReadOnly;
}
if (entry->device.config.no_stat_file) {
flags |= location::FsEntryFlag::FsEntryFlag_NoStatFile;
}
if (entry->device.config.no_stat_dir) {
flags |= location::FsEntryFlag::FsEntryFlag_NoStatDir;
}
out.emplace_back(entry->mount, entry->name, flags);
}
}

149
sphaira/source/utils/devoptab_http.cpp
View File

@@ -19,7 +19,7 @@
namespace sphaira::devoptab {
namespace {
constexpr int DEFAULT_HTTP_TIMEOUT = 3000; // 3 seconds.
constexpr long DEFAULT_HTTP_TIMEOUT = 3000; // 3 seconds.
#define CURL_EASY_SETOPT_LOG(handle, opt, v) \
if (auto r = curl_easy_setopt(handle, opt, v); r != CURLE_OK) { \
@@ -36,8 +36,10 @@ struct HttpMountConfig {
std::string url{};
std::string user{};
std::string pass{};
std::optional<int> port{};
int timeout{DEFAULT_HTTP_TIMEOUT};
std::optional<long> port{};
long timeout{DEFAULT_HTTP_TIMEOUT};
bool no_stat_file{true};
bool no_stat_dir{true};
};
using HttpMountConfigs = std::vector<HttpMountConfig>;
@@ -127,8 +129,8 @@ std::string build_url(const std::string& base, const std::string& path, bool is_
void http_set_common_options(Device& client, const std::string& url) {
curl_easy_reset(client.curl);
CURL_EASY_SETOPT_LOG(client.curl, CURLOPT_URL, url.c_str());
CURL_EASY_SETOPT_LOG(client.curl, CURLOPT_TIMEOUT, (long)client.config.timeout);
CURL_EASY_SETOPT_LOG(client.curl, CURLOPT_CONNECTTIMEOUT, (long)client.config.timeout);
CURL_EASY_SETOPT_LOG(client.curl, CURLOPT_TIMEOUT, client.config.timeout);
CURL_EASY_SETOPT_LOG(client.curl, CURLOPT_CONNECTTIMEOUT, client.config.timeout);
CURL_EASY_SETOPT_LOG(client.curl, CURLOPT_AUTOREFERER, 1L);
CURL_EASY_SETOPT_LOG(client.curl, CURLOPT_FOLLOWLOCATION, 1L);
CURL_EASY_SETOPT_LOG(client.curl, CURLOPT_SSL_VERIFYPEER, 0L);
@@ -139,7 +141,7 @@ void http_set_common_options(Device& client, const std::string& url) {
CURL_EASY_SETOPT_LOG(client.curl, CURLOPT_BUFFERSIZE, 1024L * 512L);
if (client.config.port) {
CURL_EASY_SETOPT_LOG(client.curl, CURLOPT_PORT, (long)client.config.port.value());
CURL_EASY_SETOPT_LOG(client.curl, CURLOPT_PORT, client.config.port.value());
}
// enable all forms of compression supported by libcurl.
@@ -175,66 +177,74 @@ bool http_dirlist(Device& client, const std::string& path, DirEntries& out) {
return false;
}
chunk.emplace_back('\0'); // null-terminate the chunk
const char* dilim = "<a href=\"";
const char* ptr = chunk.data();
log_write("[HTTP] Received %zu bytes for directory listing\n", chunk.size());
// try and parse the href links.
// it works with python http.serve, npm http-server and rclone http server.
while ((ptr = std::strstr(ptr, dilim))) {
// skip the delimiter.
ptr += std::strlen(dilim);
SCOPED_TIMESTAMP("http_dirlist parse");
const auto href_begin = ptr;
const auto href_end = std::strstr(href_begin, "\">");
if (!href_end) {
continue;
// very fast/basic html parsing.
// takes 17ms to parse 3MB html with 7641 entries.
// todo: if i ever add an xml parser to sphaira, use that instead.
// todo: for the above, benchmark the parser to ensure its faster than the my code.
std::string_view chunk_view{chunk.data(), chunk.size()};
const auto table_start = chunk_view.find("<table");
const auto table_end = chunk_view.rfind("</table>");
if (table_start != std::string_view::npos && table_end != std::string_view::npos && table_end > table_start) {
const std::string_view href_tag_start = "<a href=\"";
const std::string_view href_tag_end = "\">";
const std::string_view anchor_tag_end = "</a>";
const auto table_view = chunk_view.substr(table_start, table_end - table_start);
size_t pos = 0;
out.reserve(10000);
for (;;) {
const auto href_pos = table_view.find(href_tag_start, pos);
if (href_pos == std::string_view::npos) {
break; // no more href.
}
pos = href_pos + href_tag_start.length();
const auto href_begin = pos;
const auto href_end = table_view.find(href_tag_end, href_begin);
if (href_end == std::string_view::npos) {
break; // no more href.
}
const auto name_begin = href_end + href_tag_end.length();
const auto name_end = table_view.find(anchor_tag_end, name_begin);
if (name_end == std::string_view::npos) {
break; // no more names.
}
pos = name_end + anchor_tag_end.length();
const auto href_view = table_view.substr(href_begin, href_end - href_begin);
const auto name_view = table_view.substr(name_begin, name_end - name_begin);
// skip empty names/links, root dir entry and links that are not actual files/dirs (e.g. sorting/filter controls).
if (name_view.empty() || href_view.empty() || name_view == "/" || href_view.starts_with('?') || href_view.starts_with('#')) {
continue;
}
// skip parent directory entry and external links.
if (href_view == ".." || name_view == ".." || href_view.starts_with("../") || name_view.starts_with("../") || href_view.find("://") != std::string::npos) {
continue;
}
std::string name{name_view};
const std::string href{href_view};
const auto is_dir = name.ends_with('/');
if (is_dir) {
name.pop_back(); // remove the trailing '/'
}
out.emplace_back(name, href, is_dir);
}
const auto href_len = href_end - href_begin;
const auto name_begin = href_end + std::strlen("\">");
const auto name_end = std::strstr(name_begin, "</a>");
if (!name_end) {
continue;
}
const auto name_len = name_end - name_begin;
if (href_len <= 0 || name_len <= 0) {
continue;
}
// skip if inside <script> or <style> tags (simple check)
const auto script_tag = std::strstr(href_begin - 32, "<script");
const auto style_tag = std::strstr(href_begin - 32, "<style");
if ((script_tag && script_tag < href_begin) || (style_tag && style_tag < href_begin)) {
continue;
}
std::string href(href_begin, href_len);
std::string name(name_begin, name_len);
// skip parent directory entry and external links.
if (href == ".." || name == ".." || href.starts_with("../") || name.starts_with("../") || href.find("://") != std::string::npos) {
continue;
}
// skip links that are not actual files/dirs (e.g. sorting/filter controls)
if (href.starts_with('?')) {
continue;
}
if (name.empty() || href.empty() || name == "/" || href.starts_with('?') || href.starts_with('#')) {
continue;
}
const auto is_dir = name.ends_with('/');
if (is_dir) {
name.pop_back(); // remove the trailing '/'
}
out.emplace_back(name, href, is_dir);
}
log_write("[HTTP] Parsed %zu entries from directory listing\n", out.size());
return true;
}
@@ -574,7 +584,11 @@ Result MountHttpAll() {
// todo: idk what the default should be.
e->back().port = ini_parse_getl(Value, 8000);
} else if (!std::strcmp(Key, "timeout")) {
e->back().timeout = ini_parse_getl(Value, DEFAULT_HTTP_TIMEOUT);
e->back().timeout = ini_parse_getl(Value, e->back().timeout);
} else if (!std::strcmp(Key, "no_stat_file")) {
e->back().no_stat_file = ini_parse_getbool(Value, e->back().no_stat_file);
} else if (!std::strcmp(Key, "no_stat_dir")) {
e->back().no_stat_dir = ini_parse_getbool(Value, e->back().no_stat_dir);
} else {
log_write("[HTTP] INI: Unknown key %s=%s\n", Key, Value);
}
@@ -646,7 +660,16 @@ Result GetHttpMounts(location::StdioEntries& out) {
for (const auto& entry : g_entries) {
if (entry) {
out.emplace_back(entry->mount, entry->name, true);
u32 flags = 0;
flags |= location::FsEntryFlag::FsEntryFlag_ReadOnly;
if (entry->device.config.no_stat_file) {
flags |= location::FsEntryFlag::FsEntryFlag_NoStatFile;
}
if (entry->device.config.no_stat_dir) {
flags |= location::FsEntryFlag::FsEntryFlag_NoStatDir;
}
out.emplace_back(entry->mount, entry->name, flags);
}
}

29
sphaira/source/utils/devoptab_nfs.cpp
View File

@@ -28,6 +28,8 @@ struct NfsMountConfig {
int version{DEFAULT_NFS_VERSION};
int timeout{DEFAULT_NFS_TIMEOUT};
bool read_only{};
bool no_stat_file{false};
bool no_stat_dir{true};
};
using NfsMountConfigs = std::vector<NfsMountConfig>;
@@ -581,15 +583,19 @@ Result MountNfsAll() {
} else if (!std::strcmp(Key, "name")) {
e->back().name = Value;
} else if (!std::strcmp(Key, "uid")) {
e->back().uid = ini_parse_getl(Value, DEFAULT_NFS_UID);
e->back().uid = ini_parse_getl(Value, e->back().uid);
} else if (!std::strcmp(Key, "gid")) {
e->back().gid = ini_parse_getl(Value, DEFAULT_NFS_GID);
e->back().gid = ini_parse_getl(Value, e->back().gid);
} else if (!std::strcmp(Key, "version")) {
e->back().version = ini_parse_getl(Value, DEFAULT_NFS_VERSION);
e->back().version = ini_parse_getl(Value, e->back().version);
} else if (!std::strcmp(Key, "timeout")) {
e->back().timeout = ini_parse_getl(Value, DEFAULT_NFS_TIMEOUT);
e->back().timeout = ini_parse_getl(Value, e->back().timeout);
} else if (!std::strcmp(Key, "read_only")) {
e->back().read_only = ini_parse_getbool(Value, false);
e->back().read_only = ini_parse_getbool(Value, e->back().read_only);
} else if (!std::strcmp(Key, "no_stat_file")) {
e->back().no_stat_file = ini_parse_getbool(Value, e->back().no_stat_file);
} else if (!std::strcmp(Key, "no_stat_dir")) {
e->back().no_stat_dir = ini_parse_getbool(Value, e->back().no_stat_dir);
} else {
log_write("[NFS] INI: Unknown key %s=%s\n", Key, Value);
}
@@ -663,7 +669,18 @@ Result GetNfsMounts(location::StdioEntries& out) {
for (const auto& entry : g_entries) {
if (entry) {
out.emplace_back(entry->mount, entry->name, entry->device.config.read_only);
u32 flags = 0;
if (entry->device.config.read_only) {
flags |= location::FsEntryFlag::FsEntryFlag_ReadOnly;
}
if (entry->device.config.no_stat_file) {
flags |= location::FsEntryFlag::FsEntryFlag_NoStatFile;
}
if (entry->device.config.no_stat_dir) {
flags |= location::FsEntryFlag::FsEntryFlag_NoStatDir;
}
out.emplace_back(entry->mount, entry->name, flags);
}
}

23
sphaira/source/utils/devoptab_smb2.cpp
View File

@@ -27,6 +27,8 @@ struct Smb2MountConfig {
std::string workstation{};
int timeout{DEFAULT_SMB2_TIMEOUT};
bool read_only{};
bool no_stat_file{false};
bool no_stat_dir{true};
};
using Smb2MountConfigs = std::vector<Smb2MountConfig>;
@@ -548,9 +550,13 @@ Result MountSmb2All() {
} else if (!std::strcmp(Key, "workstation")) {
e->back().workstation = Value;
} else if (!std::strcmp(Key, "timeout")) {
e->back().timeout = ini_parse_getl(Value, DEFAULT_SMB2_TIMEOUT);
e->back().timeout = ini_parse_getl(Value, e->back().timeout);
} else if (!std::strcmp(Key, "read_only")) {
e->back().read_only = ini_parse_getbool(Value, false);
e->back().read_only = ini_parse_getbool(Value, e->back().read_only);
} else if (!std::strcmp(Key, "no_stat_file")) {
e->back().no_stat_file = ini_parse_getbool(Value, e->back().no_stat_file);
} else if (!std::strcmp(Key, "no_stat_dir")) {
e->back().no_stat_dir = ini_parse_getbool(Value, e->back().no_stat_dir);
} else {
log_write("[SMB2] INI: Unknown key %s=%s\n", Key, Value);
}
@@ -624,7 +630,18 @@ Result GetSmb2Mounts(location::StdioEntries& out) {
for (const auto& entry : g_entries) {
if (entry) {
out.emplace_back(entry->mount, entry->name, entry->device.config.read_only);
u32 flags = 0;
if (entry->device.config.read_only) {
flags |= location::FsEntryFlag::FsEntryFlag_ReadOnly;
}
if (entry->device.config.no_stat_file) {
flags |= location::FsEntryFlag::FsEntryFlag_NoStatFile;
}
if (entry->device.config.no_stat_dir) {
flags |= location::FsEntryFlag::FsEntryFlag_NoStatDir;
}
out.emplace_back(entry->mount, entry->name, flags);
}
}