diff --git a/assets/romfs/github/ftpsrv.json b/assets/romfs/github/ftpsrv.json index 2ca701c..2333634 100644 --- a/assets/romfs/github/ftpsrv.json +++ b/assets/romfs/github/ftpsrv.json @@ -1,6 +1,6 @@ { - "name": "ftpsrv", - "url": "https://github.com/ITotalJustice/ftpsrv", + "owner": "ITotalJustice", + "repo": "ftpsrv", "assets": [ { "name": "switch_application.zip" diff --git a/assets/romfs/github/sphaira.json b/assets/romfs/github/sphaira.json index eca8a60..01274b6 100644 --- a/assets/romfs/github/sphaira.json +++ b/assets/romfs/github/sphaira.json @@ -1,9 +1,4 @@ { - "name": "sphaira", - "url": "https://github.com/ITotalJustice/sphaira", - "assets":[ - { - "name": "sphaira.zip" - } - ] + "owner": "ITotalJustice", + "repo": "sphaira" } diff --git a/assets/romfs/github/untitled.json b/assets/romfs/github/untitled.json index 5ce575f..b299b63 100644 --- a/assets/romfs/github/untitled.json +++ b/assets/romfs/github/untitled.json @@ -1,9 +1,4 @@ { - "name": "untitled", - "url": "https://github.com/ITotalJustice/untitled", - "assets":[ - { - "name": "untitled.zip" - } - ] + "owner": "ITotalJustice", + "repo": "untitled" } diff --git a/sphaira/CMakeLists.txt b/sphaira/CMakeLists.txt index 3bb168e..e27b2dc 100644 --- a/sphaira/CMakeLists.txt +++ b/sphaira/CMakeLists.txt @@ -140,7 +140,7 @@ set(NANOVG_STBI_STATIC OFF) set(NANOVG_STBTT_STATIC ON) set(YYJSON_DISABLE_READER OFF) -set(YYJSON_DISABLE_WRITER ON) +set(YYJSON_DISABLE_WRITER OFF) set(YYJSON_DISABLE_UTILS ON) set(YYJSON_DISABLE_FAST_FP_CONV ON) set(YYJSON_DISABLE_NON_STANDARD ON) diff --git a/sphaira/include/download.hpp b/sphaira/include/download.hpp index ec9c176..ab5fb4c 100644 --- a/sphaira/include/download.hpp +++ b/sphaira/include/download.hpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include namespace sphaira::curl { @@ -15,9 +15,11 @@ enum class Priority { High, // gets pushed to the front of the queue }; +struct Api; +struct ApiResult; + using Path = fs::FsPath; -using Header = std::unordered_map; -using OnComplete = std::function& data, bool success, long code)>; +using OnComplete = std::function; using OnProgress = std::function; struct Url { @@ -32,29 +34,42 @@ struct Fields { std::string m_str; }; -struct Api; +struct Header { + Header() = default; + Header(std::initializer_list> p) : m_map{p} {} + std::unordered_map m_map; + + auto Find(const std::string& key) const { + return std::find_if(m_map.cbegin(), m_map.cend(), [&key](auto& e) { + return !strcasecmp(key.c_str(), e.first.c_str()); + }); + } +}; + +struct ApiResult { + bool success; + long code; + Header header; // returned headers in request + std::vector data; // empty if downloaded a file + fs::FsPath path; // empty if downloaded memory +}; struct DownloadEventData { OnComplete callback; - std::vector data; - long code; - bool result; + ApiResult result; }; auto Init() -> bool; void Exit(); // sync functions -auto ToMemory(const Api& e) -> std::vector; -auto ToFile(const Api& e) -> bool; +auto ToMemory(const Api& e) -> ApiResult; +auto ToFile(const Api& e) -> ApiResult; // async functions auto ToMemoryAsync(const Api& e) -> bool; auto ToFileAsync(const Api& e) -> bool; -// removes url from cache (todo: deprecate this) -void ClearCache(const Url& url); - struct Api { Api() = default; @@ -119,7 +134,7 @@ struct Api { Path m_path{}; OnComplete m_on_complete = nullptr; OnProgress m_on_progress = nullptr; - Priority m_prio = Priority::Normal; + Priority m_prio = Priority::High; private: void SetOption(Url&& v) { @@ -156,4 +171,18 @@ private: } }; +namespace cache { + +bool init(); +void exit(); + +auto etag_get(const fs::FsPath& path) -> std::string; +void etag_set(const fs::FsPath& path, const std::string& value); +void etag_set(const fs::FsPath& path, const Header& value); + +auto lmt_get(const fs::FsPath& path) -> std::string; +void lmt_set(const fs::FsPath& path, const std::string& value); +void lmt_set(const fs::FsPath& path, const Header& value); + +} // namespace cache } // namespace sphaira::curl diff --git a/sphaira/include/ui/menus/appstore.hpp b/sphaira/include/ui/menus/appstore.hpp index ddfaa43..d31c48c 100644 --- a/sphaira/include/ui/menus/appstore.hpp +++ b/sphaira/include/ui/menus/appstore.hpp @@ -27,6 +27,8 @@ struct LazyImage { ~LazyImage(); int image{}; int w{}, h{}; + bool tried_cache{}; + bool cached{}; ImageDownloadState state{ImageDownloadState::None}; u8 first_pixel[4]{}; }; diff --git a/sphaira/include/ui/menus/ghdl.hpp b/sphaira/include/ui/menus/ghdl.hpp index a21bbeb..b903733 100644 --- a/sphaira/include/ui/menus/ghdl.hpp +++ b/sphaira/include/ui/menus/ghdl.hpp @@ -15,8 +15,8 @@ struct AssetEntry { struct Entry { fs::FsPath json_path; - std::string name; - std::string url; + std::string owner; + std::string repo; std::vector assets; }; diff --git a/sphaira/include/ui/menus/main_menu.hpp b/sphaira/include/ui/menus/main_menu.hpp index 9cedb0b..3b4d759 100644 --- a/sphaira/include/ui/menus/main_menu.hpp +++ b/sphaira/include/ui/menus/main_menu.hpp @@ -28,6 +28,10 @@ struct MainMenu final : Widget { void OnFocusGained() override; void OnFocusLost() override; + auto IsMenu() const -> bool override { + return true; + } + private: void OnLRPress(std::shared_ptr menu, Button b); void AddOnLPress(); diff --git a/sphaira/include/ui/menus/menu_base.hpp b/sphaira/include/ui/menus/menu_base.hpp index 2deec84..6d0b058 100644 --- a/sphaira/include/ui/menus/menu_base.hpp +++ b/sphaira/include/ui/menus/menu_base.hpp @@ -12,6 +12,11 @@ struct MenuBase : Widget { virtual void Update(Controller* controller, TouchInfo* touch); virtual void Draw(NVGcontext* vg, Theme* theme); + + auto IsMenu() const -> bool override { + return true; + } + void SetTitle(std::string title); void SetTitleSubHeading(std::string sub_heading); void SetSubHeading(std::string sub_heading); diff --git a/sphaira/include/ui/menus/themezer.hpp b/sphaira/include/ui/menus/themezer.hpp index fa5bba9..c13c3ce 100644 --- a/sphaira/include/ui/menus/themezer.hpp +++ b/sphaira/include/ui/menus/themezer.hpp @@ -15,28 +15,14 @@ enum class ImageDownloadState { }; struct LazyImage { - LazyImage() = default; ~LazyImage(); int image{}; int w{}, h{}; + bool tried_cache{}; + bool cached{}; ImageDownloadState state{ImageDownloadState::None}; - u8 first_pixel[4]{}; }; -// "mutation setLike($type: String!, $id: String!, $value: Boolean!) {\n setLike(type: $type, id: $id, value: $value)\n}\n" - -// https://api.themezer.net/?query=query($nsfw:Boolean,$target:String,$page:Int,$limit:Int,$sort:String,$order:String,$query:String){themeList(nsfw:$nsfw,target:$target,page:$page,limit:$limit,sort:$sort,order:$order,query:$query){id,creator{id,display_name},details{name,description},last_updated,dl_count,like_count,target,preview{original,thumb}}}&variables={"nsfw":false,"target":null,"page":1,"limit":10,"sort":"updated","order":"desc","query":null} -// https://api.themezer.net/?query=query($nsfw:Boolean,$page:Int,$limit:Int,$sort:String,$order:String,$query:String){packList(nsfw:$nsfw,page:$page,limit:$limit,sort:$sort,order:$order,query:$query){id,creator{id,display_name},details{name,description},last_updated,dl_count,like_count,themes{id,creator{display_name},details{name,description},last_updated,dl_count,like_count,target,preview{original,thumb}}}}&variables={"nsfw":false,"page":1,"limit":10,"sort":"updated","order":"desc","query":null} -// https://api.themezer.net/?query=query($id:String!){pack(id:$id){id,creator{display_name},details{name,description},last_updated,categories,dl_count,like_count,themes{id,details{name},layout{id,details{name}},categories,target,preview{original,thumb},last_updated,dl_count,like_count}}}&variables={"id":"16d"} - -// https://api.themezer.net/?query=query{nxinstaller(id:"t9a6"){themes{filename,url,mimetype}}} -// https://api.themezer.net/?query=query{downloadTheme(id:"t9a6"){filename,url,mimetype}} -// https://api.themezer.net/?query=query{downloadPack(id:"t9a6"){filename,url,mimetype}} - -// {"data":{"setLike":true}} -// https://api.themezer.net/?query=mutation{setLike(type:"packs",id:"5",value:true){data{setLike}}} -// https://api.themezer.net/?query=mutation($type:String!,$id:String!,$value:Boolean!){setLike(type:$type,id:$id,value:$value){data{setLike}}}&variables={"type":"packs","id":"5","value":true} - enum MenuState { MenuState_Normal, MenuState_Search, @@ -55,6 +41,11 @@ enum class PageLoadState { Error, }; +// all commented out entries are those that we don't query for. +// this saves time not only processing the json, but also the download +// of said json. +// by reducing the fields to only what we need, the size is 4-5x smaller. + struct Creator { std::string id; std::string display_name; @@ -62,11 +53,11 @@ struct Creator { struct Details { std::string name; - std::string description; + // std::string description; }; struct Preview { - std::string original; + // std::string original; std::string thumb; LazyImage lazy_image; }; @@ -81,13 +72,13 @@ using DownloadTheme = DownloadPack; struct ThemeEntry { std::string id; - Creator creator; - Details details; - std::string last_updated; - u64 dl_count; - u64 like_count; - std::vector categories; - std::string target; + // Creator creator; + // Details details; + // std::string last_updated; + // u64 dl_count; + // u64 like_count; + // std::vector categories; + // std::string target; Preview preview; }; @@ -106,10 +97,10 @@ struct PackListEntry { std::string id; Creator creator; Details details; - std::string last_updated; - std::vector categories; - u64 dl_count; - u64 like_count; + // std::string last_updated; + // std::vector categories; + // u64 dl_count; + // u64 like_count; std::vector themes; }; diff --git a/sphaira/include/ui/types.hpp b/sphaira/include/ui/types.hpp index 4eae5ea..b291867 100644 --- a/sphaira/include/ui/types.hpp +++ b/sphaira/include/ui/types.hpp @@ -117,12 +117,27 @@ struct TimeStamp { start = armGetSystemTick(); } - auto GetNs() -> u64 { + auto GetNs() const -> u64 { const auto end_ticks = armGetSystemTick(); return armTicksToNs(end_ticks) - armTicksToNs(start); } - auto GetSeconds() -> double { + auto GetMs() const -> u64 { + const auto ns = GetNs(); + return ns/1000/1000; + } + + auto GetSeconds() const -> u64 { + const auto ns = GetNs(); + return ns/1000/1000/1000; + } + + auto GetMsD() const -> double { + const double ns = GetNs(); + return ns/1000.0/1000.0; + } + + auto GetSecondsD() const -> double { const double ns = GetNs(); return ns/1000.0/1000.0/1000.0; } diff --git a/sphaira/include/ui/widget.hpp b/sphaira/include/ui/widget.hpp index 381d0de..9f1bc32 100644 --- a/sphaira/include/ui/widget.hpp +++ b/sphaira/include/ui/widget.hpp @@ -26,6 +26,10 @@ struct Widget : public Object { return m_focus; } + virtual auto IsMenu() const -> bool { + return false; + } + auto HasAction(Button button) const -> bool; void SetAction(Button button, Action action); void SetActions(std::same_as> auto ...args) { diff --git a/sphaira/source/app.cpp b/sphaira/source/app.cpp index a3c6d82..53b7592 100644 --- a/sphaira/source/app.cpp +++ b/sphaira/source/app.cpp @@ -228,9 +228,22 @@ void App::Loop() { ui::gfx::updateHighlightAnimation(); - auto events = evman::popall(); - // while (auto e = evman::pop()) { - for (auto& e : events) { + // fire all events in in a 3ms timeslice + TimeStamp ts_event; + const u64 event_timeout = 3; + + // limit events to a max per frame in order to not block for too long. + while (true) { + if (ts_event.GetMs() >= event_timeout) { + log_write("event loop timed-out\n"); + break; + } + + auto event = evman::pop(); + if (!event.has_value()) { + break; + } + std::visit([this](auto&& arg){ using T = std::decay_t; if constexpr(std::is_same_v) { @@ -278,11 +291,11 @@ void App::Loop() { } } else if constexpr(std::is_same_v) { log_write("[DownloadEventData] got event\n"); - arg.callback(arg.data, arg.result, arg.code); + arg.callback(arg.result); } else { static_assert(false, "non-exhaustive visitor!"); } - }, e); + }, event.value()); } u32 w{},h{}; @@ -663,10 +676,29 @@ void App::Draw() { nvgBeginFrame(this->vg, s_width, s_height, 1.f); nvgScale(vg, m_scale.x, m_scale.y); - // NOTE: widgets should never pop themselves from drawing! - for (auto& p : m_widgets) { - if (!p->IsHidden()) { - p->Draw(vg, &m_theme); + // find the last menu in the list, start drawing from there + auto menu_it = m_widgets.rend(); + for (auto it = m_widgets.rbegin(); it != m_widgets.rend(); it++) { + const auto& p = *it; + if (!p->IsHidden() && p->IsMenu()) { + menu_it = it; + break; + } + } + + // reverse itr so loop backwards to go forwarders. + if (menu_it != m_widgets.rend()) { + for (auto it = menu_it; ; it--) { + const auto& p = *it; + + // draw everything not hidden on top of the menu. + if (!p->IsHidden()) { + p->Draw(vg, &m_theme); + } + + if (it == m_widgets.rbegin()) { + break; + } } } @@ -942,6 +974,7 @@ App::App(const char* argv0) { } curl::Init(); + curl::cache::init(); // Create the deko3d device this->device = dk::DeviceMaker{} @@ -1097,6 +1130,7 @@ App::~App() { i18n::exit(); curl::Exit(); + curl::cache::exit(); // this has to be called before any cleanup to ensure the lifetime of // nvg is still active as some widgets may need to free images. diff --git a/sphaira/source/download.cpp b/sphaira/source/download.cpp index b7a78e0..c04092e 100644 --- a/sphaira/source/download.cpp +++ b/sphaira/source/download.cpp @@ -10,12 +10,11 @@ #include #include #include +#include namespace sphaira::curl { namespace { -using DownloadResult = std::pair; - #define CURL_EASY_SETOPT_LOG(handle, opt, v) \ if (auto r = curl_easy_setopt(handle, opt, v); r != CURLE_OK) { \ log_write("curl_easy_setopt(%s, %s) msg: %s\n", #opt, #v, curl_easy_strerror(r)); \ @@ -40,48 +39,134 @@ std::atomic_bool g_running{}; CURLSH* g_curl_share{}; Mutex g_mutex_share[CURL_LOCK_DATA_LAST]{}; -struct UrlCache { - auto AddToCache(const Url& url, bool force = false) { - mutexLock(&mutex); - ON_SCOPE_EXIT(mutexUnlock(&mutex)); - auto it = std::find_if(cache.cbegin(), cache.cend(), [&url](const auto& e){ - return e.m_str == url.m_str; - }); +struct DataStruct { + std::vector data; + u64 offset{}; + FsFile f{}; + s64 file_offset{}; +}; - if (it == cache.cend()) { - cache.emplace_back(url); +auto generate_key_from_path(const fs::FsPath& path) -> std::string { + const auto key = crc32Calculate(path.s, path.size()); + return std::to_string(key); +} + +struct CacheEntry { + constexpr CacheEntry(const fs::FsPath& _path, const char* _header_key) + : json_path{_path} + , header_key{_header_key} { + + } + + bool init() { + if (m_json) { return true; + } + + // enable for testing etag is working. + // fs::FsNativeSd().DeleteFile(json_path); + + auto json_in = yyjson_read_file(json_path, YYJSON_READ_NOFLAG, nullptr, nullptr); + if (json_in) { + log_write("loading old json doc\n"); + m_json = yyjson_doc_mut_copy(json_in, nullptr); + yyjson_doc_free(json_in); + m_root = yyjson_mut_doc_get_root(m_json); } else { - if (force) { - return true; - } else { - return false; + log_write("creating new json doc\n"); + m_json = yyjson_mut_doc_new(nullptr); + m_root = yyjson_mut_obj(m_json); + yyjson_mut_doc_set_root(m_json, m_root); + } + + return m_json && m_root; + } + + void exit() { + if (!m_json) { + return; + } + + if (!yyjson_mut_write_file(json_path, m_json, YYJSON_WRITE_NOFLAG, nullptr, nullptr)) { + log_write("failed to write etag json: %s\n", json_path.s); + } + + yyjson_mut_doc_free(m_json); + m_json = nullptr; + m_root = nullptr; + } + + void set_internal(const fs::FsPath& path, const std::string& value) { + const auto kkey = generate_key_from_path(path); + + // check if we already have this entry + const auto it = m_cache.find(kkey); + if (it != m_cache.end() && it->second == value) { + log_write("already has etag, not updating, path: %s key: %s value: %s\n", path.s, kkey.c_str(), value.c_str()); + return; + } + + if (it != m_cache.end()) { + log_write("updating etag, path: %s old: %s new: %s\n", path.s, it->first.c_str(), it->second.c_str(), value.c_str()); + } else { + log_write("setting new etag, path: %s key: %s value: %s\n", path.s, kkey.c_str(), value.c_str()); + } + + // insert new entry into cache, this will never fail. + const auto& [jkey, jvalue] = *m_cache.insert_or_assign(it, kkey, value); + + // check if we need to add a new entry to root or simply update the value. + auto etag_key = yyjson_mut_obj_getn(m_root, kkey.c_str(), kkey.length()); + if (!etag_key) { + if (!yyjson_mut_obj_add_str(m_json, m_root, jkey.c_str(), jvalue.c_str())) { + log_write("failed to set new etag key: %s\n", jkey.c_str()); + } + } else { + if (!yyjson_mut_set_strn(etag_key, jvalue.c_str(), jvalue.length())) { + log_write("failed to update etag key: %s\n", jkey.c_str()); } } } - void RemoveFromCache(const Url& url) { - mutexLock(&mutex); - ON_SCOPE_EXIT(mutexUnlock(&mutex)); - auto it = std::find_if(cache.cbegin(), cache.cend(), [&url](const auto& e){ - return e.m_str == url.m_str; - }); + auto get(const fs::FsPath& path) -> std::string { + if (!fs::FsNativeSd().FileExists(path)) { + return {}; + } - if (it != cache.cend()) { - cache.erase(it); + const auto kkey = generate_key_from_path(path); + const auto it = m_cache.find(kkey); + if (it != m_cache.end()) { + return it->second; + } + + auto etag_key = yyjson_mut_obj_getn(m_root, kkey.c_str(), kkey.length()); + R_UNLESS(etag_key, {}); + + const auto val = yyjson_mut_get_str(etag_key); + const auto val_len = yyjson_mut_get_len(etag_key); + R_UNLESS(val && val_len, {}); + + const std::string ret = {val, val_len}; + m_cache.insert_or_assign(it, kkey, ret); + return ret; + } + + void set(const fs::FsPath& path, const std::string& value) { + set_internal(path, value); + } + + void set(const fs::FsPath& path, const curl::Header& value) { + if (auto it = value.Find(header_key); it != value.m_map.end()) { + set_internal(path, it->second); } } - std::vector cache; - Mutex mutex{}; -}; + const fs::FsPath json_path; + const char* header_key; -struct DataStruct { - std::vector data; - u64 offset{}; - FsFileSystem fs{}; - FsFile f{}; - s64 file_offset{}; + yyjson_mut_doc* m_json{}; + yyjson_mut_val* m_root{}; + std::unordered_map m_cache{}; }; struct ThreadEntry { @@ -179,7 +264,10 @@ struct ThreadQueue { ThreadEntry g_threads[MAX_THREADS]{}; ThreadQueue g_thread_queue; -UrlCache g_url_cache; + +CacheEntry g_etag{"/switch/sphaira/cache/etag.json", "etag"}; +CacheEntry g_lmt{"/switch/sphaira/cache/lmt.json", "last-modified"}; +Mutex g_cache_mutex; void GetDownloadTempPath(fs::FsPath& buf) { static Mutex mutex{}; @@ -275,27 +363,45 @@ auto WriteFileCallback(void *contents, size_t size, size_t num_files, void *user return realsize; } -auto DownloadInternal(CURL* curl, DataStruct& chunk, const Api& e) -> DownloadResult { - fs::FsPath safe_buf; +auto header_callback(char* b, size_t size, size_t nitems, void* userdata) -> size_t { + auto header = static_cast(userdata); + const auto numbytes = size * nitems; + + if (b && numbytes) { + const auto dilem = (const char*)memchr(b, ':', numbytes); + if (dilem) { + const int key_len = dilem - b; + const int value_len = numbytes - key_len - 4; // "\r\n" + if (key_len > 0 && value_len > 0) { + const std::string key(b, key_len); + const std::string value(dilem + 2, value_len); + header->m_map.insert_or_assign(key, value); + } + } + } + + return numbytes; +} + +auto DownloadInternal(CURL* curl, const Api& e) -> ApiResult { fs::FsPath tmp_buf; const bool has_file = !e.m_path.empty() && e.m_path != ""; const bool has_post = !e.m_fields.m_str.empty() && e.m_fields.m_str != ""; - ON_SCOPE_EXIT(if (has_file) { fsFsClose(&chunk.fs); } ); + DataStruct chunk; + Header header_out; + fs::FsNativeSd fs; if (has_file) { - std::strcpy(safe_buf, e.m_path); GetDownloadTempPath(tmp_buf); - R_TRY_RESULT(fsOpenSdCardFileSystem(&chunk.fs), {}); + fs.CreateDirectoryRecursivelyWithPath(tmp_buf, true); - fs::CreateDirectoryRecursivelyWithPath(&chunk.fs, tmp_buf); - - if (auto rc = fsFsCreateFile(&chunk.fs, tmp_buf, 0, 0); R_FAILED(rc) && rc != FsError_PathAlreadyExists) { + if (auto rc = fs.CreateFile(tmp_buf, 0, 0, true); R_FAILED(rc) && rc != FsError_PathAlreadyExists) { log_write("failed to create file: %s\n", tmp_buf); return {}; } - if (R_FAILED(fsFsOpenFile(&chunk.fs, tmp_buf, FsOpenMode_Write|FsOpenMode_Append, &chunk.f))) { + if (R_FAILED(fs.OpenFile(tmp_buf, FsOpenMode_Write|FsOpenMode_Append, &chunk.f))) { log_write("failed to open file: %s\n", tmp_buf); return {}; } @@ -304,6 +410,7 @@ auto DownloadInternal(CURL* curl, DataStruct& chunk, const Api& e) -> DownloadRe // reserve the first chunk chunk.data.reserve(CHUNK_SIZE); + curl_easy_reset(curl); CURL_EASY_SETOPT_LOG(curl, CURLOPT_URL, e.m_url.m_str.c_str()); CURL_EASY_SETOPT_LOG(curl, CURLOPT_USERAGENT, "TotalJustice"); CURL_EASY_SETOPT_LOG(curl, CURLOPT_FOLLOWLOCATION, 1L); @@ -312,6 +419,8 @@ auto DownloadInternal(CURL* curl, DataStruct& chunk, const Api& e) -> DownloadRe CURL_EASY_SETOPT_LOG(curl, CURLOPT_FAILONERROR, 1L); CURL_EASY_SETOPT_LOG(curl, CURLOPT_SHARE, g_curl_share); CURL_EASY_SETOPT_LOG(curl, CURLOPT_BUFFERSIZE, 1024*512); + CURL_EASY_SETOPT_LOG(curl, CURLOPT_HEADERFUNCTION, header_callback); + CURL_EASY_SETOPT_LOG(curl, CURLOPT_HEADERDATA, &header_out); if (has_post) { CURL_EASY_SETOPT_LOG(curl, CURLOPT_POSTFIELDS, e.m_fields.m_str.c_str()); @@ -321,19 +430,21 @@ auto DownloadInternal(CURL* curl, DataStruct& chunk, const Api& e) -> DownloadRe struct curl_slist* list = NULL; ON_SCOPE_EXIT(if (list) { curl_slist_free_all(list); } ); - for (auto& [key, value] : e.m_header) { - // append value (if any). - auto header_str = key; + for (auto& [key, value] : e.m_header.m_map) { if (value.empty()) { - header_str += ":"; - } else { - header_str += ": " + value; + continue; } + // create header key value pair. + const auto header_str = key + ": " + value; + // try to append header chunk. auto temp = curl_slist_append(list, header_str.c_str()); if (temp) { + log_write("adding header: %s\n", header_str.c_str()); list = temp; + } else { + log_write("failed to append header\n"); } } @@ -362,20 +473,21 @@ auto DownloadInternal(CURL* curl, DataStruct& chunk, const Api& e) -> DownloadRe curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); if (has_file) { + ON_SCOPE_EXIT( fs.DeleteFile(tmp_buf, true) ); if (res == CURLE_OK && chunk.offset) { fsFileWrite(&chunk.f, chunk.file_offset, chunk.data.data(), chunk.offset, FsWriteOption_None); } + fsFileClose(&chunk.f); - if (res != CURLE_OK) { - fsFsDeleteFile(&chunk.fs, tmp_buf); - } else { - fsFsDeleteFile(&chunk.fs, safe_buf); - fs::CreateDirectoryRecursivelyWithPath(&chunk.fs, safe_buf); - if (R_FAILED(fsFsRenameFile(&chunk.fs, tmp_buf, safe_buf))) { - fsFsDeleteFile(&chunk.fs, tmp_buf); + + if (res == CURLE_OK && http_code != 304) { + fs.DeleteFile(e.m_path, true); + fs.CreateDirectoryRecursivelyWithPath(e.m_path, true); + if (R_FAILED(fs.RenameFile(tmp_buf, e.m_path, true))) { success = false; } } + chunk.data.clear(); } else { // empty data if we failed if (res != CURLE_OK) { @@ -384,17 +496,17 @@ auto DownloadInternal(CURL* curl, DataStruct& chunk, const Api& e) -> DownloadRe } log_write("Downloaded %s %s\n", e.m_url.m_str.c_str(), curl_easy_strerror(res)); - return {success, http_code}; + return {success, http_code, header_out, chunk.data, e.m_path}; } -auto DownloadInternal(DataStruct& chunk, const Api& e) -> DownloadResult { +auto DownloadInternal(const Api& e) -> ApiResult { auto curl = curl_easy_init(); if (!curl) { log_write("curl init failed\n"); return {}; } ON_SCOPE_EXIT(curl_easy_cleanup(curl)); - return DownloadInternal(curl, chunk, e); + return DownloadInternal(curl, e); } void DownloadThread(void* p) { @@ -409,11 +521,10 @@ void DownloadThread(void* p) { continue; } - DataStruct chunk; #if 1 - const auto [result, code] = DownloadInternal(data->m_curl, chunk, data->m_api); + const auto result = DownloadInternal(data->m_curl, data->m_api); if (g_running) { - DownloadEventData event_data{data->m_api.m_on_complete, std::move(chunk.data), code, result}; + const DownloadEventData event_data{data->m_api.m_on_complete, result}; evman::push(std::move(event_data), false); } else { break; @@ -549,41 +660,23 @@ void Exit() { curl_global_cleanup(); } -auto ToMemory(const Api& e) -> std::vector { +auto ToMemory(const Api& e) -> ApiResult { if (!e.m_path.empty()) { return {}; } - - if (g_url_cache.AddToCache(e.m_url)) { - DataStruct chunk{}; - if (DownloadInternal(chunk, e).first) { - return chunk.data; - } - } - return {}; + return DownloadInternal(e); } -auto ToFile(const Api& e) -> bool { +auto ToFile(const Api& e) -> ApiResult { if (e.m_path.empty()) { - return false; + return {}; } - - if (g_url_cache.AddToCache(e.m_url)) { - DataStruct chunk{}; - if (DownloadInternal(chunk, e).first) { - return true; - } - } - return false; + return DownloadInternal(e); } auto ToMemoryAsync(const Api& api) -> bool { #if USE_THREAD_QUEUE - if (g_url_cache.AddToCache(api.m_url)) { - return g_thread_queue.Add(api); - } else { - return false; - } + return g_thread_queue.Add(api); #else // mutexLock(&g_thread_queue.m_mutex); // ON_SCOPE_EXIT(mutexUnlock(&g_thread_queue.m_mutex)); @@ -601,11 +694,7 @@ auto ToMemoryAsync(const Api& api) -> bool { auto ToFileAsync(const Api& e) -> bool { #if USE_THREAD_QUEUE - if (g_url_cache.AddToCache(e.m_url)) { - return g_thread_queue.Add(e); - } else { - return false; - } + return g_thread_queue.Add(e); #else // mutexLock(&g_thread_queue.m_mutex); // ON_SCOPE_EXIT(mutexUnlock(&g_thread_queue.m_mutex)); @@ -621,9 +710,66 @@ auto ToFileAsync(const Api& e) -> bool { #endif } -void ClearCache(const Url& url) { - g_url_cache.AddToCache(url); - g_url_cache.RemoveFromCache(url); +namespace cache { + +bool init() { + mutexLock(&g_cache_mutex); + ON_SCOPE_EXIT(mutexUnlock(&g_cache_mutex)); + + if (!g_etag.m_json) { + R_UNLESS(g_etag.init(), false); + } + + if (!g_lmt.m_json) { + R_UNLESS(g_lmt.init(), false); + } + + return true; } +void exit() { + mutexLock(&g_cache_mutex); + ON_SCOPE_EXIT(mutexUnlock(&g_cache_mutex)); + + g_etag.exit(); + g_lmt.exit(); +} + +auto etag_get(const fs::FsPath& path) -> std::string { + mutexLock(&g_cache_mutex); + ON_SCOPE_EXIT(mutexUnlock(&g_cache_mutex)); + return g_etag.get(path); +} + +void etag_set(const fs::FsPath& path, const std::string& value) { + mutexLock(&g_cache_mutex); + ON_SCOPE_EXIT(mutexUnlock(&g_cache_mutex)); + g_etag.set(path, value); +} + +void etag_set(const fs::FsPath& path, const Header& value) { + mutexLock(&g_cache_mutex); + ON_SCOPE_EXIT(mutexUnlock(&g_cache_mutex)); + g_etag.set(path, value); +} + +auto lmt_get(const fs::FsPath& path) -> std::string { + mutexLock(&g_cache_mutex); + ON_SCOPE_EXIT(mutexUnlock(&g_cache_mutex)); + return g_lmt.get(path); +} + +void lmt_set(const fs::FsPath& path, const std::string& value) { + mutexLock(&g_cache_mutex); + ON_SCOPE_EXIT(mutexUnlock(&g_cache_mutex)); + g_lmt.set(path, value); +} + +void lmt_set(const fs::FsPath& path, const Header& value) { + mutexLock(&g_cache_mutex); + ON_SCOPE_EXIT(mutexUnlock(&g_cache_mutex)); + g_lmt.set(path, value); +} + +} // namespace cache } // namespace sphaira::curl diff --git a/sphaira/source/i18n.cpp b/sphaira/source/i18n.cpp index a97151b..1c49e61 100644 --- a/sphaira/source/i18n.cpp +++ b/sphaira/source/i18n.cpp @@ -21,7 +21,7 @@ std::string get_internal(const char* str, size_t len) { } // add default entry - g_tr_cache.emplace(kkey, kkey); + const auto it = g_tr_cache.emplace(kkey, kkey).first; if (!json || !root) { log_write("no json or root\n"); @@ -43,7 +43,7 @@ std::string get_internal(const char* str, size_t len) { // update entry in cache const std::string ret = {val, val_len}; - g_tr_cache.insert_or_assign(kkey, ret); + g_tr_cache.insert_or_assign(it, kkey, ret); return ret; } diff --git a/sphaira/source/ui/menus/appstore.cpp b/sphaira/source/ui/menus/appstore.cpp index 0c63b65..23da9ea 100644 --- a/sphaira/source/ui/menus/appstore.cpp +++ b/sphaira/source/ui/menus/appstore.cpp @@ -206,17 +206,17 @@ auto LoadAndParseManifest(const Entry& e) -> ManifestEntries { return ParseManifest(std::span{(const char*)data.data(), data.size()}); } -void EntryLoadImageFile(fs::Fs& fs, const fs::FsPath& path, LazyImage& image) { +auto EntryLoadImageFile(fs::Fs& fs, const fs::FsPath& path, LazyImage& image) -> bool { // already have the image if (image.image) { - log_write("warning, tried to load image: %s when already loaded\n", path); - return; + // log_write("warning, tried to load image: %s when already loaded\n", path); + return true; } auto vg = App::GetVg(); std::vector image_buf; if (R_FAILED(fs.read_entire_file(path, image_buf))) { - image.state = ImageDownloadState::Failed; + log_write("failed to load image from file: %s\n", path.s); } else { int channels_in_file; auto buf = stbi_load_from_memory(image_buf.data(), image_buf.size(), &image.w, &image.h, &channels_in_file, 4); @@ -228,20 +228,21 @@ void EntryLoadImageFile(fs::Fs& fs, const fs::FsPath& path, LazyImage& image) { } if (!image.image) { - image.state = ImageDownloadState::Failed; - log_write("failed to load image from file: %s\n", path); + log_write("failed to load image from file: %s\n", path.s); + return false; } else { // log_write("loaded image from file: %s\n", path); + return true; } } -void EntryLoadImageFile(const fs::FsPath& path, LazyImage& image) { +auto EntryLoadImageFile(const fs::FsPath& path, LazyImage& image) -> bool { if (!strncasecmp("romfs:/", path, 7)) { fs::FsStdio fs; - EntryLoadImageFile(fs, path, image); + return EntryLoadImageFile(fs, path, image); } else { fs::FsNativeSd fs; - EntryLoadImageFile(fs, path, image); + return EntryLoadImageFile(fs, path, image); } } @@ -402,7 +403,7 @@ auto InstallApp(ProgressBox* pbox, const Entry& entry) -> bool { curl::Url{url}, curl::Path{zip_out}, curl::OnProgress{pbox->OnDownloadProgressCallback()} - )) { + ).success) { log_write("error with download\n"); return false; } @@ -682,8 +683,8 @@ EntryMenu::EntryMenu(Entry& entry, const LazyImage& default_icon, Menu& menu) curl::Url{URL_POST_FEEDBACK}, curl::Path{file}, curl::Fields{post}, - curl::OnComplete{[](std::vector& data, bool success, long code){ - if (success) { + curl::OnComplete{[](auto& result){ + if (result.success) { log_write("got feedback!\n"); } else { log_write("failed to send feedback :("); @@ -709,24 +710,29 @@ EntryMenu::EntryMenu(Entry& entry, const LazyImage& default_icon, Menu& menu) const auto path = BuildBannerCachePath(m_entry); const auto url = BuildBannerUrl(m_entry); - - if (fs::FsNativeSd().FileExists(path)) { - EntryLoadImageFile(path, m_banner); - } + m_banner.cached = EntryLoadImageFile(path, m_banner); // race condition if we pop the widget before the download completes - if (!m_banner.image) { - curl::Api().ToFileAsync( - curl::Url{url}, - curl::Path{path}, - curl::Priority::High, - curl::OnComplete{[this, path](std::vector& data, bool success, long code){ - if (success) { + curl::Api().ToFileAsync( + curl::Url{url}, + curl::Path{path}, + curl::Header{ + { "if-none-match", curl::cache::etag_get(path) }, + { "if-modified-since", curl::cache::lmt_get(path) }, + }, + curl::OnComplete{[this, path](auto& result){ + if (result.success) { + curl::cache::etag_set(result.path, result.header); + curl::cache::lmt_set(result.path, result.header); + + if (result.code == 304) { + m_banner.cached = false; + } else { EntryLoadImageFile(path, m_banner); } } - }); - } + } + }); SetSubHeading(m_entry.binary); SetSubHeading(m_entry.description); @@ -1021,53 +1027,33 @@ Menu::Menu(const std::vector& nro_entries) : MenuBase{"AppStore"_i18n} ); m_repo_download_state = ImageDownloadState::Progress; - FsTimeStampRaw time_stamp{}; - u64 current_time{}; - bool download_file = false; - if (R_SUCCEEDED(fs.GetFsOpenResult())) { - fs.GetFileTimeStampRaw(REPO_PATH, &time_stamp); - timeGetCurrentTime(TimeType_Default, ¤t_time); - } + curl::Api().ToFileAsync( + curl::Url{URL_JSON}, + curl::Path{REPO_PATH}, + curl::Header{ + { "if-none-match", curl::cache::etag_get(REPO_PATH) }, + { "if-modified-since", curl::cache::lmt_get(REPO_PATH) }, + }, + curl::OnComplete{[this](auto& result){ + if (result.success) { + curl::cache::etag_set(result.path, result.header); + curl::cache::lmt_set(result.path, result.header); - // this fails if we don't have the file or on fw < 3.0.0 - if (!time_stamp.is_valid) { - download_file = true; - } else { - // check the date, if older than 1hour, then fetch new file - // this relaxes the spam to their server, don't want to fetch repo - // every time the user opens the app! - const auto time_file = time_stamp.created; - const auto time_cur = current_time; - const auto day = 60 * 60; - if (time_file > time_cur || time_cur - time_file >= day) { - log_write("repo.json expired, downloading new! time_file: %zu time_cur: %zu\n", time_file, time_cur); - download_file = true; - } else { - log_write("repo.json not expired! time_file: %zu time_cur: %zu\n", time_file, time_cur); - } - } - - // todo: remove me soon - // download_file = true; - - if (download_file) { - curl::Api().ToFileAsync( - curl::Url{URL_JSON}, - curl::Path{REPO_PATH}, - curl::OnComplete{[this](std::vector& data, bool success, long code){ - if (success) { - m_repo_download_state = ImageDownloadState::Done; - if (HasFocus()) { - ScanHomebrew(); - } + if (result.code == 304) { + log_write("appstore json not updated\n"); } else { - m_repo_download_state = ImageDownloadState::Failed; + log_write("appstore json updated\n"); } + + m_repo_download_state = ImageDownloadState::Done; + if (HasFocus()) { + ScanHomebrew(); + } + } else { + m_repo_download_state = ImageDownloadState::Failed; } - }); - } else { - m_repo_download_state = ImageDownloadState::Done; - } + } + }); m_filter = (Filter)ini_getl(INI_SECTION, "filter", m_filter, App::CONFIG_PATH); m_sort = (SortType)ini_getl(INI_SECTION, "sort", m_sort, App::CONFIG_PATH); @@ -1111,41 +1097,69 @@ void Menu::Draw(NVGcontext* vg, Theme* theme) { gfx::drawRect(vg, SCREEN_WIDTH - 50+2, 102 + sb_h * sb_y, 10-4, sb_h + (sb_h * 2) - 4, theme->elements[ThemeEntryID_TEXT_SELECTED].colour); } + // max images per frame, in order to not hit io / gpu too hard. + const int image_load_max = 2; + int image_load_count = 0; + for (u64 i = 0, pos = SCROLL, y = 110, w = 370, h = 155; pos < nro_total && i < max_entry_display; y += h + 10) { for (u64 j = 0, x = 75; j < 3 && pos < nro_total && i < max_entry_display; j++, i++, pos++, x += w + 10) { const auto index = m_entries_current[pos]; auto& e = m_entries[index]; + auto& image = e.image; + + // try and load cached image. + if (image_load_count < image_load_max && !image.image && !image.tried_cache) { + image.tried_cache = true; + image.cached = EntryLoadImageFile(BuildIconCachePath(e), image); + if (image.cached) { + image_load_count++; + } + } // lazy load image - if (!e.image.image) { - switch (e.image.state) { + if (!image.image || image.cached) { + switch (image.state) { case ImageDownloadState::None: { const auto path = BuildIconCachePath(e); - if (fs::FsNativeSd().FileExists(path)) { - EntryLoadImageFile(path, e.image); - } else { - const auto url = BuildIconUrl(e); - e.image.state = ImageDownloadState::Progress; - curl::Api().ToFileAsync( - curl::Url{url}, - curl::Path{path}, - curl::Priority::High, - curl::OnComplete{[this, index](std::vector& data, bool success, long code) { - if (success) { - m_entries[index].image.state = ImageDownloadState::Done; - } else { - m_entries[index].image.state = ImageDownloadState::Failed; - log_write("failed to download image\n"); + const auto url = BuildIconUrl(e); + image.state = ImageDownloadState::Progress; + curl::Api().ToFileAsync( + curl::Url{url}, + curl::Path{path}, + curl::Header{ + { "if-none-match", curl::cache::etag_get(path) }, + { "if-modified-since", curl::cache::lmt_get(path) }, + }, + curl::OnComplete{[this, &image](auto& result) { + if (result.success) { + curl::cache::etag_set(result.path, result.header); + curl::cache::lmt_set(result.path, result.header); + + image.state = ImageDownloadState::Done; + // data hasn't changed + if (result.code == 304) { + log_write("downloaded appstore image, was cached\n"); + image.cached = false; } + } else { + image.state = ImageDownloadState::Failed; + log_write("failed to download image\n"); } - }); - } + } + }); } break; case ImageDownloadState::Progress: { } break; case ImageDownloadState::Done: { - EntryLoadImageFile(BuildIconCachePath(e), e.image); + if (image_load_count < image_load_max) { + image.cached = false; + if (!EntryLoadImageFile(BuildIconCachePath(e), e.image)) { + image.state = ImageDownloadState::Failed; + } else { + image_load_count++; + } + } } break; case ImageDownloadState::Failed: { } break; @@ -1164,7 +1178,7 @@ void Menu::Draw(NVGcontext* vg, Theme* theme) { // const float image_size = 256 / image_scale; // const float image_size_h = 150 / image_scale; DrawIcon(vg, e.image, m_default_image, x + 20, y + 20, 115, 115, true, image_scale); - // gfx::drawImage(vg, x + 20, y + 20, image_size, image_size_h, e.image.image ? e.image.image : m_default_image); + // gfx::drawImage(vg, x + 20, y + 20, image_size, image_size_h, image.image ? image.image : m_default_image); nvgSave(vg); nvgScissor(vg, x, y, w - 30.f, h); // clip diff --git a/sphaira/source/ui/menus/filebrowser.cpp b/sphaira/source/ui/menus/filebrowser.cpp index 8b4666c..9111c9b 100644 --- a/sphaira/source/ui/menus/filebrowser.cpp +++ b/sphaira/source/ui/menus/filebrowser.cpp @@ -233,13 +233,13 @@ auto GetRomIcon(ProgressBox* pbox, std::string filename, std::string extension, // try and download icon if (!pbox->ShouldExit()) { pbox->NewTransfer("Downloading "_i18n + gh_thumbnail_url); - auto png_image = curl::Api().ToMemory( + const auto result = curl::Api().ToMemory( curl::Url{gh_thumbnail_url}, curl::OnProgress{pbox->OnDownloadProgressCallback()} ); - if (!png_image.empty()) { - return png_image; + if (result.success && !result.data.empty()) { + return result.data; } } diff --git a/sphaira/source/ui/menus/ghdl.cpp b/sphaira/source/ui/menus/ghdl.cpp index c7a7a92..95460fe 100644 --- a/sphaira/source/ui/menus/ghdl.cpp +++ b/sphaira/source/ui/menus/ghdl.cpp @@ -24,11 +24,16 @@ namespace sphaira::ui::menu::gh { namespace { -auto GenerateApiUrl(const std::string& url) { - if (url.starts_with("https://api.github.com/repos/")) { - return url; - } - return "https://api.github.com/repos/" + url.substr(std::strlen("https://github.com/")) + "/releases/latest"; +constexpr auto CACHE_PATH = "/switch/sphaira/cache/github"; + +auto GenerateApiUrl(const Entry& e) { + return "https://api.github.com/repos/" + e.owner + "/" + e.repo + "/releases/latest"; +} + +auto apiBuildAssetCache(const std::string& url) -> fs::FsPath { + fs::FsPath path; + std::snprintf(path, sizeof(path), "%s/%u.json", CACHE_PATH, crc32Calculate(url.data(), url.size())); + return path; } void from_json(yyjson_val* json, AssetEntry& e) { @@ -39,11 +44,10 @@ void from_json(yyjson_val* json, AssetEntry& e) { } void from_json(const fs::FsPath& path, Entry& e) { - yyjson_read_err err; - JSON_INIT_VEC_FILE(path, nullptr, &err); + JSON_INIT_VEC_FILE(path, nullptr, nullptr); JSON_OBJ_ITR( - JSON_SET_STR(name); - JSON_SET_STR(url); + JSON_SET_STR(owner); + JSON_SET_STR(repo); JSON_SET_ARR_OBJ(assets); ); } @@ -58,8 +62,8 @@ void from_json(yyjson_val* json, GhApiAsset& e) { ); } -void from_json(const std::vector& data, GhApiEntry& e) { - JSON_INIT_VEC(data, nullptr); +void from_json(const fs::FsPath& path, GhApiEntry& e) { + JSON_INIT_VEC_FILE(path, nullptr, nullptr); JSON_OBJ_ITR( JSON_SET_STR(tag_name); JSON_SET_STR(name); @@ -67,12 +71,13 @@ void from_json(const std::vector& data, GhApiEntry& e) { ); } -auto DownloadApp(ProgressBox* pbox, const GhApiAsset& gh_asset, const AssetEntry& entry = {}) -> bool { - static const fs::FsPath temp_file{"/switch/sphaira/cache/ghdl.temp"}; +auto DownloadApp(ProgressBox* pbox, const GhApiAsset& gh_asset, const AssetEntry* entry) -> bool { + static const fs::FsPath temp_file{"/switch/sphaira/cache/github/ghdl.temp"}; constexpr auto chunk_size = 1024 * 512; // 512KiB fs::FsNativeSd fs; R_TRY_RESULT(fs.GetFsOpenResult(), false); + ON_SCOPE_EXIT(fs.DeleteFile(temp_file)); if (gh_asset.browser_download_url.empty()) { log_write("failed to find asset\n"); @@ -84,22 +89,19 @@ auto DownloadApp(ProgressBox* pbox, const GhApiAsset& gh_asset, const AssetEntry pbox->NewTransfer("Downloading "_i18n + gh_asset.name); log_write("starting download: %s\n", gh_asset.browser_download_url.c_str()); - curl::ClearCache(gh_asset.browser_download_url); if (!curl::Api().ToFile( curl::Url{gh_asset.browser_download_url}, curl::Path{temp_file}, curl::OnProgress{pbox->OnDownloadProgressCallback()} - )){ + ).success){ log_write("error with download\n"); return false; } } - ON_SCOPE_EXIT(fs.DeleteFile(temp_file)); - fs::FsPath root_path{"/"}; - if (!entry.path.empty()) { - root_path = entry.path; + if (entry && !entry->path.empty()) { + root_path = entry->path; } // 3. extract the zip / file @@ -185,7 +187,6 @@ auto DownloadApp(ProgressBox* pbox, const GhApiAsset& gh_asset, const AssetEntry } } } else { - fs::FsNativeSd fs; fs.CreateDirectoryRecursivelyWithPath(root_path, true); fs.DeleteFile(root_path); if (R_FAILED(fs.RenameFile(temp_file, root_path, true))) { @@ -197,60 +198,43 @@ auto DownloadApp(ProgressBox* pbox, const GhApiAsset& gh_asset, const AssetEntry return true; } -auto DownloadAssets(ProgressBox* pbox, const std::string& url, GhApiEntry& out) -> bool { +auto DownloadAssetJson(ProgressBox* pbox, const std::string& url, GhApiEntry& out) -> bool { // 1. download the json if (!pbox->ShouldExit()) { pbox->NewTransfer("Downloading json"_i18n); log_write("starting download\n"); - curl::ClearCache(url); - const auto json = curl::Api().ToMemory( + const auto path = apiBuildAssetCache(url); + + const auto result = curl::Api().ToFile( curl::Url{url}, - curl::OnProgress{pbox->OnDownloadProgressCallback()} + curl::Path{path}, + curl::OnProgress{pbox->OnDownloadProgressCallback()}, + curl::Header{ + { "Accept", "application/vnd.github+json" }, + { "if-none-match", curl::cache::etag_get(path) }, + { "if-modified-since", curl::cache::lmt_get(path) }, + } ); - if (json.empty()) { - log_write("error with download\n"); + if (!result.success) { + log_write("json empty\n"); return false; } - from_json(json, out); + curl::cache::etag_set(result.path, result.header); + curl::cache::lmt_set(result.path, result.header); + from_json(result.path, out); } - log_write("got: %s tag: %s\n", out.name.c_str(), out.tag_name.c_str()); - return !out.assets.empty(); } -auto DownloadApp(ProgressBox* pbox, const std::string& url, const AssetEntry& entry) -> bool { - // 1. download the json - GhApiEntry gh_entry{}; - if (!DownloadAssets(pbox, url, gh_entry)) { - return false; - } - - if (gh_entry.assets.empty()) { - log_write("no assets\n"); - return false; - } - - const auto it = std::find_if( - gh_entry.assets.cbegin(), gh_entry.assets.cend(), [&entry](auto& e) { - return entry.name == e.name; - } - ); - - if (it == gh_entry.assets.cend() || it->browser_download_url.empty()) { - log_write("failed to find asset\n"); - return false; - } - - return DownloadApp(pbox, *it, entry); -} - } // namespace Menu::Menu() : MenuBase{"GitHub"_i18n} { + fs::FsNativeSd().CreateDirectoryRecursively(CACHE_PATH); + this->SetActions( std::make_pair(Button::R2, Action{[this](){ }}), @@ -282,60 +266,63 @@ Menu::Menu() : MenuBase{"GitHub"_i18n} { return; } - // fetch all assets from github and present them to the user. - if (GetEntry().assets.empty()) { - // hack - static GhApiEntry gh_entry; - gh_entry = {}; + // hack + static GhApiEntry gh_entry; + gh_entry = {}; - App::Push(std::make_shared("Downloading "_i18n + GetEntry().name, [this](auto pbox){ - return DownloadAssets(pbox, GetEntry().url, gh_entry); - }, [this](bool success){ - if (success) { - PopupList::Items asset_items; - for (auto&p : gh_entry.assets) { - asset_items.emplace_back(p.name); - } + App::Push(std::make_shared("Downloading "_i18n + GetEntry().repo, [this](auto pbox){ + return DownloadAssetJson(pbox, GenerateApiUrl(GetEntry()), gh_entry); + }, [this](bool success){ + if (success) { + const auto& assets = GetEntry().assets; + PopupList::Items asset_items; + std::vector asset_ptr; + std::vector api_assets; + bool using_name = false; - App::Push(std::make_shared("Select asset to download for "_i18n + GetEntry().name, asset_items, [this](auto op_index){ - if (!op_index) { - return; + for (auto&p : gh_entry.assets) { + bool found = false; + + for (auto& e : assets) { + if (!e.name.empty()) { + using_name = true; } - const auto index = *op_index; - const auto& asset_entry = gh_entry.assets[index]; - App::Push(std::make_shared("Downloading "_i18n + GetEntry().name, [this, &asset_entry](auto pbox){ - return DownloadApp(pbox, asset_entry); - }, [this](bool success){ - if (success) { - App::Notify("Downloaded " + GetEntry().name); - } - }, 2)); - })); - } - }, 2)); - } else { - PopupList::Items asset_items; - for (auto&p : GetEntry().assets) { - asset_items.emplace_back(p.name); - } - - App::Push(std::make_shared("Select asset to download for "_i18n + GetEntry().name, asset_items, [this](auto op_index){ - if (!op_index) { - return; - } - - const auto index = *op_index; - const auto& asset_entry = GetEntry().assets[index]; - App::Push(std::make_shared("Downloading "_i18n + GetEntry().name, [this, &asset_entry](auto pbox){ - return DownloadApp(pbox, GetEntry().url, asset_entry); - }, [this](bool success){ - if (success) { - App::Notify("Downloaded " + GetEntry().name); + if (p.name.find(e.name) != p.name.npos) { + found = true; + asset_ptr.emplace_back(&e); + break; + } } - }, 2)); - })); - } + + if (!using_name || found) { + asset_items.emplace_back(p.name); + api_assets.emplace_back(p); + } + } + + App::Push(std::make_shared("Select asset to download for "_i18n + GetEntry().repo, asset_items, [this, api_assets, asset_ptr](auto op_index){ + if (!op_index) { + return; + } + + const auto index = *op_index; + const auto& asset_entry = api_assets[index]; + const AssetEntry* ptr{}; + if (asset_ptr.size()) { + ptr = asset_ptr[index]; + } + + App::Push(std::make_shared("Downloading "_i18n + GetEntry().repo, [this, &asset_entry, ptr](auto pbox){ + return DownloadApp(pbox, asset_entry, ptr); + }, [this](bool success){ + if (success) { + App::Notify("Downloaded " + GetEntry().repo); + } + }, 2)); + })); + } + }, 2)); }}), std::make_pair(Button::B, Action{"Back"_i18n, [this](){ @@ -406,10 +393,10 @@ void Menu::Draw(NVGcontext* vg, Theme* theme) { nvgSave(vg); const auto txt_clip = std::min(GetY() + GetH(), y + h) - y; nvgScissor(vg, x + text_xoffset, y, w-(x+text_xoffset+50), txt_clip); - gfx::drawText(vg, x + text_xoffset, y + (h / 2.f), 20.f, e.name.c_str(), NULL, NVG_ALIGN_LEFT | NVG_ALIGN_MIDDLE, theme->elements[text_id].colour); + gfx::drawTextArgs(vg, x + text_xoffset, y + (h / 2.f), 20.f, NVG_ALIGN_LEFT | NVG_ALIGN_MIDDLE, theme->elements[text_id].colour, "%s By %s", e.repo.c_str(), e.owner.c_str()); nvgRestore(vg); - gfx::drawTextArgs(vg, x + w - text_xoffset, y + (h / 2.f), 16.f, NVG_ALIGN_RIGHT | NVG_ALIGN_MIDDLE, theme->elements[text_id].colour, e.url.c_str()); + gfx::drawTextArgs(vg, x + w - text_xoffset, y + (h / 2.f), 16.f, NVG_ALIGN_RIGHT | NVG_ALIGN_MIDDLE, theme->elements[text_id].colour, GenerateApiUrl(e).c_str()); y += h; if (!InYBounds(y)) { @@ -479,17 +466,11 @@ void Menu::LoadEntriesFromPath(const fs::FsPath& path) { const auto full_path = fs::AppendPath(path, d->d_name); from_json(full_path, entry); - // check that we have a name and url - if (entry.name.empty() || entry.url.empty()) { + // check that we have a owner and repo + if (entry.owner.empty() || entry.repo.empty()) { continue; } - // ensure this url is for github - if (!entry.url.starts_with("https://github.com/") && !entry.url.starts_with("https://api.github.com/repos/")) { - continue; - } - - entry.url = GenerateApiUrl(entry.url); entry.json_path = full_path; m_entries.emplace_back(entry); } @@ -497,7 +478,18 @@ void Menu::LoadEntriesFromPath(const fs::FsPath& path) { void Menu::Sort() { const auto sorter = [this](Entry& lhs, Entry& rhs) -> bool { - return strcmp(lhs.name.c_str(), rhs.name.c_str()) < 0; + // handle fallback if multiple entries are added with the same name + // used for forks of a project. + // in the rare case of the user adding the same owner and repo, + // fallback to the filepath, which *is* unqiue + auto r = strcasecmp(lhs.repo.c_str(), rhs.repo.c_str()); + if (!r) { + r = strcasecmp(lhs.owner.c_str(), rhs.owner.c_str()); + if (!r) { + r = strcasecmp(lhs.json_path, rhs.json_path); + } + } + return r < 0; }; std::sort(m_entries.begin(), m_entries.end(), sorter); diff --git a/sphaira/source/ui/menus/homebrew.cpp b/sphaira/source/ui/menus/homebrew.cpp index 7c1bb64..e9bfbdb 100644 --- a/sphaira/source/ui/menus/homebrew.cpp +++ b/sphaira/source/ui/menus/homebrew.cpp @@ -272,7 +272,7 @@ void Menu::InstallHomebrew() { void Menu::ScanHomebrew() { TimeStamp ts; nro_scan("/switch", m_entries, m_hide_sphaira.Get()); - log_write("nros found: %zu time_taken: %.2f\n", m_entries.size(), ts.GetSeconds()); + log_write("nros found: %zu time_taken: %.2f\n", m_entries.size(), ts.GetSecondsD()); struct IniUser { std::vector& entires; @@ -330,6 +330,22 @@ void Menu::Sort() { const auto order = m_order.Get(); const auto sorter = [this, sort, order](const NroEntry& lhs, const NroEntry& rhs) -> bool { + const auto name_cmp = [order](const NroEntry& lhs, const NroEntry& rhs) -> bool { + auto r = strcasecmp(lhs.GetName(), rhs.GetName()); + if (!r) { + auto r = strcasecmp(lhs.GetAuthor(), rhs.GetAuthor()); + if (!r) { + auto r = strcasecmp(lhs.path, rhs.path); + } + } + + if (order == OrderType_Decending) { + return r < 0; + } else { + return r > 0; + } + }; + switch (sort) { case SortType_UpdatedStar: if (lhs.has_star.value() && !rhs.has_star.value()) { @@ -348,7 +364,7 @@ void Menu::Sort() { } if (lhs_timestamp == rhs_timestamp) { - return strcasecmp(lhs.GetName(), rhs.GetName()) < 0; + return name_cmp(lhs, rhs); } else if (order == OrderType_Decending) { return lhs_timestamp > rhs_timestamp; } else { @@ -364,7 +380,7 @@ void Menu::Sort() { } case SortType_Size: { if (lhs.size == rhs.size) { - return strcasecmp(lhs.GetName(), rhs.GetName()) < 0; + return name_cmp(lhs, rhs); } else if (order == OrderType_Decending) { return lhs.size > rhs.size; } else { @@ -379,11 +395,7 @@ void Menu::Sort() { return false; } case SortType_Alphabetical: { - if (order == OrderType_Decending) { - return strcasecmp(lhs.GetName(), rhs.GetName()) < 0; - } else { - return strcasecmp(lhs.GetName(), rhs.GetName()) > 0; - } + return name_cmp(lhs, rhs); } break; } diff --git a/sphaira/source/ui/menus/main_menu.cpp b/sphaira/source/ui/menus/main_menu.cpp index a47fc9d..4fc4dd5 100644 --- a/sphaira/source/ui/menus/main_menu.cpp +++ b/sphaira/source/ui/menus/main_menu.cpp @@ -23,6 +23,9 @@ namespace sphaira::ui::menu::main { namespace { +constexpr const char* GITHUB_URL{"https://api.github.com/repos/ITotalJustice/sphaira/releases/latest"}; +constexpr fs::FsPath CACHE_PATH{"/switch/sphaira/cache/sphaira_latest.json"}; + auto InstallUpdate(ProgressBox* pbox, const std::string url, const std::string version) -> bool { static fs::FsPath zip_out{"/switch/sphaira/cache/update.zip"}; constexpr auto chunk_size = 1024 * 512; // 512KiB @@ -35,12 +38,11 @@ auto InstallUpdate(ProgressBox* pbox, const std::string url, const std::string v pbox->NewTransfer("Downloading "_i18n + version); log_write("starting download: %s\n", url.c_str()); - curl::ClearCache(url); if (!curl::Api().ToFile( curl::Url{url}, curl::Path{zip_out}, curl::OnProgress{pbox->OnDownloadProgressCallback()} - )) { + ).success) { log_write("error with download\n"); return false; } @@ -140,17 +142,33 @@ auto InstallUpdate(ProgressBox* pbox, const std::string url, const std::string v } // namespace MainMenu::MainMenu() { - curl::Api().ToMemoryAsync( - curl::Url{"https://api.github.com/repos/ITotalJustice/sphaira/releases/latest"}, - curl::OnComplete{[this](std::vector& data, bool success, long code){ + curl::Api().ToFileAsync( + curl::Url{GITHUB_URL}, + curl::Path{CACHE_PATH}, + curl::Header{ + { "Accept", "application/vnd.github+json" }, + { "if-none-match", curl::cache::etag_get(CACHE_PATH) }, + { "if-modified-since", curl::cache::lmt_get(CACHE_PATH) }, + }, + curl::OnComplete{[this](auto& result){ + log_write("inside github download\n"); m_update_state = UpdateState::Error; ON_SCOPE_EXIT( log_write("update status: %u\n", (u8)m_update_state) ); - if (!success) { + if (!result.success) { return false; } - auto json = yyjson_read((const char*)data.data(), data.size(), 0); + if (result.code == 304) { + log_write("data hasn't changed\n"); + } else { + log_write("etag changed\n"); + } + + curl::cache::etag_set(result.path, result.header); + curl::cache::lmt_set(result.path, result.header); + + auto json = yyjson_read_file(CACHE_PATH, YYJSON_READ_NOFLAG, nullptr, nullptr); R_UNLESS(json, false); ON_SCOPE_EXIT(yyjson_doc_free(json)); @@ -368,11 +386,11 @@ void MainMenu::Draw(NVGcontext* vg, Theme* theme) { void MainMenu::OnFocusGained() { Widget::OnFocusGained(); - this->SetHidden(false); m_current_menu->OnFocusGained(); } void MainMenu::OnFocusLost() { + Widget::OnFocusLost(); m_current_menu->OnFocusLost(); } diff --git a/sphaira/source/ui/menus/themezer.cpp b/sphaira/source/ui/menus/themezer.cpp index 02346e6..1c9201a 100644 --- a/sphaira/source/ui/menus/themezer.cpp +++ b/sphaira/source/ui/menus/themezer.cpp @@ -53,9 +53,11 @@ constexpr const char* REQUEST_ORDER[]{ // https://api.themezer.net/?query=query($nsfw:Boolean,$page:Int,$limit:Int,$sort:String,$order:String,$query:String,$creators:[String!]){packList(nsfw:$nsfw,page:$page,limit:$limit,sort:$sort,order:$order,query:$query,creators:$creators){id,creator{id,display_name},details{name,description},last_updated,dl_count,like_count,themes{id,creator{display_name},details{name,description},last_updated,dl_count,like_count,target,preview{original,thumb}}}}&variables={"nsfw":false,"page":1,"limit":10,"sort":"updated","order":"desc","query":null,"creators":["695065006068334622"]} // i know, this is cursed +// todo: send actual POST request rather than GET. auto apiBuildUrlListInternal(const Config& e, bool is_pack) -> std::string { std::string api = "https://api.themezer.net/?query=query"; - std::string fields = "{id,creator{id,display_name},details{name,description},last_updated,dl_count,like_count"; + // std::string fields = "{id,creator{id,display_name},details{name,description},last_updated,dl_count,like_count"; + std::string fields = "{id,creator{id,display_name},details{name}"; const char* boolarr[2] = { "false", "true" }; std::string cmd; @@ -65,7 +67,8 @@ auto apiBuildUrlListInternal(const Config& e, bool is_pack) -> std::string { if (is_pack) { cmd = "packList"; - fields += ",themes{id,creator{display_name},details{name,description},last_updated,dl_count,like_count,target,preview{original,thumb}}"; + // fields += ",themes{id,creator{display_name},details{name,description},last_updated,dl_count,like_count,target,preview{original,thumb}}"; + fields += ",themes{id, preview{thumb}}"; } else { cmd = "themeList"; p0 += ",$target:String"; @@ -113,11 +116,13 @@ auto apiBuildFilePack(const PackListEntry& e) -> fs::FsPath { return path; } +#if 0 auto apiBuildUrlPack(const PackListEntry& e) -> std::string { char url[2048]; std::snprintf(url, sizeof(url), "https://api.themezer.net/?query=query($id:String!){pack(id:$id){id,creator{display_name},details{name,description},last_updated,categories,dl_count,like_count,themes{id,details{name},layout{id,details{name}},categories,target,preview{original,thumb},last_updated,dl_count,like_count}}}&variables={\"id\":\"%s\"}", e.id.c_str()); return url; } +#endif auto apiBuildUrlThemeList(const Config& e) -> std::string { return apiBuildUrlListInternal(e, false); @@ -127,19 +132,25 @@ auto apiBuildUrlListPacks(const Config& e) -> std::string { return apiBuildUrlListInternal(e, true); } +auto apiBuildListPacksCache(const Config& e) -> fs::FsPath { + fs::FsPath path; + std::snprintf(path, sizeof(path), "%s/%u_page.json", CACHE_PATH, e.page); + return path; +} + auto apiBuildIconCache(const ThemeEntry& e) -> fs::FsPath { fs::FsPath path; std::snprintf(path, sizeof(path), "%s/%s_thumb.jpg", CACHE_PATH, e.id.c_str()); return path; } -auto loadThemeImage(ThemeEntry& e) -> void { +auto loadThemeImage(ThemeEntry& e) -> bool { auto& image = e.preview.lazy_image; // already have the image if (e.preview.lazy_image.image) { // log_write("warning, tried to load image: %s when already loaded\n", path.c_str()); - return; + return true; } auto vg = App::GetVg(); @@ -148,22 +159,23 @@ auto loadThemeImage(ThemeEntry& e) -> void { const auto path = apiBuildIconCache(e); if (R_FAILED(fs.read_entire_file(path, image_buf))) { - e.preview.lazy_image.state = ImageDownloadState::Failed; + log_write("failed to load image from file: %s\n", path.s); } else { int channels_in_file; auto buf = stbi_load_from_memory(image_buf.data(), image_buf.size(), &image.w, &image.h, &channels_in_file, 4); if (buf) { ON_SCOPE_EXIT(stbi_image_free(buf)); - std::memcpy(image.first_pixel, buf, sizeof(image.first_pixel)); image.image = nvgCreateImageRGBA(vg, image.w, image.h, 0, buf); } } if (!image.image) { - image.state = ImageDownloadState::Failed; + log_write("failed to load image from file: %s\n", path.s); log_write("failed to load image from file: %s\n", path); + return false; } else { // log_write("loaded image from file: %s\n", path); + return true; } } @@ -197,13 +209,13 @@ void from_json(yyjson_val* json, Creator& e) { void from_json(yyjson_val* json, Details& e) { JSON_OBJ_ITR( JSON_SET_STR(name); - JSON_SET_STR(description); + // JSON_SET_STR(description); ); } void from_json(yyjson_val* json, Preview& e) { JSON_OBJ_ITR( - JSON_SET_STR(original); + // JSON_SET_STR(original); JSON_SET_STR(thumb); ); } @@ -219,13 +231,13 @@ void from_json(yyjson_val* json, DownloadPack& e) { void from_json(yyjson_val* json, ThemeEntry& e) { JSON_OBJ_ITR( JSON_SET_STR(id); - JSON_SET_OBJ(creator); - JSON_SET_OBJ(details); - JSON_SET_STR(last_updated); - JSON_SET_UINT(dl_count); - JSON_SET_UINT(like_count); - JSON_SET_ARR_STR(categories); - JSON_SET_STR(target); + // JSON_SET_OBJ(creator); + // JSON_SET_OBJ(details); + // JSON_SET_STR(last_updated); + // JSON_SET_UINT(dl_count); + // JSON_SET_UINT(like_count); + // JSON_SET_ARR_STR(categories); + // JSON_SET_STR(target); JSON_SET_OBJ(preview); ); } @@ -235,10 +247,10 @@ void from_json(yyjson_val* json, PackListEntry& e) { JSON_SET_STR(id); JSON_SET_OBJ(creator); JSON_SET_OBJ(details); - JSON_SET_STR(last_updated); - JSON_SET_ARR_STR(categories); - JSON_SET_UINT(dl_count); - JSON_SET_UINT(like_count); + // JSON_SET_STR(last_updated); + // JSON_SET_ARR_STR(categories); + // JSON_SET_UINT(dl_count); + // JSON_SET_UINT(like_count); JSON_SET_ARR_OBJ(themes); ); } @@ -263,8 +275,8 @@ void from_json(const std::vector& data, DownloadPack& e) { ); } -void from_json(const std::vector& data, PackList& e) { - JSON_INIT_VEC(data, "data"); +void from_json(const fs::FsPath& path, PackList& e) { + JSON_INIT_VEC_FILE(path, "data", nullptr); JSON_OBJ_ITR( JSON_SET_ARR_OBJ(packList); JSON_SET_OBJ(pagination); @@ -287,19 +299,17 @@ auto InstallTheme(ProgressBox* pbox, const PackListEntry& entry) -> bool { const auto url = apiBuildUrlDownloadPack(entry); log_write("using url: %s\n", url.c_str()); - curl::ClearCache(url); - const auto data = curl::Api().ToMemory( + const auto result = curl::Api().ToMemory( curl::Url{url}, curl::OnProgress{pbox->OnDownloadProgressCallback()} ); - if (data.empty()) { + if (!result.success || result.data.empty()) { log_write("error with download: %s\n", url.c_str()); - return false; } - from_json(data, download_pack); + from_json(result.data, download_pack); } // 2. download the zip @@ -307,18 +317,10 @@ auto InstallTheme(ProgressBox* pbox, const PackListEntry& entry) -> bool { pbox->NewTransfer("Downloading "_i18n + entry.details.name); log_write("starting download: %s\n", download_pack.url.c_str()); - curl::ClearCache(download_pack.url); if (!curl::Api().ToFile( curl::Url{download_pack.url}, curl::Path{zip_out}, - curl::OnProgress{[pbox](u32 dltotal, u32 dlnow, u32 ultotal, u32 ulnow){ - if (pbox->ShouldExit()) { - return false; - } - pbox->UpdateTransfer(dlnow, dltotal); - return true; - } - })) { + curl::OnProgress{pbox->OnDownloadProgressCallback()}).success) { log_write("error with download\n"); return false; } @@ -423,6 +425,8 @@ LazyImage::~LazyImage() { } Menu::Menu() : MenuBase{"Themezer"_i18n} { + fs::FsNativeSd().CreateDirectoryRecursively(CACHE_PATH); + SetAction(Button::B, Action{"Back"_i18n, [this]{ SetPop(); }}); @@ -603,6 +607,10 @@ void Menu::Draw(NVGcontext* vg, Theme* theme) { gfx::drawRect(vg, SCREEN_WIDTH - 50+2, 102 + sb_h * sb_y, 10-4, sb_h + (sb_h * 2) - 4, theme->elements[ThemeEntryID_TEXT_SELECTED].colour); } + // max images per frame, in order to not hit io / gpu too hard. + const int image_load_max = 2; + int image_load_count = 0; + nvgSave(vg); nvgScissor(vg, 30, 87, 1220 - 30, 646 - 87); // clip @@ -626,44 +634,69 @@ void Menu::Draw(NVGcontext* vg, Theme* theme) { if (e.themes.size()) { auto& theme = e.themes[0]; auto& image = e.themes[0].preview.lazy_image; - if (!image.image) { + + // try and load cached image. + if (image_load_count < image_load_max && !image.image && !image.tried_cache) { + image.tried_cache = true; + image.cached = loadThemeImage(theme); + if (image.cached) { + image_load_count++; + } + } + + if (!image.image || image.cached) { switch (image.state) { case ImageDownloadState::None: { const auto path = apiBuildIconCache(theme); log_write("downloading theme!: %s\n", path); - if (fs::FsNativeSd().FileExists(path)) { - loadThemeImage(theme); - } else { - const auto url = theme.preview.thumb; - log_write("downloading url: %s\n", url.c_str()); - image.state = ImageDownloadState::Progress; - curl::Api().ToFileAsync( - curl::Url{url}, - curl::Path{path}, - curl::Priority::High, - curl::OnComplete{[this, index, &image](std::vector& data, bool success, long code) { - if (success) { - image.state = ImageDownloadState::Done; - log_write("downloaded themezer image\n"); + const auto url = theme.preview.thumb; + log_write("downloading url: %s\n", url.c_str()); + image.state = ImageDownloadState::Progress; + curl::Api().ToFileAsync( + curl::Url{url}, + curl::Path{path}, + curl::Header{ + { "if-none-match", curl::cache::etag_get(path) }, + { "if-modified-since", curl::cache::lmt_get(path) }, + }, + curl::OnComplete{[this, &image](auto& result) { + if (result.success) { + curl::cache::etag_set(result.path, result.header); + curl::cache::lmt_set(result.path, result.header); + + image.state = ImageDownloadState::Done; + // data hasn't changed + if (result.code == 304) { + log_write("downloaded themezer image, was cached\n"); + image.cached = false; } else { - image.state = ImageDownloadState::Failed; - log_write("failed to download image\n"); + log_write("downloaded new themezer image\n"); } + } else { + image.state = ImageDownloadState::Failed; + log_write("failed to download image\n"); } - }); - } + } + }); } break; case ImageDownloadState::Progress: { } break; case ImageDownloadState::Done: { - loadThemeImage(theme); + image.cached = false; + if (!loadThemeImage(theme)) { + image.state = ImageDownloadState::Failed; + } else { + image_load_count++; + } } break; case ImageDownloadState::Failed: { } break; } - } else { + } + + if (image.image) { gfx::drawImageRounded(vg, x + xoff, y, 320, 180, image.image); } } @@ -711,26 +744,31 @@ void Menu::PackListDownload() { config.order_index = m_order.Get(); config.nsfw = m_nsfw.Get(); const auto packList_url = apiBuildUrlListPacks(config); - const auto themeList_url = apiBuildUrlThemeList(config); + const auto packlist_path = apiBuildListPacksCache(config); log_write("\npackList_url: %s\n\n", packList_url.c_str()); - log_write("\nthemeList_url: %s\n\n", themeList_url.c_str()); - curl::ClearCache(packList_url); - curl::Api().ToMemoryAsync( + curl::Api().ToFileAsync( curl::Url{packList_url}, - curl::Priority::High, - curl::OnComplete{[this, page_index](std::vector& data, bool success, long code){ + curl::Path{packlist_path}, + curl::Header{ + { "if-none-match", curl::cache::etag_get(packlist_path) }, + { "if-modified-since", curl::cache::lmt_get(packlist_path) }, + }, + curl::OnComplete{[this, page_index](auto& result){ log_write("got themezer data\n"); - if (!success) { + if (!result.success) { auto& page = m_pages[page_index-1]; page.m_ready = PageLoadState::Error; log_write("failed to get themezer data...\n"); return; } + curl::cache::etag_set(result.path, result.header); + curl::cache::lmt_set(result.path, result.header); + PackList a; - from_json(data, a); + from_json(result.path, a); m_pages.resize(a.pagination.page_count); auto& page = m_pages[page_index-1];