OmniNX/sphaira
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

many fixes and performance improvements for network requests (see commit details)

Browse Source 
- add etag support
- add last-modified support

with the above 2 changes, this means that all downloads can be cached. when attempting to download a file,
if the file is an image, load from cache. after, the download is processed with the above tags sent. if a 304 code
is received, then the file hasn't changed. otherwise, the new tags are saved and the downloaded file is now used (in the
case of an image, the new image is now loaded over the cached one).

this results in a *huge* speed improvement and overall a huge amount of bandwidth is saved for both the client and server.

- themezer requests now only request the data needed.

this results in a json file that is 4-5x smaller, meaning a much faster download and parsing time.

- loading images is capped to 2 images a frame. this was done to avoid fs being the bottle neck.
  a 9 page listing will take 5 frames. scrolling through lists is more responsive.

- downloads are pushed to the front of the queue as they're added. the point of this is to prioritise
  data that we need now.

- fix potential crash when sorting files based on names as its possible for a file to have the same name
  in the metadata. this fallsback to sorting by path, which is unique.

- add timeout for processing events. this was done in order to not block the main thread for too long.

- github json files have changed from a name + url to a repo + author pair.
- drawing widgets now starts from the last file in the array. as a menu takes up the whole screen, it
 is pointless drawing menu's underneath. this halves gpu usage.
- download url caching has been removed. this was added to fix a race condition when opening /
  closing a widget which starts a download when created. this would result in 2 same files being
  downloaded at the same time. this is no longer an issue and was overhead per download request.
This commit is contained in:
ITotalJustice
2024-12-29 00:33:31 +00:00
parent 2edfe91ad6
commit 5e315bd65f
21 changed files with 749 additions and 455 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
assets/romfs/github/ftpsrv.json
View File

@@ -1,6 +1,6 @@
{
"name": "ftpsrv",
"url": "https://github.com/ITotalJustice/ftpsrv",
"owner": "ITotalJustice",
"repo": "ftpsrv",
"assets": [
{
"name": "switch_application.zip"

9
assets/romfs/github/sphaira.json
View File

@@ -1,9 +1,4 @@
{
"name": "sphaira",
"url": "https://github.com/ITotalJustice/sphaira",
"assets":[
{
"name": "sphaira.zip"
}
]
"owner": "ITotalJustice",
"repo": "sphaira"
}

9
assets/romfs/github/untitled.json
View File

@@ -1,9 +1,4 @@
{
"name": "untitled",
"url": "https://github.com/ITotalJustice/untitled",
"assets":[
{
"name": "untitled.zip"
}
]
"owner": "ITotalJustice",
"repo": "untitled"
}

2
sphaira/CMakeLists.txt
View File

@@ -140,7 +140,7 @@ set(NANOVG_STBI_STATIC OFF)
set(NANOVG_STBTT_STATIC ON)
set(YYJSON_DISABLE_READER OFF)
set(YYJSON_DISABLE_WRITER ON)
set(YYJSON_DISABLE_WRITER OFF)
set(YYJSON_DISABLE_UTILS ON)
set(YYJSON_DISABLE_FAST_FP_CONV ON)
set(YYJSON_DISABLE_NON_STANDARD ON)

55
sphaira/include/download.hpp
View File

@@ -5,7 +5,7 @@
#include <string>
#include <functional>
#include <unordered_map>
#include <type_traits>
#include <algorithm>
#include <switch.h>
namespace sphaira::curl {
@@ -15,9 +15,11 @@ enum class Priority {
High, // gets pushed to the front of the queue
};
struct Api;
struct ApiResult;
using Path = fs::FsPath;
using Header = std::unordered_map<std::string, std::string>;
using OnComplete = std::function<void(std::vector<u8>& data, bool success, long code)>;
using OnComplete = std::function<void(ApiResult& result)>;
using OnProgress = std::function<bool(u32 dltotal, u32 dlnow, u32 ultotal, u32 ulnow)>;
struct Url {
@@ -32,29 +34,42 @@ struct Fields {
std::string m_str;
};
struct Api;
struct Header {
Header() = default;
Header(std::initializer_list<std::pair<const std::string, std::string>> p) : m_map{p} {}
std::unordered_map<std::string, std::string> m_map;
auto Find(const std::string& key) const {
return std::find_if(m_map.cbegin(), m_map.cend(), [&key](auto& e) {
return !strcasecmp(key.c_str(), e.first.c_str());
});
}
};
struct ApiResult {
bool success;
long code;
Header header; // returned headers in request
std::vector<u8> data; // empty if downloaded a file
fs::FsPath path; // empty if downloaded memory
};
struct DownloadEventData {
OnComplete callback;
std::vector<u8> data;
long code;
bool result;
ApiResult result;
};
auto Init() -> bool;
void Exit();
// sync functions
auto ToMemory(const Api& e) -> std::vector<u8>;
auto ToFile(const Api& e) -> bool;
auto ToMemory(const Api& e) -> ApiResult;
auto ToFile(const Api& e) -> ApiResult;
// async functions
auto ToMemoryAsync(const Api& e) -> bool;
auto ToFileAsync(const Api& e) -> bool;
// removes url from cache (todo: deprecate this)
void ClearCache(const Url& url);
struct Api {
Api() = default;
@@ -119,7 +134,7 @@ struct Api {
Path m_path{};
OnComplete m_on_complete = nullptr;
OnProgress m_on_progress = nullptr;
Priority m_prio = Priority::Normal;
Priority m_prio = Priority::High;
private:
void SetOption(Url&& v) {
@@ -156,4 +171,18 @@ private:
}
};
namespace cache {
bool init();
void exit();
auto etag_get(const fs::FsPath& path) -> std::string;
void etag_set(const fs::FsPath& path, const std::string& value);
void etag_set(const fs::FsPath& path, const Header& value);
auto lmt_get(const fs::FsPath& path) -> std::string;
void lmt_set(const fs::FsPath& path, const std::string& value);
void lmt_set(const fs::FsPath& path, const Header& value);
} // namespace cache
} // namespace sphaira::curl

2
sphaira/include/ui/menus/appstore.hpp
View File

@@ -27,6 +27,8 @@ struct LazyImage {
~LazyImage();
int image{};
int w{}, h{};
bool tried_cache{};
bool cached{};
ImageDownloadState state{ImageDownloadState::None};
u8 first_pixel[4]{};
};

4
sphaira/include/ui/menus/ghdl.hpp
View File

@@ -15,8 +15,8 @@ struct AssetEntry {
struct Entry {
fs::FsPath json_path;
std::string name;
std::string url;
std::string owner;
std::string repo;
std::vector<AssetEntry> assets;
};

4
sphaira/include/ui/menus/main_menu.hpp
View File

@@ -28,6 +28,10 @@ struct MainMenu final : Widget {
void OnFocusGained() override;
void OnFocusLost() override;
auto IsMenu() const -> bool override {
return true;
}
private:
void OnLRPress(std::shared_ptr<MenuBase> menu, Button b);
void AddOnLPress();

5
sphaira/include/ui/menus/menu_base.hpp
View File

@@ -12,6 +12,11 @@ struct MenuBase : Widget {
virtual void Update(Controller* controller, TouchInfo* touch);
virtual void Draw(NVGcontext* vg, Theme* theme);
auto IsMenu() const -> bool override {
return true;
}
void SetTitle(std::string title);
void SetTitleSubHeading(std::string sub_heading);
void SetSubHeading(std::string sub_heading);

49
sphaira/include/ui/menus/themezer.hpp
View File

@@ -15,28 +15,14 @@ enum class ImageDownloadState {
};
struct LazyImage {
LazyImage() = default;
~LazyImage();
int image{};
int w{}, h{};
bool tried_cache{};
bool cached{};
ImageDownloadState state{ImageDownloadState::None};
u8 first_pixel[4]{};
};
// "mutation setLike($type: String!, $id: String!, $value: Boolean!) {\n setLike(type: $type, id: $id, value: $value)\n}\n"
// https://api.themezer.net/?query=query($nsfw:Boolean,$target:String,$page:Int,$limit:Int,$sort:String,$order:String,$query:String){themeList(nsfw:$nsfw,target:$target,page:$page,limit:$limit,sort:$sort,order:$order,query:$query){id,creator{id,display_name},details{name,description},last_updated,dl_count,like_count,target,preview{original,thumb}}}&variables={"nsfw":false,"target":null,"page":1,"limit":10,"sort":"updated","order":"desc","query":null}
// https://api.themezer.net/?query=query($nsfw:Boolean,$page:Int,$limit:Int,$sort:String,$order:String,$query:String){packList(nsfw:$nsfw,page:$page,limit:$limit,sort:$sort,order:$order,query:$query){id,creator{id,display_name},details{name,description},last_updated,dl_count,like_count,themes{id,creator{display_name},details{name,description},last_updated,dl_count,like_count,target,preview{original,thumb}}}}&variables={"nsfw":false,"page":1,"limit":10,"sort":"updated","order":"desc","query":null}
// https://api.themezer.net/?query=query($id:String!){pack(id:$id){id,creator{display_name},details{name,description},last_updated,categories,dl_count,like_count,themes{id,details{name},layout{id,details{name}},categories,target,preview{original,thumb},last_updated,dl_count,like_count}}}&variables={"id":"16d"}
// https://api.themezer.net/?query=query{nxinstaller(id:"t9a6"){themes{filename,url,mimetype}}}
// https://api.themezer.net/?query=query{downloadTheme(id:"t9a6"){filename,url,mimetype}}
// https://api.themezer.net/?query=query{downloadPack(id:"t9a6"){filename,url,mimetype}}
// {"data":{"setLike":true}}
// https://api.themezer.net/?query=mutation{setLike(type:"packs",id:"5",value:true){data{setLike}}}
// https://api.themezer.net/?query=mutation($type:String!,$id:String!,$value:Boolean!){setLike(type:$type,id:$id,value:$value){data{setLike}}}&variables={"type":"packs","id":"5","value":true}
enum MenuState {
MenuState_Normal,
MenuState_Search,
@@ -55,6 +41,11 @@ enum class PageLoadState {
Error,
};
// all commented out entries are those that we don't query for.
// this saves time not only processing the json, but also the download
// of said json.
// by reducing the fields to only what we need, the size is 4-5x smaller.
struct Creator {
std::string id;
std::string display_name;
@@ -62,11 +53,11 @@ struct Creator {
struct Details {
std::string name;
std::string description;
// std::string description;
};
struct Preview {
std::string original;
// std::string original;
std::string thumb;
LazyImage lazy_image;
};
@@ -81,13 +72,13 @@ using DownloadTheme = DownloadPack;
struct ThemeEntry {
std::string id;
Creator creator;
Details details;
std::string last_updated;
u64 dl_count;
u64 like_count;
std::vector<std::string> categories;
std::string target;
// Creator creator;
// Details details;
// std::string last_updated;
// u64 dl_count;
// u64 like_count;
// std::vector<std::string> categories;
// std::string target;
Preview preview;
};
@@ -106,10 +97,10 @@ struct PackListEntry {
std::string id;
Creator creator;
Details details;
std::string last_updated;
std::vector<std::string> categories;
u64 dl_count;
u64 like_count;
// std::string last_updated;
// std::vector<std::string> categories;
// u64 dl_count;
// u64 like_count;
std::vector<ThemeEntry> themes;
};

19
sphaira/include/ui/types.hpp
View File

@@ -117,12 +117,27 @@ struct TimeStamp {
start = armGetSystemTick();
}
auto GetNs() -> u64 {
auto GetNs() const -> u64 {
const auto end_ticks = armGetSystemTick();
return armTicksToNs(end_ticks) - armTicksToNs(start);
}
auto GetSeconds() -> double {
auto GetMs() const -> u64 {
const auto ns = GetNs();
return ns/1000/1000;
}
auto GetSeconds() const -> u64 {
const auto ns = GetNs();
return ns/1000/1000/1000;
}
auto GetMsD() const -> double {
const double ns = GetNs();
return ns/1000.0/1000.0;
}
auto GetSecondsD() const -> double {
const double ns = GetNs();
return ns/1000.0/1000.0/1000.0;
}

4
sphaira/include/ui/widget.hpp
View File

@@ -26,6 +26,10 @@ struct Widget : public Object {
return m_focus;
}
virtual auto IsMenu() const -> bool {
return false;
}
auto HasAction(Button button) const -> bool;
void SetAction(Button button, Action action);
void SetActions(std::same_as<std::pair<Button, Action>> auto ...args) {

48
sphaira/source/app.cpp
View File

@@ -228,9 +228,22 @@ void App::Loop() {
ui::gfx::updateHighlightAnimation();
auto events = evman::popall();
// while (auto e = evman::pop()) {
for (auto& e : events) {
// fire all events in in a 3ms timeslice
TimeStamp ts_event;
const u64 event_timeout = 3;
// limit events to a max per frame in order to not block for too long.
while (true) {
if (ts_event.GetMs() >= event_timeout) {
log_write("event loop timed-out\n");
break;
}
auto event = evman::pop();
if (!event.has_value()) {
break;
}
std::visit([this](auto&& arg){
using T = std::decay_t<decltype(arg)>;
if constexpr(std::is_same_v<T, evman::LaunchNroEventData>) {
@@ -278,11 +291,11 @@ void App::Loop() {
}
} else if constexpr(std::is_same_v<T, curl::DownloadEventData>) {
log_write("[DownloadEventData] got event\n");
arg.callback(arg.data, arg.result, arg.code);
arg.callback(arg.result);
} else {
static_assert(false, "non-exhaustive visitor!");
}
}, e);
}, event.value());
}
u32 w{},h{};
@@ -663,11 +676,30 @@ void App::Draw() {
nvgBeginFrame(this->vg, s_width, s_height, 1.f);
nvgScale(vg, m_scale.x, m_scale.y);
// NOTE: widgets should never pop themselves from drawing!
for (auto& p : m_widgets) {
// find the last menu in the list, start drawing from there
auto menu_it = m_widgets.rend();
for (auto it = m_widgets.rbegin(); it != m_widgets.rend(); it++) {
const auto& p = *it;
if (!p->IsHidden() && p->IsMenu()) {
menu_it = it;
break;
}
}
// reverse itr so loop backwards to go forwarders.
if (menu_it != m_widgets.rend()) {
for (auto it = menu_it; ; it--) {
const auto& p = *it;
// draw everything not hidden on top of the menu.
if (!p->IsHidden()) {
p->Draw(vg, &m_theme);
}
if (it == m_widgets.rbegin()) {
break;
}
}
}
m_notif_manager.Draw(vg, &m_theme);
@@ -942,6 +974,7 @@ App::App(const char* argv0) {
}
curl::Init();
curl::cache::init();
// Create the deko3d device
this->device = dk::DeviceMaker{}
@@ -1097,6 +1130,7 @@ App::~App() {
i18n::exit();
curl::Exit();
curl::cache::exit();
// this has to be called before any cleanup to ensure the lifetime of
// nvg is still active as some widgets may need to free images.

344
sphaira/source/download.cpp
View File

@@ -10,12 +10,11 @@
#include <deque>
#include <mutex>
#include <curl/curl.h>
#include <yyjson.h>
namespace sphaira::curl {
namespace {
using DownloadResult = std::pair<bool, long>;
#define CURL_EASY_SETOPT_LOG(handle, opt, v) \
if (auto r = curl_easy_setopt(handle, opt, v); r != CURLE_OK) { \
log_write("curl_easy_setopt(%s, %s) msg: %s\n", #opt, #v, curl_easy_strerror(r)); \
@@ -40,50 +39,136 @@ std::atomic_bool g_running{};
CURLSH* g_curl_share{};
Mutex g_mutex_share[CURL_LOCK_DATA_LAST]{};
struct UrlCache {
auto AddToCache(const Url& url, bool force = false) {
mutexLock(&mutex);
ON_SCOPE_EXIT(mutexUnlock(&mutex));
auto it = std::find_if(cache.cbegin(), cache.cend(), [&url](const auto& e){
return e.m_str == url.m_str;
});
if (it == cache.cend()) {
cache.emplace_back(url);
return true;
} else {
if (force) {
return true;
} else {
return false;
}
}
}
void RemoveFromCache(const Url& url) {
mutexLock(&mutex);
ON_SCOPE_EXIT(mutexUnlock(&mutex));
auto it = std::find_if(cache.cbegin(), cache.cend(), [&url](const auto& e){
return e.m_str == url.m_str;
});
if (it != cache.cend()) {
cache.erase(it);
}
}
std::vector<Url> cache;
Mutex mutex{};
};
struct DataStruct {
std::vector<u8> data;
u64 offset{};
FsFileSystem fs{};
FsFile f{};
s64 file_offset{};
};
auto generate_key_from_path(const fs::FsPath& path) -> std::string {
const auto key = crc32Calculate(path.s, path.size());
return std::to_string(key);
}
struct CacheEntry {
constexpr CacheEntry(const fs::FsPath& _path, const char* _header_key)
: json_path{_path}
, header_key{_header_key} {
}
bool init() {
if (m_json) {
return true;
}
// enable for testing etag is working.
// fs::FsNativeSd().DeleteFile(json_path);
auto json_in = yyjson_read_file(json_path, YYJSON_READ_NOFLAG, nullptr, nullptr);
if (json_in) {
log_write("loading old json doc\n");
m_json = yyjson_doc_mut_copy(json_in, nullptr);
yyjson_doc_free(json_in);
m_root = yyjson_mut_doc_get_root(m_json);
} else {
log_write("creating new json doc\n");
m_json = yyjson_mut_doc_new(nullptr);
m_root = yyjson_mut_obj(m_json);
yyjson_mut_doc_set_root(m_json, m_root);
}
return m_json && m_root;
}
void exit() {
if (!m_json) {
return;
}
if (!yyjson_mut_write_file(json_path, m_json, YYJSON_WRITE_NOFLAG, nullptr, nullptr)) {
log_write("failed to write etag json: %s\n", json_path.s);
}
yyjson_mut_doc_free(m_json);
m_json = nullptr;
m_root = nullptr;
}
void set_internal(const fs::FsPath& path, const std::string& value) {
const auto kkey = generate_key_from_path(path);
// check if we already have this entry
const auto it = m_cache.find(kkey);
if (it != m_cache.end() && it->second == value) {
log_write("already has etag, not updating, path: %s key: %s value: %s\n", path.s, kkey.c_str(), value.c_str());
return;
}
if (it != m_cache.end()) {
log_write("updating etag, path: %s old: %s new: %s\n", path.s, it->first.c_str(), it->second.c_str(), value.c_str());
} else {
log_write("setting new etag, path: %s key: %s value: %s\n", path.s, kkey.c_str(), value.c_str());
}
// insert new entry into cache, this will never fail.
const auto& [jkey, jvalue] = *m_cache.insert_or_assign(it, kkey, value);
// check if we need to add a new entry to root or simply update the value.
auto etag_key = yyjson_mut_obj_getn(m_root, kkey.c_str(), kkey.length());
if (!etag_key) {
if (!yyjson_mut_obj_add_str(m_json, m_root, jkey.c_str(), jvalue.c_str())) {
log_write("failed to set new etag key: %s\n", jkey.c_str());
}
} else {
if (!yyjson_mut_set_strn(etag_key, jvalue.c_str(), jvalue.length())) {
log_write("failed to update etag key: %s\n", jkey.c_str());
}
}
}
auto get(const fs::FsPath& path) -> std::string {
if (!fs::FsNativeSd().FileExists(path)) {
return {};
}
const auto kkey = generate_key_from_path(path);
const auto it = m_cache.find(kkey);
if (it != m_cache.end()) {
return it->second;
}
auto etag_key = yyjson_mut_obj_getn(m_root, kkey.c_str(), kkey.length());
R_UNLESS(etag_key, {});
const auto val = yyjson_mut_get_str(etag_key);
const auto val_len = yyjson_mut_get_len(etag_key);
R_UNLESS(val && val_len, {});
const std::string ret = {val, val_len};
m_cache.insert_or_assign(it, kkey, ret);
return ret;
}
void set(const fs::FsPath& path, const std::string& value) {
set_internal(path, value);
}
void set(const fs::FsPath& path, const curl::Header& value) {
if (auto it = value.Find(header_key); it != value.m_map.end()) {
set_internal(path, it->second);
}
}
const fs::FsPath json_path;
const char* header_key;
yyjson_mut_doc* m_json{};
yyjson_mut_val* m_root{};
std::unordered_map<std::string, std::string> m_cache{};
};
struct ThreadEntry {
auto Create() -> Result {
m_curl = curl_easy_init();
@@ -179,7 +264,10 @@ struct ThreadQueue {
ThreadEntry g_threads[MAX_THREADS]{};
ThreadQueue g_thread_queue;
UrlCache g_url_cache;
CacheEntry g_etag{"/switch/sphaira/cache/etag.json", "etag"};
CacheEntry g_lmt{"/switch/sphaira/cache/lmt.json", "last-modified"};
Mutex g_cache_mutex;
void GetDownloadTempPath(fs::FsPath& buf) {
static Mutex mutex{};
@@ -275,27 +363,45 @@ auto WriteFileCallback(void *contents, size_t size, size_t num_files, void *user
return realsize;
}
auto DownloadInternal(CURL* curl, DataStruct& chunk, const Api& e) -> DownloadResult {
fs::FsPath safe_buf;
auto header_callback(char* b, size_t size, size_t nitems, void* userdata) -> size_t {
auto header = static_cast<Header*>(userdata);
const auto numbytes = size * nitems;
if (b && numbytes) {
const auto dilem = (const char*)memchr(b, ':', numbytes);
if (dilem) {
const int key_len = dilem - b;
const int value_len = numbytes - key_len - 4; // "\r\n"
if (key_len > 0 && value_len > 0) {
const std::string key(b, key_len);
const std::string value(dilem + 2, value_len);
header->m_map.insert_or_assign(key, value);
}
}
}
return numbytes;
}
auto DownloadInternal(CURL* curl, const Api& e) -> ApiResult {
fs::FsPath tmp_buf;
const bool has_file = !e.m_path.empty() && e.m_path != "";
const bool has_post = !e.m_fields.m_str.empty() && e.m_fields.m_str != "";
ON_SCOPE_EXIT(if (has_file) { fsFsClose(&chunk.fs); } );
DataStruct chunk;
Header header_out;
fs::FsNativeSd fs;
if (has_file) {
std::strcpy(safe_buf, e.m_path);
GetDownloadTempPath(tmp_buf);
R_TRY_RESULT(fsOpenSdCardFileSystem(&chunk.fs), {});
fs.CreateDirectoryRecursivelyWithPath(tmp_buf, true);
fs::CreateDirectoryRecursivelyWithPath(&chunk.fs, tmp_buf);
if (auto rc = fsFsCreateFile(&chunk.fs, tmp_buf, 0, 0); R_FAILED(rc) && rc != FsError_PathAlreadyExists) {
if (auto rc = fs.CreateFile(tmp_buf, 0, 0, true); R_FAILED(rc) && rc != FsError_PathAlreadyExists) {
log_write("failed to create file: %s\n", tmp_buf);
return {};
}
if (R_FAILED(fsFsOpenFile(&chunk.fs, tmp_buf, FsOpenMode_Write|FsOpenMode_Append, &chunk.f))) {
if (R_FAILED(fs.OpenFile(tmp_buf, FsOpenMode_Write|FsOpenMode_Append, &chunk.f))) {
log_write("failed to open file: %s\n", tmp_buf);
return {};
}
@@ -304,6 +410,7 @@ auto DownloadInternal(CURL* curl, DataStruct& chunk, const Api& e) -> DownloadRe
// reserve the first chunk
chunk.data.reserve(CHUNK_SIZE);
curl_easy_reset(curl);
CURL_EASY_SETOPT_LOG(curl, CURLOPT_URL, e.m_url.m_str.c_str());
CURL_EASY_SETOPT_LOG(curl, CURLOPT_USERAGENT, "TotalJustice");
CURL_EASY_SETOPT_LOG(curl, CURLOPT_FOLLOWLOCATION, 1L);
@@ -312,6 +419,8 @@ auto DownloadInternal(CURL* curl, DataStruct& chunk, const Api& e) -> DownloadRe
CURL_EASY_SETOPT_LOG(curl, CURLOPT_FAILONERROR, 1L);
CURL_EASY_SETOPT_LOG(curl, CURLOPT_SHARE, g_curl_share);
CURL_EASY_SETOPT_LOG(curl, CURLOPT_BUFFERSIZE, 1024*512);
CURL_EASY_SETOPT_LOG(curl, CURLOPT_HEADERFUNCTION, header_callback);
CURL_EASY_SETOPT_LOG(curl, CURLOPT_HEADERDATA, &header_out);
if (has_post) {
CURL_EASY_SETOPT_LOG(curl, CURLOPT_POSTFIELDS, e.m_fields.m_str.c_str());
@@ -321,19 +430,21 @@ auto DownloadInternal(CURL* curl, DataStruct& chunk, const Api& e) -> DownloadRe
struct curl_slist* list = NULL;
ON_SCOPE_EXIT(if (list) { curl_slist_free_all(list); } );
for (auto& [key, value] : e.m_header) {
// append value (if any).
auto header_str = key;
for (auto& [key, value] : e.m_header.m_map) {
if (value.empty()) {
header_str += ":";
} else {
header_str += ": " + value;
continue;
}
// create header key value pair.
const auto header_str = key + ": " + value;
// try to append header chunk.
auto temp = curl_slist_append(list, header_str.c_str());
if (temp) {
log_write("adding header: %s\n", header_str.c_str());
list = temp;
} else {
log_write("failed to append header\n");
}
}
@@ -362,20 +473,21 @@ auto DownloadInternal(CURL* curl, DataStruct& chunk, const Api& e) -> DownloadRe
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code);
if (has_file) {
ON_SCOPE_EXIT( fs.DeleteFile(tmp_buf, true) );
if (res == CURLE_OK && chunk.offset) {
fsFileWrite(&chunk.f, chunk.file_offset, chunk.data.data(), chunk.offset, FsWriteOption_None);
}
fsFileClose(&chunk.f);
if (res != CURLE_OK) {
fsFsDeleteFile(&chunk.fs, tmp_buf);
} else {
fsFsDeleteFile(&chunk.fs, safe_buf);
fs::CreateDirectoryRecursivelyWithPath(&chunk.fs, safe_buf);
if (R_FAILED(fsFsRenameFile(&chunk.fs, tmp_buf, safe_buf))) {
fsFsDeleteFile(&chunk.fs, tmp_buf);
if (res == CURLE_OK && http_code != 304) {
fs.DeleteFile(e.m_path, true);
fs.CreateDirectoryRecursivelyWithPath(e.m_path, true);
if (R_FAILED(fs.RenameFile(tmp_buf, e.m_path, true))) {
success = false;
}
}
chunk.data.clear();
} else {
// empty data if we failed
if (res != CURLE_OK) {
@@ -384,17 +496,17 @@ auto DownloadInternal(CURL* curl, DataStruct& chunk, const Api& e) -> DownloadRe
}
log_write("Downloaded %s %s\n", e.m_url.m_str.c_str(), curl_easy_strerror(res));
return {success, http_code};
return {success, http_code, header_out, chunk.data, e.m_path};
}
auto DownloadInternal(DataStruct& chunk, const Api& e) -> DownloadResult {
auto DownloadInternal(const Api& e) -> ApiResult {
auto curl = curl_easy_init();
if (!curl) {
log_write("curl init failed\n");
return {};
}
ON_SCOPE_EXIT(curl_easy_cleanup(curl));
return DownloadInternal(curl, chunk, e);
return DownloadInternal(curl, e);
}
void DownloadThread(void* p) {
@@ -409,11 +521,10 @@ void DownloadThread(void* p) {
continue;
}
DataStruct chunk;
#if 1
const auto [result, code] = DownloadInternal(data->m_curl, chunk, data->m_api);
const auto result = DownloadInternal(data->m_curl, data->m_api);
if (g_running) {
DownloadEventData event_data{data->m_api.m_on_complete, std::move(chunk.data), code, result};
const DownloadEventData event_data{data->m_api.m_on_complete, result};
evman::push(std::move(event_data), false);
} else {
break;
@@ -549,41 +660,23 @@ void Exit() {
curl_global_cleanup();
}
auto ToMemory(const Api& e) -> std::vector<u8> {
auto ToMemory(const Api& e) -> ApiResult {
if (!e.m_path.empty()) {
return {};
}
return DownloadInternal(e);
}
if (g_url_cache.AddToCache(e.m_url)) {
DataStruct chunk{};
if (DownloadInternal(chunk, e).first) {
return chunk.data;
}
}
auto ToFile(const Api& e) -> ApiResult {
if (e.m_path.empty()) {
return {};
}
auto ToFile(const Api& e) -> bool {
if (e.m_path.empty()) {
return false;
}
if (g_url_cache.AddToCache(e.m_url)) {
DataStruct chunk{};
if (DownloadInternal(chunk, e).first) {
return true;
}
}
return false;
return DownloadInternal(e);
}
auto ToMemoryAsync(const Api& api) -> bool {
#if USE_THREAD_QUEUE
if (g_url_cache.AddToCache(api.m_url)) {
return g_thread_queue.Add(api);
} else {
return false;
}
#else
// mutexLock(&g_thread_queue.m_mutex);
// ON_SCOPE_EXIT(mutexUnlock(&g_thread_queue.m_mutex));
@@ -601,11 +694,7 @@ auto ToMemoryAsync(const Api& api) -> bool {
auto ToFileAsync(const Api& e) -> bool {
#if USE_THREAD_QUEUE
if (g_url_cache.AddToCache(e.m_url)) {
return g_thread_queue.Add(e);
} else {
return false;
}
#else
// mutexLock(&g_thread_queue.m_mutex);
// ON_SCOPE_EXIT(mutexUnlock(&g_thread_queue.m_mutex));
@@ -621,9 +710,66 @@ auto ToFileAsync(const Api& e) -> bool {
#endif
}
void ClearCache(const Url& url) {
g_url_cache.AddToCache(url);
g_url_cache.RemoveFromCache(url);
namespace cache {
bool init() {
mutexLock(&g_cache_mutex);
ON_SCOPE_EXIT(mutexUnlock(&g_cache_mutex));
if (!g_etag.m_json) {
R_UNLESS(g_etag.init(), false);
}
if (!g_lmt.m_json) {
R_UNLESS(g_lmt.init(), false);
}
return true;
}
void exit() {
mutexLock(&g_cache_mutex);
ON_SCOPE_EXIT(mutexUnlock(&g_cache_mutex));
g_etag.exit();
g_lmt.exit();
}
auto etag_get(const fs::FsPath& path) -> std::string {
mutexLock(&g_cache_mutex);
ON_SCOPE_EXIT(mutexUnlock(&g_cache_mutex));
return g_etag.get(path);
}
void etag_set(const fs::FsPath& path, const std::string& value) {
mutexLock(&g_cache_mutex);
ON_SCOPE_EXIT(mutexUnlock(&g_cache_mutex));
g_etag.set(path, value);
}
void etag_set(const fs::FsPath& path, const Header& value) {
mutexLock(&g_cache_mutex);
ON_SCOPE_EXIT(mutexUnlock(&g_cache_mutex));
g_etag.set(path, value);
}
auto lmt_get(const fs::FsPath& path) -> std::string {
mutexLock(&g_cache_mutex);
ON_SCOPE_EXIT(mutexUnlock(&g_cache_mutex));
return g_lmt.get(path);
}
void lmt_set(const fs::FsPath& path, const std::string& value) {
mutexLock(&g_cache_mutex);
ON_SCOPE_EXIT(mutexUnlock(&g_cache_mutex));
g_lmt.set(path, value);
}
void lmt_set(const fs::FsPath& path, const Header& value) {
mutexLock(&g_cache_mutex);
ON_SCOPE_EXIT(mutexUnlock(&g_cache_mutex));
g_lmt.set(path, value);
}
} // namespace cache
} // namespace sphaira::curl

4
sphaira/source/i18n.cpp
View File

@@ -21,7 +21,7 @@ std::string get_internal(const char* str, size_t len) {
}
// add default entry
g_tr_cache.emplace(kkey, kkey);
const auto it = g_tr_cache.emplace(kkey, kkey).first;
if (!json || !root) {
log_write("no json or root\n");
@@ -43,7 +43,7 @@ std::string get_internal(const char* str, size_t len) {
// update entry in cache
const std::string ret = {val, val_len};
g_tr_cache.insert_or_assign(kkey, ret);
g_tr_cache.insert_or_assign(it, kkey, ret);
return ret;
}

154
sphaira/source/ui/menus/appstore.cpp
View File

@@ -206,17 +206,17 @@ auto LoadAndParseManifest(const Entry& e) -> ManifestEntries {
return ParseManifest(std::span{(const char*)data.data(), data.size()});
}
void EntryLoadImageFile(fs::Fs& fs, const fs::FsPath& path, LazyImage& image) {
auto EntryLoadImageFile(fs::Fs& fs, const fs::FsPath& path, LazyImage& image) -> bool {
// already have the image
if (image.image) {
log_write("warning, tried to load image: %s when already loaded\n", path);
return;
// log_write("warning, tried to load image: %s when already loaded\n", path);
return true;
}
auto vg = App::GetVg();
std::vector<u8> image_buf;
if (R_FAILED(fs.read_entire_file(path, image_buf))) {
image.state = ImageDownloadState::Failed;
log_write("failed to load image from file: %s\n", path.s);
} else {
int channels_in_file;
auto buf = stbi_load_from_memory(image_buf.data(), image_buf.size(), &image.w, &image.h, &channels_in_file, 4);
@@ -228,20 +228,21 @@ void EntryLoadImageFile(fs::Fs& fs, const fs::FsPath& path, LazyImage& image) {
}
if (!image.image) {
image.state = ImageDownloadState::Failed;
log_write("failed to load image from file: %s\n", path);
log_write("failed to load image from file: %s\n", path.s);
return false;
} else {
// log_write("loaded image from file: %s\n", path);
return true;
}
}
void EntryLoadImageFile(const fs::FsPath& path, LazyImage& image) {
auto EntryLoadImageFile(const fs::FsPath& path, LazyImage& image) -> bool {
if (!strncasecmp("romfs:/", path, 7)) {
fs::FsStdio fs;
EntryLoadImageFile(fs, path, image);
return EntryLoadImageFile(fs, path, image);
} else {
fs::FsNativeSd fs;
EntryLoadImageFile(fs, path, image);
return EntryLoadImageFile(fs, path, image);
}
}
@@ -402,7 +403,7 @@ auto InstallApp(ProgressBox* pbox, const Entry& entry) -> bool {
curl::Url{url},
curl::Path{zip_out},
curl::OnProgress{pbox->OnDownloadProgressCallback()}
)) {
).success) {
log_write("error with download\n");
return false;
}
@@ -682,8 +683,8 @@ EntryMenu::EntryMenu(Entry& entry, const LazyImage& default_icon, Menu& menu)
curl::Url{URL_POST_FEEDBACK},
curl::Path{file},
curl::Fields{post},
curl::OnComplete{[](std::vector<u8>& data, bool success, long code){
if (success) {
curl::OnComplete{[](auto& result){
if (result.success) {
log_write("got feedback!\n");
} else {
log_write("failed to send feedback :(");
@@ -709,24 +710,29 @@ EntryMenu::EntryMenu(Entry& entry, const LazyImage& default_icon, Menu& menu)
const auto path = BuildBannerCachePath(m_entry);
const auto url = BuildBannerUrl(m_entry);
if (fs::FsNativeSd().FileExists(path)) {
EntryLoadImageFile(path, m_banner);
}
m_banner.cached = EntryLoadImageFile(path, m_banner);
// race condition if we pop the widget before the download completes
if (!m_banner.image) {
curl::Api().ToFileAsync(
curl::Url{url},
curl::Path{path},
curl::Priority::High,
curl::OnComplete{[this, path](std::vector<u8>& data, bool success, long code){
if (success) {
curl::Header{
{ "if-none-match", curl::cache::etag_get(path) },
{ "if-modified-since", curl::cache::lmt_get(path) },
},
curl::OnComplete{[this, path](auto& result){
if (result.success) {
curl::cache::etag_set(result.path, result.header);
curl::cache::lmt_set(result.path, result.header);
if (result.code == 304) {
m_banner.cached = false;
} else {
EntryLoadImageFile(path, m_banner);
}
}
});
}
});
SetSubHeading(m_entry.binary);
SetSubHeading(m_entry.description);
@@ -1021,41 +1027,24 @@ Menu::Menu(const std::vector<NroEntry>& nro_entries) : MenuBase{"AppStore"_i18n}
);
m_repo_download_state = ImageDownloadState::Progress;
FsTimeStampRaw time_stamp{};
u64 current_time{};
bool download_file = false;
if (R_SUCCEEDED(fs.GetFsOpenResult())) {
fs.GetFileTimeStampRaw(REPO_PATH, &time_stamp);
timeGetCurrentTime(TimeType_Default, &current_time);
}
// this fails if we don't have the file or on fw < 3.0.0
if (!time_stamp.is_valid) {
download_file = true;
} else {
// check the date, if older than 1hour, then fetch new file
// this relaxes the spam to their server, don't want to fetch repo
// every time the user opens the app!
const auto time_file = time_stamp.created;
const auto time_cur = current_time;
const auto day = 60 * 60;
if (time_file > time_cur || time_cur - time_file >= day) {
log_write("repo.json expired, downloading new! time_file: %zu time_cur: %zu\n", time_file, time_cur);
download_file = true;
} else {
log_write("repo.json not expired! time_file: %zu time_cur: %zu\n", time_file, time_cur);
}
}
// todo: remove me soon
// download_file = true;
if (download_file) {
curl::Api().ToFileAsync(
curl::Url{URL_JSON},
curl::Path{REPO_PATH},
curl::OnComplete{[this](std::vector<u8>& data, bool success, long code){
if (success) {
curl::Header{
{ "if-none-match", curl::cache::etag_get(REPO_PATH) },
{ "if-modified-since", curl::cache::lmt_get(REPO_PATH) },
},
curl::OnComplete{[this](auto& result){
if (result.success) {
curl::cache::etag_set(result.path, result.header);
curl::cache::lmt_set(result.path, result.header);
if (result.code == 304) {
log_write("appstore json not updated\n");
} else {
log_write("appstore json updated\n");
}
m_repo_download_state = ImageDownloadState::Done;
if (HasFocus()) {
ScanHomebrew();
@@ -1065,9 +1054,6 @@ Menu::Menu(const std::vector<NroEntry>& nro_entries) : MenuBase{"AppStore"_i18n}
}
}
});
} else {
m_repo_download_state = ImageDownloadState::Done;
}
m_filter = (Filter)ini_getl(INI_SECTION, "filter", m_filter, App::CONFIG_PATH);
m_sort = (SortType)ini_getl(INI_SECTION, "sort", m_sort, App::CONFIG_PATH);
@@ -1111,41 +1097,69 @@ void Menu::Draw(NVGcontext* vg, Theme* theme) {
gfx::drawRect(vg, SCREEN_WIDTH - 50+2, 102 + sb_h * sb_y, 10-4, sb_h + (sb_h * 2) - 4, theme->elements[ThemeEntryID_TEXT_SELECTED].colour);
}
// max images per frame, in order to not hit io / gpu too hard.
const int image_load_max = 2;
int image_load_count = 0;
for (u64 i = 0, pos = SCROLL, y = 110, w = 370, h = 155; pos < nro_total && i < max_entry_display; y += h + 10) {
for (u64 j = 0, x = 75; j < 3 && pos < nro_total && i < max_entry_display; j++, i++, pos++, x += w + 10) {
const auto index = m_entries_current[pos];
auto& e = m_entries[index];
auto& image = e.image;
// try and load cached image.
if (image_load_count < image_load_max && !image.image && !image.tried_cache) {
image.tried_cache = true;
image.cached = EntryLoadImageFile(BuildIconCachePath(e), image);
if (image.cached) {
image_load_count++;
}
}
// lazy load image
if (!e.image.image) {
switch (e.image.state) {
if (!image.image || image.cached) {
switch (image.state) {
case ImageDownloadState::None: {
const auto path = BuildIconCachePath(e);
if (fs::FsNativeSd().FileExists(path)) {
EntryLoadImageFile(path, e.image);
} else {
const auto url = BuildIconUrl(e);
e.image.state = ImageDownloadState::Progress;
image.state = ImageDownloadState::Progress;
curl::Api().ToFileAsync(
curl::Url{url},
curl::Path{path},
curl::Priority::High,
curl::OnComplete{[this, index](std::vector<u8>& data, bool success, long code) {
if (success) {
m_entries[index].image.state = ImageDownloadState::Done;
curl::Header{
{ "if-none-match", curl::cache::etag_get(path) },
{ "if-modified-since", curl::cache::lmt_get(path) },
},
curl::OnComplete{[this, &image](auto& result) {
if (result.success) {
curl::cache::etag_set(result.path, result.header);
curl::cache::lmt_set(result.path, result.header);
image.state = ImageDownloadState::Done;
// data hasn't changed
if (result.code == 304) {
log_write("downloaded appstore image, was cached\n");
image.cached = false;
}
} else {
m_entries[index].image.state = ImageDownloadState::Failed;
image.state = ImageDownloadState::Failed;
log_write("failed to download image\n");
}
}
});
}
} break;
case ImageDownloadState::Progress: {
} break;
case ImageDownloadState::Done: {
EntryLoadImageFile(BuildIconCachePath(e), e.image);
if (image_load_count < image_load_max) {
image.cached = false;
if (!EntryLoadImageFile(BuildIconCachePath(e), e.image)) {
image.state = ImageDownloadState::Failed;
} else {
image_load_count++;
}
}
} break;
case ImageDownloadState::Failed: {
} break;
@@ -1164,7 +1178,7 @@ void Menu::Draw(NVGcontext* vg, Theme* theme) {
// const float image_size = 256 / image_scale;
// const float image_size_h = 150 / image_scale;
DrawIcon(vg, e.image, m_default_image, x + 20, y + 20, 115, 115, true, image_scale);
// gfx::drawImage(vg, x + 20, y + 20, image_size, image_size_h, e.image.image ? e.image.image : m_default_image);
// gfx::drawImage(vg, x + 20, y + 20, image_size, image_size_h, image.image ? image.image : m_default_image);
nvgSave(vg);
nvgScissor(vg, x, y, w - 30.f, h); // clip

6
sphaira/source/ui/menus/filebrowser.cpp
View File

@@ -233,13 +233,13 @@ auto GetRomIcon(ProgressBox* pbox, std::string filename, std::string extension,
// try and download icon
if (!pbox->ShouldExit()) {
pbox->NewTransfer("Downloading "_i18n + gh_thumbnail_url);
auto png_image = curl::Api().ToMemory(
const auto result = curl::Api().ToMemory(
curl::Url{gh_thumbnail_url},
curl::OnProgress{pbox->OnDownloadProgressCallback()}
);
if (!png_image.empty()) {
return png_image;
if (result.success && !result.data.empty()) {
return result.data;
}
}

186
sphaira/source/ui/menus/ghdl.cpp
View File

@@ -24,11 +24,16 @@
namespace sphaira::ui::menu::gh {
namespace {
auto GenerateApiUrl(const std::string& url) {
if (url.starts_with("https://api.github.com/repos/")) {
return url;
constexpr auto CACHE_PATH = "/switch/sphaira/cache/github";
auto GenerateApiUrl(const Entry& e) {
return "https://api.github.com/repos/" + e.owner + "/" + e.repo + "/releases/latest";
}
return "https://api.github.com/repos/" + url.substr(std::strlen("https://github.com/")) + "/releases/latest";
auto apiBuildAssetCache(const std::string& url) -> fs::FsPath {
fs::FsPath path;
std::snprintf(path, sizeof(path), "%s/%u.json", CACHE_PATH, crc32Calculate(url.data(), url.size()));
return path;
}
void from_json(yyjson_val* json, AssetEntry& e) {
@@ -39,11 +44,10 @@ void from_json(yyjson_val* json, AssetEntry& e) {
}
void from_json(const fs::FsPath& path, Entry& e) {
yyjson_read_err err;
JSON_INIT_VEC_FILE(path, nullptr, &err);
JSON_INIT_VEC_FILE(path, nullptr, nullptr);
JSON_OBJ_ITR(
JSON_SET_STR(name);
JSON_SET_STR(url);
JSON_SET_STR(owner);
JSON_SET_STR(repo);
JSON_SET_ARR_OBJ(assets);
);
}
@@ -58,8 +62,8 @@ void from_json(yyjson_val* json, GhApiAsset& e) {
);
}
void from_json(const std::vector<u8>& data, GhApiEntry& e) {
JSON_INIT_VEC(data, nullptr);
void from_json(const fs::FsPath& path, GhApiEntry& e) {
JSON_INIT_VEC_FILE(path, nullptr, nullptr);
JSON_OBJ_ITR(
JSON_SET_STR(tag_name);
JSON_SET_STR(name);
@@ -67,12 +71,13 @@ void from_json(const std::vector<u8>& data, GhApiEntry& e) {
);
}
auto DownloadApp(ProgressBox* pbox, const GhApiAsset& gh_asset, const AssetEntry& entry = {}) -> bool {
static const fs::FsPath temp_file{"/switch/sphaira/cache/ghdl.temp"};
auto DownloadApp(ProgressBox* pbox, const GhApiAsset& gh_asset, const AssetEntry* entry) -> bool {
static const fs::FsPath temp_file{"/switch/sphaira/cache/github/ghdl.temp"};
constexpr auto chunk_size = 1024 * 512; // 512KiB
fs::FsNativeSd fs;
R_TRY_RESULT(fs.GetFsOpenResult(), false);
ON_SCOPE_EXIT(fs.DeleteFile(temp_file));
if (gh_asset.browser_download_url.empty()) {
log_write("failed to find asset\n");
@@ -84,22 +89,19 @@ auto DownloadApp(ProgressBox* pbox, const GhApiAsset& gh_asset, const AssetEntry
pbox->NewTransfer("Downloading "_i18n + gh_asset.name);
log_write("starting download: %s\n", gh_asset.browser_download_url.c_str());
curl::ClearCache(gh_asset.browser_download_url);
if (!curl::Api().ToFile(
curl::Url{gh_asset.browser_download_url},
curl::Path{temp_file},
curl::OnProgress{pbox->OnDownloadProgressCallback()}
)){
).success){
log_write("error with download\n");
return false;
}
}
ON_SCOPE_EXIT(fs.DeleteFile(temp_file));
fs::FsPath root_path{"/"};
if (!entry.path.empty()) {
root_path = entry.path;
if (entry && !entry->path.empty()) {
root_path = entry->path;
}
// 3. extract the zip / file
@@ -185,7 +187,6 @@ auto DownloadApp(ProgressBox* pbox, const GhApiAsset& gh_asset, const AssetEntry
}
}
} else {
fs::FsNativeSd fs;
fs.CreateDirectoryRecursivelyWithPath(root_path, true);
fs.DeleteFile(root_path);
if (R_FAILED(fs.RenameFile(temp_file, root_path, true))) {
@@ -197,60 +198,43 @@ auto DownloadApp(ProgressBox* pbox, const GhApiAsset& gh_asset, const AssetEntry
return true;
}
auto DownloadAssets(ProgressBox* pbox, const std::string& url, GhApiEntry& out) -> bool {
auto DownloadAssetJson(ProgressBox* pbox, const std::string& url, GhApiEntry& out) -> bool {
// 1. download the json
if (!pbox->ShouldExit()) {
pbox->NewTransfer("Downloading json"_i18n);
log_write("starting download\n");
curl::ClearCache(url);
const auto json = curl::Api().ToMemory(
const auto path = apiBuildAssetCache(url);
const auto result = curl::Api().ToFile(
curl::Url{url},
curl::OnProgress{pbox->OnDownloadProgressCallback()}
curl::Path{path},
curl::OnProgress{pbox->OnDownloadProgressCallback()},
curl::Header{
{ "Accept", "application/vnd.github+json" },
{ "if-none-match", curl::cache::etag_get(path) },
{ "if-modified-since", curl::cache::lmt_get(path) },
}
);
if (json.empty()) {
log_write("error with download\n");
if (!result.success) {
log_write("json empty\n");
return false;
}
from_json(json, out);
curl::cache::etag_set(result.path, result.header);
curl::cache::lmt_set(result.path, result.header);
from_json(result.path, out);
}
log_write("got: %s tag: %s\n", out.name.c_str(), out.tag_name.c_str());
return !out.assets.empty();
}
auto DownloadApp(ProgressBox* pbox, const std::string& url, const AssetEntry& entry) -> bool {
// 1. download the json
GhApiEntry gh_entry{};
if (!DownloadAssets(pbox, url, gh_entry)) {
return false;
}
if (gh_entry.assets.empty()) {
log_write("no assets\n");
return false;
}
const auto it = std::find_if(
gh_entry.assets.cbegin(), gh_entry.assets.cend(), [&entry](auto& e) {
return entry.name == e.name;
}
);
if (it == gh_entry.assets.cend() || it->browser_download_url.empty()) {
log_write("failed to find asset\n");
return false;
}
return DownloadApp(pbox, *it, entry);
}
} // namespace
Menu::Menu() : MenuBase{"GitHub"_i18n} {
fs::FsNativeSd().CreateDirectoryRecursively(CACHE_PATH);
this->SetActions(
std::make_pair(Button::R2, Action{[this](){
}}),
@@ -282,60 +266,63 @@ Menu::Menu() : MenuBase{"GitHub"_i18n} {
return;
}
// fetch all assets from github and present them to the user.
if (GetEntry().assets.empty()) {
// hack
static GhApiEntry gh_entry;
gh_entry = {};
App::Push(std::make_shared<ProgressBox>("Downloading "_i18n + GetEntry().name, [this](auto pbox){
return DownloadAssets(pbox, GetEntry().url, gh_entry);
App::Push(std::make_shared<ProgressBox>("Downloading "_i18n + GetEntry().repo, [this](auto pbox){
return DownloadAssetJson(pbox, GenerateApiUrl(GetEntry()), gh_entry);
}, [this](bool success){
if (success) {
const auto& assets = GetEntry().assets;
PopupList::Items asset_items;
std::vector<const AssetEntry*> asset_ptr;
std::vector<GhApiAsset> api_assets;
bool using_name = false;
for (auto&p : gh_entry.assets) {
asset_items.emplace_back(p.name);
bool found = false;
for (auto& e : assets) {
if (!e.name.empty()) {
using_name = true;
}
App::Push(std::make_shared<PopupList>("Select asset to download for "_i18n + GetEntry().name, asset_items, [this](auto op_index){
if (p.name.find(e.name) != p.name.npos) {
found = true;
asset_ptr.emplace_back(&e);
break;
}
}
if (!using_name || found) {
asset_items.emplace_back(p.name);
api_assets.emplace_back(p);
}
}
App::Push(std::make_shared<PopupList>("Select asset to download for "_i18n + GetEntry().repo, asset_items, [this, api_assets, asset_ptr](auto op_index){
if (!op_index) {
return;
}
const auto index = *op_index;
const auto& asset_entry = gh_entry.assets[index];
App::Push(std::make_shared<ProgressBox>("Downloading "_i18n + GetEntry().name, [this, &asset_entry](auto pbox){
return DownloadApp(pbox, asset_entry);
const auto& asset_entry = api_assets[index];
const AssetEntry* ptr{};
if (asset_ptr.size()) {
ptr = asset_ptr[index];
}
App::Push(std::make_shared<ProgressBox>("Downloading "_i18n + GetEntry().repo, [this, &asset_entry, ptr](auto pbox){
return DownloadApp(pbox, asset_entry, ptr);
}, [this](bool success){
if (success) {
App::Notify("Downloaded " + GetEntry().name);
App::Notify("Downloaded " + GetEntry().repo);
}
}, 2));
}));
}
}, 2));
} else {
PopupList::Items asset_items;
for (auto&p : GetEntry().assets) {
asset_items.emplace_back(p.name);
}
App::Push(std::make_shared<PopupList>("Select asset to download for "_i18n + GetEntry().name, asset_items, [this](auto op_index){
if (!op_index) {
return;
}
const auto index = *op_index;
const auto& asset_entry = GetEntry().assets[index];
App::Push(std::make_shared<ProgressBox>("Downloading "_i18n + GetEntry().name, [this, &asset_entry](auto pbox){
return DownloadApp(pbox, GetEntry().url, asset_entry);
}, [this](bool success){
if (success) {
App::Notify("Downloaded " + GetEntry().name);
}
}, 2));
}));
}
}}),
std::make_pair(Button::B, Action{"Back"_i18n, [this](){
@@ -406,10 +393,10 @@ void Menu::Draw(NVGcontext* vg, Theme* theme) {
nvgSave(vg);
const auto txt_clip = std::min(GetY() + GetH(), y + h) - y;
nvgScissor(vg, x + text_xoffset, y, w-(x+text_xoffset+50), txt_clip);
gfx::drawText(vg, x + text_xoffset, y + (h / 2.f), 20.f, e.name.c_str(), NULL, NVG_ALIGN_LEFT | NVG_ALIGN_MIDDLE, theme->elements[text_id].colour);
gfx::drawTextArgs(vg, x + text_xoffset, y + (h / 2.f), 20.f, NVG_ALIGN_LEFT | NVG_ALIGN_MIDDLE, theme->elements[text_id].colour, "%s By %s", e.repo.c_str(), e.owner.c_str());
nvgRestore(vg);
gfx::drawTextArgs(vg, x + w - text_xoffset, y + (h / 2.f), 16.f, NVG_ALIGN_RIGHT | NVG_ALIGN_MIDDLE, theme->elements[text_id].colour, e.url.c_str());
gfx::drawTextArgs(vg, x + w - text_xoffset, y + (h / 2.f), 16.f, NVG_ALIGN_RIGHT | NVG_ALIGN_MIDDLE, theme->elements[text_id].colour, GenerateApiUrl(e).c_str());
y += h;
if (!InYBounds(y)) {
@@ -479,17 +466,11 @@ void Menu::LoadEntriesFromPath(const fs::FsPath& path) {
const auto full_path = fs::AppendPath(path, d->d_name);
from_json(full_path, entry);
// check that we have a name and url
if (entry.name.empty() || entry.url.empty()) {
// check that we have a owner and repo
if (entry.owner.empty() || entry.repo.empty()) {
continue;
}
// ensure this url is for github
if (!entry.url.starts_with("https://github.com/") && !entry.url.starts_with("https://api.github.com/repos/")) {
continue;
}
entry.url = GenerateApiUrl(entry.url);
entry.json_path = full_path;
m_entries.emplace_back(entry);
}
@@ -497,7 +478,18 @@ void Menu::LoadEntriesFromPath(const fs::FsPath& path) {
void Menu::Sort() {
const auto sorter = [this](Entry& lhs, Entry& rhs) -> bool {
return strcmp(lhs.name.c_str(), rhs.name.c_str()) < 0;
// handle fallback if multiple entries are added with the same name
// used for forks of a project.
// in the rare case of the user adding the same owner and repo,
// fallback to the filepath, which *is* unqiue
auto r = strcasecmp(lhs.repo.c_str(), rhs.repo.c_str());
if (!r) {
r = strcasecmp(lhs.owner.c_str(), rhs.owner.c_str());
if (!r) {
r = strcasecmp(lhs.json_path, rhs.json_path);
}
}
return r < 0;
};
std::sort(m_entries.begin(), m_entries.end(), sorter);

28
sphaira/source/ui/menus/homebrew.cpp
View File

@@ -272,7 +272,7 @@ void Menu::InstallHomebrew() {
void Menu::ScanHomebrew() {
TimeStamp ts;
nro_scan("/switch", m_entries, m_hide_sphaira.Get());
log_write("nros found: %zu time_taken: %.2f\n", m_entries.size(), ts.GetSeconds());
log_write("nros found: %zu time_taken: %.2f\n", m_entries.size(), ts.GetSecondsD());
struct IniUser {
std::vector<NroEntry>& entires;
@@ -330,6 +330,22 @@ void Menu::Sort() {
const auto order = m_order.Get();
const auto sorter = [this, sort, order](const NroEntry& lhs, const NroEntry& rhs) -> bool {
const auto name_cmp = [order](const NroEntry& lhs, const NroEntry& rhs) -> bool {
auto r = strcasecmp(lhs.GetName(), rhs.GetName());
if (!r) {
auto r = strcasecmp(lhs.GetAuthor(), rhs.GetAuthor());
if (!r) {
auto r = strcasecmp(lhs.path, rhs.path);
}
}
if (order == OrderType_Decending) {
return r < 0;
} else {
return r > 0;
}
};
switch (sort) {
case SortType_UpdatedStar:
if (lhs.has_star.value() && !rhs.has_star.value()) {
@@ -348,7 +364,7 @@ void Menu::Sort() {
}
if (lhs_timestamp == rhs_timestamp) {
return strcasecmp(lhs.GetName(), rhs.GetName()) < 0;
return name_cmp(lhs, rhs);
} else if (order == OrderType_Decending) {
return lhs_timestamp > rhs_timestamp;
} else {
@@ -364,7 +380,7 @@ void Menu::Sort() {
}
case SortType_Size: {
if (lhs.size == rhs.size) {
return strcasecmp(lhs.GetName(), rhs.GetName()) < 0;
return name_cmp(lhs, rhs);
} else if (order == OrderType_Decending) {
return lhs.size > rhs.size;
} else {
@@ -379,11 +395,7 @@ void Menu::Sort() {
return false;
}
case SortType_Alphabetical: {
if (order == OrderType_Decending) {
return strcasecmp(lhs.GetName(), rhs.GetName()) < 0;
} else {
return strcasecmp(lhs.GetName(), rhs.GetName()) > 0;
}
return name_cmp(lhs, rhs);
} break;
}

34
sphaira/source/ui/menus/main_menu.cpp
View File

@@ -23,6 +23,9 @@
namespace sphaira::ui::menu::main {
namespace {
constexpr const char* GITHUB_URL{"https://api.github.com/repos/ITotalJustice/sphaira/releases/latest"};
constexpr fs::FsPath CACHE_PATH{"/switch/sphaira/cache/sphaira_latest.json"};
auto InstallUpdate(ProgressBox* pbox, const std::string url, const std::string version) -> bool {
static fs::FsPath zip_out{"/switch/sphaira/cache/update.zip"};
constexpr auto chunk_size = 1024 * 512; // 512KiB
@@ -35,12 +38,11 @@ auto InstallUpdate(ProgressBox* pbox, const std::string url, const std::string v
pbox->NewTransfer("Downloading "_i18n + version);
log_write("starting download: %s\n", url.c_str());
curl::ClearCache(url);
if (!curl::Api().ToFile(
curl::Url{url},
curl::Path{zip_out},
curl::OnProgress{pbox->OnDownloadProgressCallback()}
)) {
).success) {
log_write("error with download\n");
return false;
}
@@ -140,17 +142,33 @@ auto InstallUpdate(ProgressBox* pbox, const std::string url, const std::string v
} // namespace
MainMenu::MainMenu() {
curl::Api().ToMemoryAsync(
curl::Url{"https://api.github.com/repos/ITotalJustice/sphaira/releases/latest"},
curl::OnComplete{[this](std::vector<u8>& data, bool success, long code){
curl::Api().ToFileAsync(
curl::Url{GITHUB_URL},
curl::Path{CACHE_PATH},
curl::Header{
{ "Accept", "application/vnd.github+json" },
{ "if-none-match", curl::cache::etag_get(CACHE_PATH) },
{ "if-modified-since", curl::cache::lmt_get(CACHE_PATH) },
},
curl::OnComplete{[this](auto& result){
log_write("inside github download\n");
m_update_state = UpdateState::Error;
ON_SCOPE_EXIT( log_write("update status: %u\n", (u8)m_update_state) );
if (!success) {
if (!result.success) {
return false;
}
auto json = yyjson_read((const char*)data.data(), data.size(), 0);
if (result.code == 304) {
log_write("data hasn't changed\n");
} else {
log_write("etag changed\n");
}
curl::cache::etag_set(result.path, result.header);
curl::cache::lmt_set(result.path, result.header);
auto json = yyjson_read_file(CACHE_PATH, YYJSON_READ_NOFLAG, nullptr, nullptr);
R_UNLESS(json, false);
ON_SCOPE_EXIT(yyjson_doc_free(json));
@@ -368,11 +386,11 @@ void MainMenu::Draw(NVGcontext* vg, Theme* theme) {
void MainMenu::OnFocusGained() {
Widget::OnFocusGained();
this->SetHidden(false);
m_current_menu->OnFocusGained();
}
void MainMenu::OnFocusLost() {
Widget::OnFocusLost();
m_current_menu->OnFocusLost();
}

148
sphaira/source/ui/menus/themezer.cpp
View File

@@ -53,9 +53,11 @@ constexpr const char* REQUEST_ORDER[]{
// https://api.themezer.net/?query=query($nsfw:Boolean,$page:Int,$limit:Int,$sort:String,$order:String,$query:String,$creators:[String!]){packList(nsfw:$nsfw,page:$page,limit:$limit,sort:$sort,order:$order,query:$query,creators:$creators){id,creator{id,display_name},details{name,description},last_updated,dl_count,like_count,themes{id,creator{display_name},details{name,description},last_updated,dl_count,like_count,target,preview{original,thumb}}}}&variables={"nsfw":false,"page":1,"limit":10,"sort":"updated","order":"desc","query":null,"creators":["695065006068334622"]}
// i know, this is cursed
// todo: send actual POST request rather than GET.
auto apiBuildUrlListInternal(const Config& e, bool is_pack) -> std::string {
std::string api = "https://api.themezer.net/?query=query";
std::string fields = "{id,creator{id,display_name},details{name,description},last_updated,dl_count,like_count";
// std::string fields = "{id,creator{id,display_name},details{name,description},last_updated,dl_count,like_count";
std::string fields = "{id,creator{id,display_name},details{name}";
const char* boolarr[2] = { "false", "true" };
std::string cmd;
@@ -65,7 +67,8 @@ auto apiBuildUrlListInternal(const Config& e, bool is_pack) -> std::string {
if (is_pack) {
cmd = "packList";
fields += ",themes{id,creator{display_name},details{name,description},last_updated,dl_count,like_count,target,preview{original,thumb}}";
// fields += ",themes{id,creator{display_name},details{name,description},last_updated,dl_count,like_count,target,preview{original,thumb}}";
fields += ",themes{id, preview{thumb}}";
} else {
cmd = "themeList";
p0 += ",$target:String";
@@ -113,11 +116,13 @@ auto apiBuildFilePack(const PackListEntry& e) -> fs::FsPath {
return path;
}
#if 0
auto apiBuildUrlPack(const PackListEntry& e) -> std::string {
char url[2048];
std::snprintf(url, sizeof(url), "https://api.themezer.net/?query=query($id:String!){pack(id:$id){id,creator{display_name},details{name,description},last_updated,categories,dl_count,like_count,themes{id,details{name},layout{id,details{name}},categories,target,preview{original,thumb},last_updated,dl_count,like_count}}}&variables={\"id\":\"%s\"}", e.id.c_str());
return url;
}
#endif
auto apiBuildUrlThemeList(const Config& e) -> std::string {
return apiBuildUrlListInternal(e, false);
@@ -127,19 +132,25 @@ auto apiBuildUrlListPacks(const Config& e) -> std::string {
return apiBuildUrlListInternal(e, true);
}
auto apiBuildListPacksCache(const Config& e) -> fs::FsPath {
fs::FsPath path;
std::snprintf(path, sizeof(path), "%s/%u_page.json", CACHE_PATH, e.page);
return path;
}
auto apiBuildIconCache(const ThemeEntry& e) -> fs::FsPath {
fs::FsPath path;
std::snprintf(path, sizeof(path), "%s/%s_thumb.jpg", CACHE_PATH, e.id.c_str());
return path;
}
auto loadThemeImage(ThemeEntry& e) -> void {
auto loadThemeImage(ThemeEntry& e) -> bool {
auto& image = e.preview.lazy_image;
// already have the image
if (e.preview.lazy_image.image) {
// log_write("warning, tried to load image: %s when already loaded\n", path.c_str());
return;
return true;
}
auto vg = App::GetVg();
@@ -148,22 +159,23 @@ auto loadThemeImage(ThemeEntry& e) -> void {
const auto path = apiBuildIconCache(e);
if (R_FAILED(fs.read_entire_file(path, image_buf))) {
e.preview.lazy_image.state = ImageDownloadState::Failed;
log_write("failed to load image from file: %s\n", path.s);
} else {
int channels_in_file;
auto buf = stbi_load_from_memory(image_buf.data(), image_buf.size(), &image.w, &image.h, &channels_in_file, 4);
if (buf) {
ON_SCOPE_EXIT(stbi_image_free(buf));
std::memcpy(image.first_pixel, buf, sizeof(image.first_pixel));
image.image = nvgCreateImageRGBA(vg, image.w, image.h, 0, buf);
}
}
if (!image.image) {
image.state = ImageDownloadState::Failed;
log_write("failed to load image from file: %s\n", path.s);
log_write("failed to load image from file: %s\n", path);
return false;
} else {
// log_write("loaded image from file: %s\n", path);
return true;
}
}
@@ -197,13 +209,13 @@ void from_json(yyjson_val* json, Creator& e) {
void from_json(yyjson_val* json, Details& e) {
JSON_OBJ_ITR(
JSON_SET_STR(name);
JSON_SET_STR(description);
// JSON_SET_STR(description);
);
}
void from_json(yyjson_val* json, Preview& e) {
JSON_OBJ_ITR(
JSON_SET_STR(original);
// JSON_SET_STR(original);
JSON_SET_STR(thumb);
);
}
@@ -219,13 +231,13 @@ void from_json(yyjson_val* json, DownloadPack& e) {
void from_json(yyjson_val* json, ThemeEntry& e) {
JSON_OBJ_ITR(
JSON_SET_STR(id);
JSON_SET_OBJ(creator);
JSON_SET_OBJ(details);
JSON_SET_STR(last_updated);
JSON_SET_UINT(dl_count);
JSON_SET_UINT(like_count);
JSON_SET_ARR_STR(categories);
JSON_SET_STR(target);
// JSON_SET_OBJ(creator);
// JSON_SET_OBJ(details);
// JSON_SET_STR(last_updated);
// JSON_SET_UINT(dl_count);
// JSON_SET_UINT(like_count);
// JSON_SET_ARR_STR(categories);
// JSON_SET_STR(target);
JSON_SET_OBJ(preview);
);
}
@@ -235,10 +247,10 @@ void from_json(yyjson_val* json, PackListEntry& e) {
JSON_SET_STR(id);
JSON_SET_OBJ(creator);
JSON_SET_OBJ(details);
JSON_SET_STR(last_updated);
JSON_SET_ARR_STR(categories);
JSON_SET_UINT(dl_count);
JSON_SET_UINT(like_count);
// JSON_SET_STR(last_updated);
// JSON_SET_ARR_STR(categories);
// JSON_SET_UINT(dl_count);
// JSON_SET_UINT(like_count);
JSON_SET_ARR_OBJ(themes);
);
}
@@ -263,8 +275,8 @@ void from_json(const std::vector<u8>& data, DownloadPack& e) {
);
}
void from_json(const std::vector<u8>& data, PackList& e) {
JSON_INIT_VEC(data, "data");
void from_json(const fs::FsPath& path, PackList& e) {
JSON_INIT_VEC_FILE(path, "data", nullptr);
JSON_OBJ_ITR(
JSON_SET_ARR_OBJ(packList);
JSON_SET_OBJ(pagination);
@@ -287,19 +299,17 @@ auto InstallTheme(ProgressBox* pbox, const PackListEntry& entry) -> bool {
const auto url = apiBuildUrlDownloadPack(entry);
log_write("using url: %s\n", url.c_str());
curl::ClearCache(url);
const auto data = curl::Api().ToMemory(
const auto result = curl::Api().ToMemory(
curl::Url{url},
curl::OnProgress{pbox->OnDownloadProgressCallback()}
);
if (data.empty()) {
if (!result.success || result.data.empty()) {
log_write("error with download: %s\n", url.c_str());
return false;
}
from_json(data, download_pack);
from_json(result.data, download_pack);
}
// 2. download the zip
@@ -307,18 +317,10 @@ auto InstallTheme(ProgressBox* pbox, const PackListEntry& entry) -> bool {
pbox->NewTransfer("Downloading "_i18n + entry.details.name);
log_write("starting download: %s\n", download_pack.url.c_str());
curl::ClearCache(download_pack.url);
if (!curl::Api().ToFile(
curl::Url{download_pack.url},
curl::Path{zip_out},
curl::OnProgress{[pbox](u32 dltotal, u32 dlnow, u32 ultotal, u32 ulnow){
if (pbox->ShouldExit()) {
return false;
}
pbox->UpdateTransfer(dlnow, dltotal);
return true;
}
})) {
curl::OnProgress{pbox->OnDownloadProgressCallback()}).success) {
log_write("error with download\n");
return false;
}
@@ -423,6 +425,8 @@ LazyImage::~LazyImage() {
}
Menu::Menu() : MenuBase{"Themezer"_i18n} {
fs::FsNativeSd().CreateDirectoryRecursively(CACHE_PATH);
SetAction(Button::B, Action{"Back"_i18n, [this]{
SetPop();
}});
@@ -603,6 +607,10 @@ void Menu::Draw(NVGcontext* vg, Theme* theme) {
gfx::drawRect(vg, SCREEN_WIDTH - 50+2, 102 + sb_h * sb_y, 10-4, sb_h + (sb_h * 2) - 4, theme->elements[ThemeEntryID_TEXT_SELECTED].colour);
}
// max images per frame, in order to not hit io / gpu too hard.
const int image_load_max = 2;
int image_load_count = 0;
nvgSave(vg);
nvgScissor(vg, 30, 87, 1220 - 30, 646 - 87); // clip
@@ -626,44 +634,69 @@ void Menu::Draw(NVGcontext* vg, Theme* theme) {
if (e.themes.size()) {
auto& theme = e.themes[0];
auto& image = e.themes[0].preview.lazy_image;
if (!image.image) {
// try and load cached image.
if (image_load_count < image_load_max && !image.image && !image.tried_cache) {
image.tried_cache = true;
image.cached = loadThemeImage(theme);
if (image.cached) {
image_load_count++;
}
}
if (!image.image || image.cached) {
switch (image.state) {
case ImageDownloadState::None: {
const auto path = apiBuildIconCache(theme);
log_write("downloading theme!: %s\n", path);
if (fs::FsNativeSd().FileExists(path)) {
loadThemeImage(theme);
} else {
const auto url = theme.preview.thumb;
log_write("downloading url: %s\n", url.c_str());
image.state = ImageDownloadState::Progress;
curl::Api().ToFileAsync(
curl::Url{url},
curl::Path{path},
curl::Priority::High,
curl::OnComplete{[this, index, &image](std::vector<u8>& data, bool success, long code) {
if (success) {
curl::Header{
{ "if-none-match", curl::cache::etag_get(path) },
{ "if-modified-since", curl::cache::lmt_get(path) },
},
curl::OnComplete{[this, &image](auto& result) {
if (result.success) {
curl::cache::etag_set(result.path, result.header);
curl::cache::lmt_set(result.path, result.header);
image.state = ImageDownloadState::Done;
log_write("downloaded themezer image\n");
// data hasn't changed
if (result.code == 304) {
log_write("downloaded themezer image, was cached\n");
image.cached = false;
} else {
log_write("downloaded new themezer image\n");
}
} else {
image.state = ImageDownloadState::Failed;
log_write("failed to download image\n");
}
}
});
}
} break;
case ImageDownloadState::Progress: {
} break;
case ImageDownloadState::Done: {
loadThemeImage(theme);
image.cached = false;
if (!loadThemeImage(theme)) {
image.state = ImageDownloadState::Failed;
} else {
image_load_count++;
}
} break;
case ImageDownloadState::Failed: {
} break;
}
} else {
}
if (image.image) {
gfx::drawImageRounded(vg, x + xoff, y, 320, 180, image.image);
}
}
@@ -711,26 +744,31 @@ void Menu::PackListDownload() {
config.order_index = m_order.Get();
config.nsfw = m_nsfw.Get();
const auto packList_url = apiBuildUrlListPacks(config);
const auto themeList_url = apiBuildUrlThemeList(config);
const auto packlist_path = apiBuildListPacksCache(config);
log_write("\npackList_url: %s\n\n", packList_url.c_str());
log_write("\nthemeList_url: %s\n\n", themeList_url.c_str());
curl::ClearCache(packList_url);
curl::Api().ToMemoryAsync(
curl::Api().ToFileAsync(
curl::Url{packList_url},
curl::Priority::High,
curl::OnComplete{[this, page_index](std::vector<u8>& data, bool success, long code){
curl::Path{packlist_path},
curl::Header{
{ "if-none-match", curl::cache::etag_get(packlist_path) },
{ "if-modified-since", curl::cache::lmt_get(packlist_path) },
},
curl::OnComplete{[this, page_index](auto& result){
log_write("got themezer data\n");
if (!success) {
if (!result.success) {
auto& page = m_pages[page_index-1];
page.m_ready = PageLoadState::Error;
log_write("failed to get themezer data...\n");
return;
}
curl::cache::etag_set(result.path, result.header);
curl::cache::lmt_set(result.path, result.header);
PackList a;
from_json(data, a);
from_json(result.path, a);
m_pages.resize(a.pagination.page_count);
auto& page = m_pages[page_index-1];