fs.mitm: add and use memlet module to temporarily steal applet memory while building romfs images.

Starting in 20.0.0, the browser needs more applet memory to function, so we can't steal as much any more.
Thus, we now steal 14 MB on 20.0.0+ instead of 40MB.

However, since this reduces memory available for custom system modules, we are adjusting to compensate.
ams.mitm's heap size has been reduced from 32MB to 12MB (recovering 20MB).
In addition, fs.mitm now uses a new mechanism for stealing memory from the applet pool while romfs is being built.

On net, we are compromising:
    * Custom sysmodules lose memory available to them.
        On 19.0.0/AMS 1.8.0, there was 30 MB available for custom sysmodules.
        Stealing 14 MB instead of 40 MB, we lose 26 MB of that. Reducing ams.mitm's usage will gain us back 20.
        Nintendo also appears to...use 4 extra MB, in 20.0.0, from my test homebrew.
        So on 20.0.0/AMS 1.9.0, there should be 20 MB available for custom sysmodules.
        On the bright side, on <20.0.0/AMS 1.9.0, I guess there will be 50 MB available for custom sysmodules now?
    * totk mods will lose the ability to...put every file in the romfs on sd card. There will be some unknown maximum filecount for totk mods.
        On the bright side, implementing the transient memory stealing should improve compatibility for some mods which strictly add files?
This commit is contained in:
Michael Scire
2025-05-02 20:17:16 -07:00
parent d0cb9b0eb7
commit 0ecc35c062
19 changed files with 672 additions and 12 deletions

View File

@@ -0,0 +1,41 @@
ATMOSPHERE_BUILD_CONFIGS :=
all: nx_release
THIS_MAKEFILE := $(abspath $(lastword $(MAKEFILE_LIST)))
CURRENT_DIRECTORY := $(abspath $(dir $(THIS_MAKEFILE)))
define ATMOSPHERE_ADD_TARGET
ATMOSPHERE_BUILD_CONFIGS += $(strip $1)
$(strip $1):
@echo "Building $(strip $1)"
@$$(MAKE) -f $(CURRENT_DIRECTORY)/system_module.mk ATMOSPHERE_MAKEFILE_TARGET="$(strip $1)" ATMOSPHERE_BUILD_NAME="$(strip $2)" ATMOSPHERE_BOARD="$(strip $3)" ATMOSPHERE_CPU="$(strip $4)" $(strip $5)
clean-$(strip $1):
@echo "Cleaning $(strip $1)"
@$$(MAKE) -f $(CURRENT_DIRECTORY)/system_module.mk clean ATMOSPHERE_MAKEFILE_TARGET="$(strip $1)" ATMOSPHERE_BUILD_NAME="$(strip $2)" ATMOSPHERE_BOARD="$(strip $3)" ATMOSPHERE_CPU="$(strip $4)" $(strip $5)
endef
define ATMOSPHERE_ADD_TARGETS
$(eval $(call ATMOSPHERE_ADD_TARGET, $(strip $1)_release, release, $(strip $2), $(strip $3), \
ATMOSPHERE_BUILD_SETTINGS="$(strip $4)" \
))
$(eval $(call ATMOSPHERE_ADD_TARGET, $(strip $1)_debug, debug, $(strip $2), $(strip $3), \
ATMOSPHERE_BUILD_SETTINGS="$(strip $4) -DAMS_BUILD_FOR_DEBUGGING" ATMOSPHERE_BUILD_FOR_DEBUGGING=1 \
))
$(eval $(call ATMOSPHERE_ADD_TARGET, $(strip $1)_audit, audit, $(strip $2), $(strip $3), \
ATMOSPHERE_BUILD_SETTINGS="$(strip $4) -DAMS_BUILD_FOR_AUDITING" ATMOSPHERE_BUILD_FOR_DEBUGGING=1 ATMOSPHERE_BUILD_FOR_AUDITING=1 \
))
endef
$(eval $(call ATMOSPHERE_ADD_TARGETS, nx, nx-hac-001, arm-cortex-a57,))
clean: $(foreach config,$(ATMOSPHERE_BUILD_CONFIGS),clean-$(config))
.PHONY: all clean $(foreach config,$(ATMOSPHERE_BUILD_CONFIGS), $(config) clean-$(config))

View File

@@ -0,0 +1,93 @@
{
"name": "memlet",
"title_id": "0x0100000000000421",
"title_id_range_min": "0x0100000000000421",
"title_id_range_max": "0x0100000000000421",
"main_thread_stack_size": "0x00002000",
"main_thread_priority": 44,
"default_cpu_id": 3,
"process_category": 0,
"is_retail": true,
"pool_partition": 1,
"is_64_bit": true,
"address_space_type": 3,
"disable_device_address_space_merge": true,
"filesystem_access": {
"permissions": "0xFFFFFFFFFFFFFFFF"
},
"service_access": ["fatal:u"],
"service_host": ["memlet"],
"kernel_capabilities": [{
"type": "kernel_flags",
"value": {
"highest_thread_priority": 63,
"lowest_thread_priority": 24,
"lowest_cpu_id": 3,
"highest_cpu_id": 3
}
}, {
"type": "syscalls",
"value": {
"svcSetHeapSize": "0x01",
"svcSetMemoryPermission": "0x02",
"svcSetMemoryAttribute": "0x03",
"svcMapMemory": "0x04",
"svcUnmapMemory": "0x05",
"svcQueryMemory": "0x06",
"svcExitProcess": "0x07",
"svcCreateThread": "0x08",
"svcStartThread": "0x09",
"svcExitThread": "0x0a",
"svcSleepThread": "0x0b",
"svcGetThreadPriority": "0x0c",
"svcSetThreadPriority": "0x0d",
"svcGetThreadCoreMask": "0x0e",
"svcSetThreadCoreMask": "0x0f",
"svcGetCurrentProcessorNumber": "0x10",
"svcSignalEvent": "0x11",
"svcClearEvent": "0x12",
"svcMapSharedMemory": "0x13",
"svcUnmapSharedMemory": "0x14",
"svcCreateTransferMemory": "0x15",
"svcCloseHandle": "0x16",
"svcResetSignal": "0x17",
"svcWaitSynchronization": "0x18",
"svcCancelSynchronization": "0x19",
"svcArbitrateLock": "0x1a",
"svcArbitrateUnlock": "0x1b",
"svcWaitProcessWideKeyAtomic": "0x1c",
"svcSignalProcessWideKey": "0x1d",
"svcGetSystemTick": "0x1e",
"svcConnectToNamedPort": "0x1f",
"svcSendSyncRequestLight": "0x20",
"svcSendSyncRequest": "0x21",
"svcSendSyncRequestWithUserBuffer": "0x22",
"svcSendAsyncRequestWithUserBuffer": "0x23",
"svcGetProcessId": "0x24",
"svcGetThreadId": "0x25",
"svcBreak": "0x26",
"svcOutputDebugString": "0x27",
"svcReturnFromException": "0x28",
"svcGetInfo": "0x29",
"svcWaitForAddress": "0x34",
"svcSignalToAddress": "0x35",
"svcSynchronizePreemptionState": "0x36",
"svcCreateSession": "0x40",
"svcAcceptSession": "0x41",
"svcReplyAndReceiveLight": "0x42",
"svcReplyAndReceive": "0x43",
"svcReplyAndReceiveWithUserBuffer": "0x44",
"svcCreateSharedMemory": "0x50",
"svcCallSecureMonitor": "0x7f"
}
}, {
"type": "application_type",
"value": 2
}, {
"type": "min_kernel_version",
"value": "0x0030"
}, {
"type": "handle_table_size",
"value": 0
}]
}

View File

@@ -0,0 +1,75 @@
/*
* Copyright (c) Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <stratosphere.hpp>
#include "memlet_service.hpp"
namespace ams {
namespace memlet {
namespace {
using ServerOptions = sf::hipc::DefaultServerManagerOptions;
constexpr sm::ServiceName MemServiceName = sm::ServiceName::Encode("memlet");
constexpr size_t MemMaxSessions = 1;
/* memlet. */
constexpr size_t NumServers = 1;
constexpr size_t NumSessions = MemMaxSessions;
sf::hipc::ServerManager<NumServers, ServerOptions, NumSessions> g_server_manager;
constinit sf::UnmanagedServiceObject<memlet::impl::IService, memlet::Service> g_mem_service_object;
void InitializeAndLoopIpcServer() {
/* Create services. */
R_ABORT_UNLESS(g_server_manager.RegisterObjectForServer(g_mem_service_object.GetShared(), MemServiceName, MemMaxSessions));
/* Loop forever, servicing our services. */
g_server_manager.LoopProcess();
}
}
}
namespace init {
void InitializeSystemModule() {
/* Initialize our connection to sm. */
R_ABORT_UNLESS(sm::Initialize());
/* Verify that we can sanely execute. */
ams::CheckApiVersion();
}
void FinalizeSystemModule() { /* ... */ }
void Startup() { /* ... */ }
}
void Main() {
/* Set thread name. */
os::SetThreadNamePointer(os::GetCurrentThread(), AMS_GET_SYSTEM_THREAD_NAME(memlet, Main));
AMS_ASSERT(os::GetThreadPriority(os::GetCurrentThread()) == AMS_GET_SYSTEM_THREAD_PRIORITY(memlet, Main));
/* Initialize and service our ipc service. */
memlet::InitializeAndLoopIpcServer();
}
}

View File

@@ -0,0 +1,58 @@
/*
* Copyright (c) Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <stratosphere.hpp>
#include "memlet_service.hpp"
namespace ams::memlet {
Result Service::CreateAppletSharedMemory(sf::Out<u64> out_size, sf::OutMoveHandle out_handle, u64 desired_size) {
/* Create a handle to set the output to when done. */
os::NativeHandle handle = os::InvalidNativeHandle;
ON_SCOPE_EXIT { out_handle.SetValue(handle, true); };
/* Check that the requested size has megabyte alignment and isn't too big. */
R_UNLESS(util::IsAligned(desired_size, 1_MB), os::ResultInvalidParameter());
R_UNLESS(desired_size <= 128_MB, os::ResultInvalidParameter());
/* Try to create a shared memory of the desired size, giving up 1 MB each iteration. */
os::SharedMemoryType shmem = {};
while (true) {
/* If we have zero desired-size left, we've failed. */
R_UNLESS(desired_size > 0, os::ResultOutOfMemory());
/* Try to create a shared memory. */
if (R_FAILED(os::CreateSharedMemory(std::addressof(shmem), desired_size, os::MemoryPermission_ReadWrite, os::MemoryPermission_ReadWrite))) {
/* We failed, so decrease the size to see if that works. */
desired_size -= 1_MB;
continue;
}
/* We successfully created the shared memory. */
break;
}
/* Get the native handle for the shared memory we created. */
handle = os::GetSharedMemoryHandle(std::addressof(shmem));
/* HACK: Clear the shared memory object, since we've stolen its handle, and there's no "correct" way to detach. */
shmem = {};
/* We successfully created a shared memory! */
*out_size = desired_size;
R_SUCCEED();
}
}

View File

@@ -0,0 +1,32 @@
/*
* Copyright (c) Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <stratosphere.hpp>
#define AMS_MEMLET_I_SERVICE_INTERFACE_INFO(C, H) \
AMS_SF_METHOD_INFO(C, H, 65000, Result, CreateAppletSharedMemory, (sf::Out<u64> out_size, sf::OutMoveHandle out_handle, u64 desired_size), (out_size, out_handle, desired_size))
AMS_SF_DEFINE_INTERFACE(ams::memlet::impl, IService, AMS_MEMLET_I_SERVICE_INTERFACE_INFO, 0x00000000)
namespace ams::memlet {
class Service {
public:
Result CreateAppletSharedMemory(sf::Out<u64> out_size, sf::OutMoveHandle out_handle, u64 desired_size);
};
static_assert(impl::IsIService<Service>);
}

View File

@@ -0,0 +1,131 @@
#---------------------------------------------------------------------------------
# pull in common stratosphere sysmodule configuration
#---------------------------------------------------------------------------------
THIS_MAKEFILE := $(abspath $(lastword $(MAKEFILE_LIST)))
CURRENT_DIRECTORY := $(abspath $(dir $(THIS_MAKEFILE)))
include $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/../../libraries/config/templates/stratosphere.mk
ATMOSPHERE_SYSTEM_MODULE_TARGETS := nsp
#---------------------------------------------------------------------------------
# no real need to edit anything past this point unless you need to add additional
# rules for different file extensions
#---------------------------------------------------------------------------------
ifneq ($(__RECURSIVE__),1)
#---------------------------------------------------------------------------------
export TOPDIR := $(CURDIR)
export VPATH := $(foreach dir,$(SOURCES),$(CURDIR)/$(dir)) \
$(foreach dir,$(DATA),$(CURDIR)/$(dir))
CFILES := $(call FIND_SOURCE_FILES,$(SOURCES),c)
CPPFILES := $(call FIND_SOURCE_FILES,$(SOURCES),cpp)
SFILES := $(call FIND_SOURCE_FILES,$(SOURCES),s)
BINFILES := $(foreach dir,$(DATA),$(notdir $(wildcard $(dir)/*.*)))
#---------------------------------------------------------------------------------
# use CXX for linking C++ projects, CC for standard C
#---------------------------------------------------------------------------------
ifeq ($(strip $(CPPFILES)),)
#---------------------------------------------------------------------------------
export LD := $(CC)
#---------------------------------------------------------------------------------
else
#---------------------------------------------------------------------------------
export LD := $(CXX)
#---------------------------------------------------------------------------------
endif
#---------------------------------------------------------------------------------
export OFILES := $(addsuffix .o,$(BINFILES)) \
$(CPPFILES:.cpp=.o) $(CFILES:.c=.o) $(SFILES:.s=.o)
export INCLUDE := $(foreach dir,$(INCLUDES),-I$(CURDIR)/$(dir)) \
$(foreach dir,$(LIBDIRS),-I$(dir)/include) \
$(foreach dir,$(AMS_LIBDIRS),-I$(dir)/include) \
-I$(CURDIR)/$(ATMOSPHERE_BUILD_DIR)
export LIBPATHS := $(foreach dir,$(LIBDIRS),-L$(dir)/lib) $(foreach dir,$(AMS_LIBDIRS),-L$(dir)/$(ATMOSPHERE_LIBRARY_DIR))
export BUILD_EXEFS_SRC := $(TOPDIR)/$(EXEFS_SRC)
ifeq ($(strip $(CONFIG_JSON)),)
jsons := $(wildcard *.json)
ifneq (,$(findstring $(TARGET).json,$(jsons)))
export APP_JSON := $(TOPDIR)/$(TARGET).json
else
ifneq (,$(findstring config.json,$(jsons)))
export APP_JSON := $(TOPDIR)/config.json
endif
endif
else
export APP_JSON := $(TOPDIR)/$(CONFIG_JSON)
endif
.PHONY: clean all check_lib
#---------------------------------------------------------------------------------
all: $(ATMOSPHERE_OUT_DIR) $(ATMOSPHERE_BUILD_DIR) $(ATMOSPHERE_LIBRARIES_DIR)/libstratosphere/$(ATMOSPHERE_LIBRARY_DIR)/libstratosphere.a
@$(MAKE) __RECURSIVE__=1 OUTPUT=$(CURDIR)/$(ATMOSPHERE_OUT_DIR)/$(TARGET) \
DEPSDIR=$(CURDIR)/$(ATMOSPHERE_BUILD_DIR) \
--no-print-directory -C $(ATMOSPHERE_BUILD_DIR) \
-f $(THIS_MAKEFILE)
$(ATMOSPHERE_LIBRARIES_DIR)/libstratosphere/$(ATMOSPHERE_LIBRARY_DIR)/libstratosphere.a: check_lib
@$(SILENTCMD)echo "Checked library."
ifeq ($(ATMOSPHERE_CHECKED_LIBSTRATOSPHERE),1)
check_lib:
else
check_lib:
@$(MAKE) --no-print-directory -C $(ATMOSPHERE_LIBRARIES_DIR)/libstratosphere -f $(ATMOSPHERE_LIBRARIES_DIR)/libstratosphere/libstratosphere.mk
endif
$(ATMOSPHERE_OUT_DIR) $(ATMOSPHERE_BUILD_DIR):
@[ -d $@ ] || mkdir -p $@
#---------------------------------------------------------------------------------
clean:
@echo clean ...
@rm -fr $(ATMOSPHERE_OUT_DIR) $(ATMOSPHERE_BUILD_DIR)
#---------------------------------------------------------------------------------
else
.PHONY: all
DEPENDS := $(OFILES:.o=.d)
#---------------------------------------------------------------------------------
# main targets
#---------------------------------------------------------------------------------
all : $(foreach target,$(ATMOSPHERE_SYSTEM_MODULE_TARGETS),$(OUTPUT).$(target))
$(OUTPUT).kip : $(OUTPUT).elf
$(OUTPUT).nsp : $(OUTPUT).nso $(OUTPUT).npdm
$(OUTPUT).nso : $(OUTPUT).elf
$(OUTPUT).elf : $(OFILES)
$(OFILES) : $(ATMOSPHERE_LIBRARIES_DIR)/libstratosphere/$(ATMOSPHERE_LIBRARY_DIR)/libstratosphere.a
%.npdm : %.npdm.json
@echo built ... $< $@
@npdmtool $< $@
@echo built ... $(notdir $@)
#---------------------------------------------------------------------------------
# you need a rule like this for each extension you use as binary data
#---------------------------------------------------------------------------------
%.bin.o : %.bin
#---------------------------------------------------------------------------------
@echo $(notdir $<)
@$(bin2o)
-include $(DEPENDS)
#---------------------------------------------------------------------------------------
endif
#---------------------------------------------------------------------------------------