ams: bump version to 0.14.2

subrepo:
  subdir:   "libraries"
  merged:   "2d522dc6"
upstream:
  origin:   "https://github.com/Atmosphere-NX/Atmosphere-libs"
  branch:   "master"
  commit:   "2d522dc6"
git-subrepo:
  version:  "0.4.1"
  origin:   "???"
  commit:   "???"

exosphere/loader_stub/Makefile

@@ -24,23 +24,9 @@ export VPATH	:=	$(foreach dir,$(SOURCES),$(CURDIR)/$(dir)) \
 			$(foreach dir,$(DATA),$(CURDIR)/$(dir)) \
             $(TOPDIR)/../program
 
-CFILES      :=	$(foreach dir,$(SOURCES),$(filter-out $(notdir $(wildcard $(dir)/*.arch.*.c)) $(notdir $(wildcard $(dir)/*.board.*.c)) $(notdir $(wildcard $(dir)/*.os.*.c)), \
-                                                      $(notdir $(wildcard $(dir)/*.c))))
-CFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.arch.$(ATMOSPHERE_ARCH_NAME).c)))
-CFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.board.$(ATMOSPHERE_BOARD_NAME).c)))
-CFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.os.$(ATMOSPHERE_OS_NAME).c)))
-
-CPPFILES    :=	$(foreach dir,$(SOURCES),$(filter-out $(notdir $(wildcard $(dir)/*.arch.*.cpp)) $(notdir $(wildcard $(dir)/*.board.*.cpp)) $(notdir $(wildcard $(dir)/*.os.*.cpp)), \
-                                                      $(notdir $(wildcard $(dir)/*.cpp))))
-CPPFILES    +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.arch.$(ATMOSPHERE_ARCH_NAME).cpp)))
-CPPFILES    +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.board.$(ATMOSPHERE_BOARD_NAME).cpp)))
-CPPFILES    +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.os.$(ATMOSPHERE_OS_NAME).cpp)))
-
-SFILES      :=	$(foreach dir,$(SOURCES),$(filter-out $(notdir $(wildcard $(dir)/*.arch.*.s)) $(notdir $(wildcard $(dir)/*.board.*.s)) $(notdir $(wildcard $(dir)/*.os.*.s)), \
-                                                      $(notdir $(wildcard $(dir)/*.s))))
-SFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.arch.$(ATMOSPHERE_ARCH_NAME).s)))
-SFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.board.$(ATMOSPHERE_BOARD_NAME).s)))
-SFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.os.$(ATMOSPHERE_OS_NAME).s)))
+CFILES      :=	$(call FIND_SOURCE_FILES,$(SOURCES),c)
+CPPFILES    :=	$(call FIND_SOURCE_FILES,$(SOURCES),cpp)
+SFILES      :=	$(call FIND_SOURCE_FILES,$(SOURCES),s)
 
 BINFILES    := program.lz4 boot_code.lz4
 

exosphere/loader_stub/source/secmon_loader_error.cpp

@@ -19,10 +19,12 @@
 namespace ams::diag {
 
     NORETURN void AbortImpl(const char *file, int line, const char *func, const char *expr, u64 value, const char *format, ...) {
+        AMS_UNUSED(file, line, func, expr, value, format);
         ams::secmon::loader::ErrorReboot();
     }
 
     NORETURN void AbortImpl(const char *file, int line, const char *func, const char *expr, u64 value) {
+        AMS_UNUSED(file, line, func, expr, value);
         ams::secmon::loader::ErrorReboot();
     }
 

exosphere/program/Makefile

@@ -25,23 +25,9 @@ export VPATH	:=	$(foreach dir,$(SOURCES),$(CURDIR)/$(dir)) \
 			$(TOPDIR)/sc7fw \
 			$(TOPDIR)/rebootstub
 
-CFILES      :=	$(foreach dir,$(SOURCES),$(filter-out $(notdir $(wildcard $(dir)/*.arch.*.c)) $(notdir $(wildcard $(dir)/*.board.*.c)) $(notdir $(wildcard $(dir)/*.os.*.c)), \
-                                                      $(notdir $(wildcard $(dir)/*.c))))
-CFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.arch.$(ATMOSPHERE_ARCH_NAME).c)))
-CFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.board.$(ATMOSPHERE_BOARD_NAME).c)))
-CFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.os.$(ATMOSPHERE_OS_NAME).c)))
-
-CPPFILES    :=	$(foreach dir,$(SOURCES),$(filter-out $(notdir $(wildcard $(dir)/*.arch.*.cpp)) $(notdir $(wildcard $(dir)/*.board.*.cpp)) $(notdir $(wildcard $(dir)/*.os.*.cpp)), \
-                                                      $(notdir $(wildcard $(dir)/*.cpp))))
-CPPFILES    +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.arch.$(ATMOSPHERE_ARCH_NAME).cpp)))
-CPPFILES    +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.board.$(ATMOSPHERE_BOARD_NAME).cpp)))
-CPPFILES    +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.os.$(ATMOSPHERE_OS_NAME).cpp)))
-
-SFILES      :=	$(foreach dir,$(SOURCES),$(filter-out $(notdir $(wildcard $(dir)/*.arch.*.s)) $(notdir $(wildcard $(dir)/*.board.*.s)) $(notdir $(wildcard $(dir)/*.os.*.s)), \
-                                                      $(notdir $(wildcard $(dir)/*.s))))
-SFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.arch.$(ATMOSPHERE_ARCH_NAME).s)))
-SFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.board.$(ATMOSPHERE_BOARD_NAME).s)))
-SFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.os.$(ATMOSPHERE_OS_NAME).s)))
+CFILES      :=	$(call FIND_SOURCE_FILES,$(SOURCES),c)
+CPPFILES    :=	$(call FIND_SOURCE_FILES,$(SOURCES),cpp)
+SFILES      :=	$(call FIND_SOURCE_FILES,$(SOURCES),s)
 
 BINFILES    := sc7fw.bin rebootstub.bin
 

exosphere/program/rebootstub/Makefile

@@ -23,23 +23,9 @@ export DEPSDIR  :=  $(CURDIR)/$(BUILD)
 export VPATH	:=	$(foreach dir,$(SOURCES),$(CURDIR)/$(dir)) \
 			$(foreach dir,$(DATA),$(CURDIR)/$(dir))
 
-CFILES      :=	$(foreach dir,$(SOURCES),$(filter-out $(notdir $(wildcard $(dir)/*.arch.*.c)) $(notdir $(wildcard $(dir)/*.board.*.c)) $(notdir $(wildcard $(dir)/*.os.*.c)), \
-                                                      $(notdir $(wildcard $(dir)/*.c))))
-CFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.arch.$(ATMOSPHERE_ARCH_NAME).c)))
-CFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.board.$(ATMOSPHERE_BOARD_NAME).c)))
-CFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.os.$(ATMOSPHERE_OS_NAME).c)))
-
-CPPFILES    :=	$(foreach dir,$(SOURCES),$(filter-out $(notdir $(wildcard $(dir)/*.arch.*.cpp)) $(notdir $(wildcard $(dir)/*.board.*.cpp)) $(notdir $(wildcard $(dir)/*.os.*.cpp)), \
-                                                      $(notdir $(wildcard $(dir)/*.cpp))))
-CPPFILES    +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.arch.$(ATMOSPHERE_ARCH_NAME).cpp)))
-CPPFILES    +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.board.$(ATMOSPHERE_BOARD_NAME).cpp)))
-CPPFILES    +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.os.$(ATMOSPHERE_OS_NAME).cpp)))
-
-SFILES      :=	$(foreach dir,$(SOURCES),$(filter-out $(notdir $(wildcard $(dir)/*.arch.*.s)) $(notdir $(wildcard $(dir)/*.board.*.s)) $(notdir $(wildcard $(dir)/*.os.*.s)), \
-                                                      $(notdir $(wildcard $(dir)/*.s))))
-SFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.arch.$(ATMOSPHERE_ARCH_NAME).s)))
-SFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.board.$(ATMOSPHERE_BOARD_NAME).s)))
-SFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.os.$(ATMOSPHERE_OS_NAME).s)))
+CFILES      :=	$(call FIND_SOURCE_FILES,$(SOURCES),c)
+CPPFILES    :=	$(call FIND_SOURCE_FILES,$(SOURCES),cpp)
+SFILES      :=	$(call FIND_SOURCE_FILES,$(SOURCES),s)
 
 #---------------------------------------------------------------------------------
 # use CXX for linking C++ projects, CC for standard C

exosphere/program/sc7fw/Makefile

@@ -23,23 +23,9 @@ export DEPSDIR  :=  $(CURDIR)/$(BUILD)
 export VPATH	:=	$(foreach dir,$(SOURCES),$(CURDIR)/$(dir)) \
 			$(foreach dir,$(DATA),$(CURDIR)/$(dir))
 
-CFILES      :=	$(foreach dir,$(SOURCES),$(filter-out $(notdir $(wildcard $(dir)/*.arch.*.c)) $(notdir $(wildcard $(dir)/*.board.*.c)) $(notdir $(wildcard $(dir)/*.os.*.c)), \
-                                                      $(notdir $(wildcard $(dir)/*.c))))
-CFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.arch.$(ATMOSPHERE_ARCH_NAME).c)))
-CFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.board.$(ATMOSPHERE_BOARD_NAME).c)))
-CFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.os.$(ATMOSPHERE_OS_NAME).c)))
-
-CPPFILES    :=	$(foreach dir,$(SOURCES),$(filter-out $(notdir $(wildcard $(dir)/*.arch.*.cpp)) $(notdir $(wildcard $(dir)/*.board.*.cpp)) $(notdir $(wildcard $(dir)/*.os.*.cpp)), \
-                                                      $(notdir $(wildcard $(dir)/*.cpp))))
-CPPFILES    +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.arch.$(ATMOSPHERE_ARCH_NAME).cpp)))
-CPPFILES    +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.board.$(ATMOSPHERE_BOARD_NAME).cpp)))
-CPPFILES    +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.os.$(ATMOSPHERE_OS_NAME).cpp)))
-
-SFILES      :=	$(foreach dir,$(SOURCES),$(filter-out $(notdir $(wildcard $(dir)/*.arch.*.s)) $(notdir $(wildcard $(dir)/*.board.*.s)) $(notdir $(wildcard $(dir)/*.os.*.s)), \
-                                                      $(notdir $(wildcard $(dir)/*.s))))
-SFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.arch.$(ATMOSPHERE_ARCH_NAME).s)))
-SFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.board.$(ATMOSPHERE_BOARD_NAME).s)))
-SFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.os.$(ATMOSPHERE_OS_NAME).s)))
+CFILES      :=	$(call FIND_SOURCE_FILES,$(SOURCES),c)
+CPPFILES    :=	$(call FIND_SOURCE_FILES,$(SOURCES),cpp)
+SFILES      :=	$(call FIND_SOURCE_FILES,$(SOURCES),s)
 
 #---------------------------------------------------------------------------------
 # use CXX for linking C++ projects, CC for standard C

exosphere/program/sc7fw/source/sc7fw_main.cpp

@@ -90,7 +90,7 @@ namespace ams::sc7fw {
             reg::ReadWrite(PMC + APBDEV_PMC_DPD_ENABLE, PMC_REG_BITS_ENUM(DPD_ENABLE_ON, ENABLE));
 
             /* Wait forever until we're asleep. */
-            while (true) { /* ... */ }
+            AMS_INFINITE_LOOP();
         }
 
     }
@@ -102,7 +102,9 @@ namespace ams::sc7fw {
     NORETURN void ExceptionHandler() {
         /* Write enable to MAIN_RESET. */
         reg::Write(PMC + APBDEV_PMC_CNTRL, PMC_REG_BITS_ENUM(CNTRL_MAIN_RESET, ENABLE));
-        while (true) { /* ... */ }
+
+        /* Wait forever until we're reset. */
+        AMS_INFINITE_LOOP();
     }
 
 }

exosphere/program/source/boot/secmon_boot_setup.cpp

@@ -85,6 +85,8 @@ namespace ams::secmon::boot {
             if constexpr (false) {
                 /* TODO: Consider implementing this as a reference. */
             }
+
+            AMS_UNUSED(is_prod);
         }
 
         /* NOTE: These are just latest-master-kek encrypted with BEK. */
@@ -406,6 +408,7 @@ namespace ams::secmon::boot {
 
         constexpr void UnmapDramImpl(u64 *l1, u64 *l2, u64 *l3) {
             /* Unmap the L1 entry corresponding to to the Dram entries. */
+            AMS_UNUSED(l2, l3);
             InvalidateL1Entries(l1, MemoryRegionDram.GetAddress(), MemoryRegionDram.GetSize());
         }
 

exosphere/program/source/secmon_map.cpp

@@ -41,6 +41,7 @@ namespace ams::secmon {
 
         constexpr void UnmapBootCodeImpl(u64 *l1, u64 *l2, u64 *l3, uintptr_t boot_code, size_t boot_code_size) {
             /* Unmap the L3 entries corresponding to the boot code. */
+            AMS_UNUSED(l1, l2);
             InvalidateL3Entries(l3, boot_code, boot_code_size);
         }
 

exosphere/program/source/smc/secmon_smc_memory_access.cpp

@@ -69,6 +69,7 @@ namespace ams::secmon::smc {
 
     SmcResult SmcWriteAddress(SmcArguments &args) {
         /* NOTE: This smc was deprecated in Atmosphère 0.13.0. */
+        AMS_UNUSED(args);
         return SmcResult::NotImplemented;
     }
 

exosphere/program/source/smc/secmon_smc_power_management.cpp

@@ -492,6 +492,8 @@ namespace ams::secmon::smc {
     }
 
     SmcResult SmcPowerOffCpu(SmcArguments &args) {
+        AMS_UNUSED(args);
+
         /* Get the current core id. */
         const auto core_id = hw::GetCurrentCoreId();
 

exosphere/program/source/smc/secmon_smc_register_access.cpp

@@ -155,10 +155,27 @@ namespace ams::secmon::smc {
         /* Find the access table. */
         const AccessTableEntry * const entry = GetAccessTableEntry(address);
 
-        /* If we have a table, perform the write. */
+        /* Translate our entry into an address to access. */
+        uintptr_t virtual_address = 0;
         if (entry != nullptr) {
             /* Get the address to read or write. */
-            const uintptr_t virtual_address = entry->virtual_address + (address - entry->address);
+            virtual_address = entry->virtual_address + (address - entry->address);
+        } else {
+            /* For no clearly discernable reason, SmcReadWriteRegister returns success despite not doing the read/write */
+            /* when accessing the SMMU controls for the BPMP and for APB-DMA. */
+            /* This is "probably" to fuck with hackers who got access to the SMC and are trying to get control of the */
+            /* BPMP to exploit jamais vu, deja vu, or other related DMA/wake-from-sleep vulnerabilities. */
+            constexpr uintptr_t MC = MemoryRegionPhysicalDeviceMemoryController.GetAddress();
+            SMC_R_UNLESS((address == (MC + MC_SMMU_AVPC_ASID) || address == (MC + MC_SMMU_PPCS1_ASID)), InvalidArgument);
+
+            /* For backwards compatibility, we'll allow access to these devices on 1.0.0. */
+            if (GetTargetFirmware() < TargetFirmware_2_0_0) {
+                virtual_address = MemoryRegionVirtualDeviceMemoryController.GetAddress() + (address - MC);
+            }
+        }
+
+        /* Perform the read or write, if we should. */
+        if (virtual_address != 0) {
             u32 out = 0;
 
             if (mask != ~static_cast<u32>(0)) {
@@ -169,13 +186,6 @@ namespace ams::secmon::smc {
             }
 
             args.r[1] = out;
-        } else {
-            /* For no clearly discernable reason, SmcReadWriteRegister returns success despite not doing the read/write */
-            /* when accessing the SMMU controls for the BPMP and for APB-DMA. */
-            /* This is "probably" to fuck with hackers who got access to the SMC and are trying to get control of the */
-            /* BPMP to exploit jamais vu, deja vu, or other related DMA/wake-from-sleep vulnerabilities. */
-            constexpr uintptr_t MC = MemoryRegionPhysicalDeviceMemoryController.GetAddress();
-            SMC_R_UNLESS((address == (MC + MC_SMMU_AVPC_ASID) || address == (MC + MC_SMMU_PPCS1_ASID)), InvalidArgument);
         }
 
         return SmcResult::Success;

exosphere/program/source/smc/secmon_smc_rsa.cpp

@@ -75,6 +75,7 @@ namespace ams::secmon::smc {
                 u8 msg[se::RsaSize];
             public:
                 void Set(const void *m, size_t m_size) {
+                    AMS_UNUSED(m_size);
                     std::memcpy(this->msg, m, sizeof(this->msg));
                 }
 

exosphere/warmboot/Makefile

@@ -23,23 +23,9 @@ export DEPSDIR  :=  $(CURDIR)/$(BUILD)
 export VPATH	:=	$(foreach dir,$(SOURCES),$(CURDIR)/$(dir)) \
 			$(foreach dir,$(DATA),$(CURDIR)/$(dir))
 
-CFILES      :=	$(foreach dir,$(SOURCES),$(filter-out $(notdir $(wildcard $(dir)/*.arch.*.c)) $(notdir $(wildcard $(dir)/*.board.*.c)) $(notdir $(wildcard $(dir)/*.os.*.c)), \
-                                                      $(notdir $(wildcard $(dir)/*.c))))
-CFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.arch.$(ATMOSPHERE_ARCH_NAME).c)))
-CFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.board.$(ATMOSPHERE_BOARD_NAME).c)))
-CFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.os.$(ATMOSPHERE_OS_NAME).c)))
-
-CPPFILES    :=	$(foreach dir,$(SOURCES),$(filter-out $(notdir $(wildcard $(dir)/*.arch.*.cpp)) $(notdir $(wildcard $(dir)/*.board.*.cpp)) $(notdir $(wildcard $(dir)/*.os.*.cpp)), \
-                                                      $(notdir $(wildcard $(dir)/*.cpp))))
-CPPFILES    +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.arch.$(ATMOSPHERE_ARCH_NAME).cpp)))
-CPPFILES    +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.board.$(ATMOSPHERE_BOARD_NAME).cpp)))
-CPPFILES    +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.os.$(ATMOSPHERE_OS_NAME).cpp)))
-
-SFILES      :=	$(foreach dir,$(SOURCES),$(filter-out $(notdir $(wildcard $(dir)/*.arch.*.s)) $(notdir $(wildcard $(dir)/*.board.*.s)) $(notdir $(wildcard $(dir)/*.os.*.s)), \
-                                                      $(notdir $(wildcard $(dir)/*.s))))
-SFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.arch.$(ATMOSPHERE_ARCH_NAME).s)))
-SFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.board.$(ATMOSPHERE_BOARD_NAME).s)))
-SFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.os.$(ATMOSPHERE_OS_NAME).s)))
+CFILES      :=	$(call FIND_SOURCE_FILES,$(SOURCES),c)
+CPPFILES    :=	$(call FIND_SOURCE_FILES,$(SOURCES),cpp)
+SFILES      :=	$(call FIND_SOURCE_FILES,$(SOURCES),s)
 
 #---------------------------------------------------------------------------------
 # use CXX for linking C++ projects, CC for standard C

exosphere/warmboot/source/warmboot_main.cpp

@@ -90,7 +90,9 @@ namespace ams::warmboot {
     NORETURN void ExceptionHandler() {
         /* Write enable to MAIN_RESET. */
         reg::Write(PMC + APBDEV_PMC_CNTRL, PMC_REG_BITS_ENUM(CNTRL_MAIN_RESET, ENABLE));
-        while (true) { /* ... */ }
+
+        /* Wait forever until we're reset. */
+        AMS_INFINITE_LOOP();
     }
 
 }

fusee/fusee-secondary/Makefile

@@ -89,7 +89,7 @@ export VPATH	:=	$(foreach dir,$(SOURCES),$(CURDIR)/$(dir)) \
 			$(foreach dir,$(DATA),$(CURDIR)/$(dir)) \
 			$(AMS)/exosphere $(AMS)/exosphere/warmboot $(AMS)/exosphere/program/rebootstub \
 			$(AMS)/thermosphere $(AMS)/fusee/fusee-primary $(AMS)/sept/sept-primary \
-			$(AMS)/sept/sept-secondary $(AMS)/emummc $(AMS)/mesosphere/kernel_ldr $(KIPDIRS)
+			$(AMS)/sept/sept-secondary $(AMS)/emummc $(AMS)/mesosphere $(AMS)/mesosphere/kernel_ldr $(KIPDIRS)
 
 export DEPSDIR	:=	$(CURDIR)/$(BUILD)
 
@@ -100,7 +100,7 @@ KIPFILES 	:=	loader.kip ncm.kip pm.kip sm.kip ams_mitm.kip spl.kip boot.kip
 BINFILES	:=	$(foreach dir,$(DATA),$(notdir $(wildcard $(dir)/*.*))) fusee-primary.bin \
 				exosphere.bin warmboot.bin rebootstub.bin thermosphere.bin splash_screen.bmp \
 				sept-primary.bin sept-secondary_00.enc sept-secondary_01.enc emummc.kip \
-				sept-secondary_dev_00.enc sept-secondary_dev_01.enc kernel_ldr.bin $(KIPFILES)
+				sept-secondary_dev_00.enc sept-secondary_dev_01.enc mesosphere.bin kernel_ldr.bin $(KIPFILES)
 
 #---------------------------------------------------------------------------------
 # use CXX for linking C++ projects, CC for standard C

fusee/fusee-secondary/src/car.h

@@ -13,7 +13,7 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
- 
+
 #ifndef FUSEE_CAR_H
 #define FUSEE_CAR_H
 
@@ -37,23 +37,27 @@
 
 /* Clock and reset devices. */
 typedef enum {
-    CARDEVICE_UARTA = ((0 << 5) | 0x6),
-    CARDEVICE_UARTB = ((0 << 5) | 0x7),
-    CARDEVICE_UARTC = ((1 << 5) | 0x17),
-    CARDEVICE_I2C1 = ((0 << 5) | 0xC),
-    CARDEVICE_I2C5 = ((1 << 5) | 0xF),
-    CARDEVICE_TZRAM = ((3 << 5) | 0x1E),
-    CARDEVICE_SE = ((3 << 5) | 0x1F),
-    CARDEVICE_HOST1X = ((0 << 5) | 0x1C),
-    CARDEVICE_TSEC = ((2 << 5) | 0x13),
-    CARDEVICE_SOR_SAFE = ((6 << 5) | 0x1E),
-    CARDEVICE_SOR0 = ((5 << 5) | 0x16),
-    CARDEVICE_SOR1 = ((5 << 5) | 0x17),
-    CARDEVICE_KFUSE = ((1 << 5) | 0x8),
-    CARDEVICE_CL_DVFS = ((4 << 5) | 0x1B),
+    CARDEVICE_BPMP      = ((0 << 5) | 0x1),
+    CARDEVICE_UARTA     = ((0 << 5) | 0x6),
+    CARDEVICE_UARTB     = ((0 << 5) | 0x7),
+    CARDEVICE_I2C1      = ((0 << 5) | 0xC),
+    CARDEVICE_USBD      = ((0 << 5) | 0x16),
+    CARDEVICE_HOST1X    = ((0 << 5) | 0x1C),
+    CARDEVICE_AHBDMA    = ((1 << 5) | 0x1),
+    CARDEVICE_APBDMA    = ((1 << 5) | 0x2),
+    CARDEVICE_KFUSE     = ((1 << 5) | 0x8),
+    CARDEVICE_I2C5      = ((1 << 5) | 0xF),
+    CARDEVICE_UARTC     = ((1 << 5) | 0x17),
+    CARDEVICE_USB2      = ((1 << 5) | 0x1A),
     CARDEVICE_CORESIGHT = ((2 << 5) | 0x9),
-    CARDEVICE_ACTMON = ((3 << 5) | 0x17),
-    CARDEVICE_BPMP = ((0 << 5) | 0x1)
+    CARDEVICE_TSEC      = ((2 << 5) | 0x13),
+    CARDEVICE_ACTMON    = ((3 << 5) | 0x17),
+    CARDEVICE_TZRAM     = ((3 << 5) | 0x1E),
+    CARDEVICE_SE        = ((3 << 5) | 0x1F),
+    CARDEVICE_CL_DVFS   = ((4 << 5) | 0x1B),
+    CARDEVICE_SOR0      = ((5 << 5) | 0x16),
+    CARDEVICE_SOR1      = ((5 << 5) | 0x17),
+    CARDEVICE_SOR_SAFE  = ((6 << 5) | 0x1E),
 } CarDevice;
 
 /* Clock/Reset Controller (CLK_RST_CONTROLLER_) regs */
@@ -97,31 +101,31 @@ typedef struct {
     uint32_t pllc_out;
     uint32_t pllc_misc0;
     uint32_t pllc_misc1;
-    
+
     /* PLLM 0x90-0x9c */
     uint32_t pllm_base;
     uint32_t pllm_out;
     uint32_t pllm_misc1;
     uint32_t pllm_misc2;
-    
+
     /* PLLP 0xa0-0xac */
     uint32_t pllp_base;
     uint32_t pllp_outa;
     uint32_t pllp_outb;
     uint32_t pllp_misc;
-    
+
     /* PLLA 0xb0-0xbc */
     uint32_t plla_base;
     uint32_t plla_out;
     uint32_t plla_misc0;
     uint32_t plla_misc1;
-    
+
     /* PLLU 0xc0-0xcc */
     uint32_t pllu_base;
     uint32_t pllu_out;
     uint32_t pllu_misc1;
     uint32_t pllu_misc2;
-    
+
     /* PLLD 0xd0-0xdc */
     uint32_t plld_base;
     uint32_t plld_out;
@@ -131,13 +135,13 @@ typedef struct {
     /* PLLX 0xe0-0xe4 */
     uint32_t pllx_base;
     uint32_t pllx_misc;
-    
+
     /* PLLE 0xe8-0xf4 */
     uint32_t plle_base;
     uint32_t plle_misc;
     uint32_t plle_ss_cntl1;
     uint32_t plle_ss_cntl2;
-    
+
     uint32_t lvl2_clk_gate_ovra;            /* _LVL2_CLK_GATE_OVRA_0,   0xf8 */
     uint32_t lvl2_clk_gate_ovrb;            /* _LVL2_CLK_GATE_OVRB_0,   0xfc */
 
@@ -188,7 +192,7 @@ typedef struct {
     uint32_t _0x1e0[5];
     uint32_t clk_source_tsec;               /* _CLK_SOURCE_TSEC_0,      0x1f4 */
     uint32_t _0x1f8;
-    
+
     uint32_t clk_spare2;                    /* _CLK_SPARE2_0,       0x1fc */
     uint32_t _0x200[32];
 
@@ -257,7 +261,7 @@ typedef struct {
     uint32_t lvl2_clk_gate_ovrc;        /* _LVL2_CLK_GATE_OVRC,     0x3a0 */
     uint32_t lvl2_clk_gate_ovrd;        /* _LVL2_CLK_GATE_OVRD,     0x3a4 */
     uint32_t _0x3a8[2];
-    
+
     uint32_t _0x3b0;
     uint32_t clk_source_mselect;        /* _CLK_SOURCE_MSELECT_0,       0x3b4 */
     uint32_t clk_source_tsensor;        /* _CLK_SOURCE_TSENSOR_0,       0x3b8 */
@@ -371,13 +375,13 @@ typedef struct {
     uint32_t spare_reg0;            /* _SPARE_REG0_0, 0x55c */
     uint32_t audio_sync_clk_dmic1;  /* _AUDIO_SYNC_CLK_DMIC1_0, 0x560 */
     uint32_t audio_sync_clk_dmic2;  /* _AUDIO_SYNC_CLK_DMIC2_0, 0x564 */
-    
+
     uint32_t _0x568[2];
     uint32_t plld2_ss_cfg;          /* _PLLD2_SS_CFG,           0x570 */
     uint32_t plld2_ss_ctrl1;        /* _PLLD2_SS_CTRL1_0,       0x574 */
     uint32_t plld2_ss_ctrl2;        /* _PLLD2_SS_CTRL2_0,       0x578 */
     uint32_t _0x57c[5];
-    
+
     uint32_t plldp_base;            /* _PLLDP_BASE, 0x590*/
     uint32_t plldp_misc;            /* _PLLDP_MISC, 0x594 */
     uint32_t plldp_ss_cfg;          /* _PLLDP_SS_CFG, 0x598 */
@@ -399,7 +403,7 @@ typedef struct {
     uint32_t pllx_misc4;            /* _PLLX_MISC_4_0, 0x5f0 */
     uint32_t pllx_misc5;            /* _PLLX_MISC_5_0, 0x5f4 */
     uint32_t _0x5f8[2];
-    
+
     uint32_t clk_source_xusb_core_host;             /* _CLK_SOURCE_XUSB_CORE_HOST_0,    0x600 */
     uint32_t clk_source_xusb_falcon;                /* _CLK_SOURCE_XUSB_FALCON_0,    0x604 */
     uint32_t clk_source_xusb_fs;                    /* _CLK_SOURCE_XUSB_FS_0,    0x608 */
@@ -428,7 +432,7 @@ typedef struct {
     uint32_t clk_source_uart_fst_mipi_cal;          /* _CLK_SOURCE_UART_FST_MIPI_CAL_0,    0x66c */
     uint32_t _0x670[2];
     uint32_t clk_source_vic;                        /* _CLK_SOURCE_VIC_0,    0x678 */
-    
+
     uint32_t pllp_outc;                             /* _PLLP_OUTC_0, 0x67c */
     uint32_t pllp_misc1;                            /* _PLLP_MISC1_0, 0x680 */
     uint32_t _0x684[2];
@@ -439,14 +443,14 @@ typedef struct {
     uint32_t clk_source_nvdec;                      /* _CLK_SOURCE_NVDEC_0,    0x698 */
     uint32_t clk_source_nvjpg;                      /* _CLK_SOURCE_NVJPG_0,    0x69c */
     uint32_t clk_source_nvenc;                      /* _CLK_SOURCE_NVENC_0,    0x6a0 */
-    
+
     uint32_t plla1_base;                            /* _PLLA1_BASE_0, 0x6a4 */
     uint32_t plla1_misc0;                           /* _PLLA1_MISC_0_0, 0x6a8 */
     uint32_t plla1_misc1;                           /* _PLLA1_MISC_1_0, 0x6ac */
     uint32_t plla1_misc2;                           /* _PLLA1_MISC_2_0, 0x6b0 */
     uint32_t plla1_misc3;                           /* _PLLA1_MISC_3_0, 0x6b4 */
     uint32_t audio_sync_clk_dmic3;                  /* _AUDIO_SYNC_CLK_DMIC3_0, 0x6b8 */
-    
+
     uint32_t clk_source_dmic3;                      /* _CLK_SOURCE_DMIC3_0,    0x6bc */
     uint32_t clk_source_ape;                        /* _CLK_SOURCE_APE_0,    0x6c0 */
     uint32_t clk_source_qspi;                       /* _CLK_SOURCE_QSPI_0,    0x6c4 */
@@ -455,11 +459,11 @@ typedef struct {
     uint32_t clk_source_pex_sata_usb_rx_byp;        /* _CLK_SOURCE_PEX_SATA_USB_RX_BYP_0,    0x6d0 */
     uint32_t clk_source_maud;                       /* _CLK_SOURCE_MAUD_0,    0x6d4 */
     uint32_t clk_source_tsecb;                      /* _CLK_SOURCE_TSECB_0,    0x6d8 */
-    
+
     uint32_t clk_cpug_misc1;                        /* _CLK_CPUG_MISC1_0, 0x6dc */
     uint32_t aclk_burst_policy;                     /* _ACLK_BURST_POLICY_0, 0x6e0 */
     uint32_t super_aclk_divider;                    /* _SUPER_ACLK_DIVIDER_0, 0x6e4 */
-    
+
     uint32_t nvenc_super_clk_divider;               /* _NVENC_SUPER_CLK_DIVIDER_0, 0x6e8 */
     uint32_t vi_super_clk_divider;                  /* _VI_SUPER_CLK_DIVIDER_0, 0x6ec */
     uint32_t vic_super_clk_divider;                 /* _VIC_SUPER_CLK_DIVIDER_0, 0x6f0 */
@@ -470,7 +474,7 @@ typedef struct {
     uint32_t se_super_clk_divider;                  /* _SE_SUPER_CLK_DIVIDER_0, 0x704 */
     uint32_t tsec_super_clk_divider;                /* _TSEC_SUPER_CLK_DIVIDER_0, 0x708 */
     uint32_t tsecb_super_clk_divider;               /* _TSECB_SUPER_CLK_DIVIDER_0, 0x70c */
-    
+
     uint32_t clk_source_uartape;                    /* _CLK_SOURCE_UARTAPE_0, 0x710 */
     uint32_t clk_cpug_misc2;                        /* _CLK_CPUG_MISC2_0, 0x714 */
     uint32_t clk_source_dbgapb;                     /* _CLK_SOURCE_DBGAPB_0, 0x718 */
@@ -484,7 +488,7 @@ typedef struct {
     uint32_t sdmmc4_pllc4_out0_shaper_ctrl;         /* _SDMMC4_PLLC4_OUT0_SHAPER_CTRL_0, 0x738 */
     uint32_t sdmmc4_pllc4_out1_shaper_ctrl;         /* _SDMMC4_PLLC4_OUT1_SHAPER_CTRL_0, 0x73c */
     uint32_t sdmmc4_pllc4_out2_shaper_ctrl;         /* _SDMMC4_PLLC4_OUT2_SHAPER_CTRL_0, 0x740 */
-    uint32_t sdmmc4_div_clk_shaper_ctrl;            /* _SDMMC4_DIV_CLK_SHAPER_CTRL_0, 0x744 */    
+    uint32_t sdmmc4_div_clk_shaper_ctrl;            /* _SDMMC4_DIV_CLK_SHAPER_CTRL_0, 0x744 */
 } tegra_car_t;
 
 static inline volatile tegra_car_t *car_get_regs(void) {

fusee/fusee-secondary/src/kernel_patches.c

@@ -963,11 +963,6 @@ void package2_patch_kernel(void *_kernel, size_t *kernel_size, bool is_sd_kernel
     }
 
     if (kernel_info == NULL && is_sd_kernel) {
-        /* If the kernel is mesosphere, patch it. */
-        if (*(volatile uint32_t *)((uintptr_t)_kernel + 4) == 0x3053534D) {
-            *out_ini1 = (void *)((uintptr_t)_kernel + *(volatile uint32_t *)((uintptr_t)_kernel + 8));
-            *(volatile uint64_t *)((uintptr_t)_kernel + 8) = (uint64_t)*kernel_size;
-        }
         return;
     }
 

fusee/fusee-secondary/src/nxboot.c

@@ -24,6 +24,7 @@
 #include "nxboot.h"
 #include "nxfs.h"
 #include "bct.h"
+#include "car.h"
 #include "di.h"
 #include "mc.h"
 #include "se.h"
@@ -633,6 +634,8 @@ uint32_t nxboot_main(void) {
     void *warmboot_memaddr;
     void *package1loader;
     size_t package1loader_size;
+    void *mesosphere;
+    size_t mesosphere_size;
     void *emummc;
     size_t emummc_size;
     uint32_t available_revision;
@@ -928,6 +931,28 @@ uint32_t nxboot_main(void) {
             pmc->scratch1 = (uint32_t)warmboot_memaddr;
     }
 
+    /* Configure mesosphere. */
+    /* TODO: Support non-SD/embedded mesosphere. */
+    {
+        size_t sd_meso_size = get_file_size("atmosphere/mesosphere.bin");
+        if (sd_meso_size != 0) {
+            if (sd_meso_size > PACKAGE2_SIZE_MAX) {
+                fatal_error("Error: atmosphere/mesosphere.bin is too large!\n");
+            }
+            mesosphere = malloc(sd_meso_size);
+            if (mesosphere == NULL) {
+                fatal_error("Error: failed to allocate mesosphere!\n");
+            }
+            if (read_from_file(mesosphere, sd_meso_size, "atmosphere/mesosphere.bin") != sd_meso_size) {
+                fatal_error("Error: failed to read atmosphere/mesosphere.bin!\n");
+            }
+            mesosphere_size = sd_meso_size;
+        } else {
+            mesosphere = NULL;
+            mesosphere_size = 0;
+        }
+    }
+
     print(SCREEN_LOG_LEVEL_INFO, "[NXBOOT] Rebuilding package2...\n");
 
     /* Parse stratosphere config. */
@@ -936,7 +961,7 @@ uint32_t nxboot_main(void) {
     print(SCREEN_LOG_LEVEL_INFO, u8"[NXBOOT] Configured Stratosphere...\n");
 
     /* Patch package2, adding Thermosphère + custom KIPs. */
-    package2_rebuild_and_copy(package2, MAILBOX_EXOSPHERE_CONFIGURATION->target_firmware, emummc, emummc_size);
+    package2_rebuild_and_copy(package2, MAILBOX_EXOSPHERE_CONFIGURATION->target_firmware, mesosphere, mesosphere_size, emummc, emummc_size);
 
     /* Set detected FS version. */
     MAILBOX_EXOSPHERE_CONFIGURATION->emummc_cfg.base_cfg.fs_version = stratosphere_get_fs_version();
@@ -991,6 +1016,12 @@ uint32_t nxboot_main(void) {
     /* Wait for the splash screen to have been displayed for as long as it should be. */
     splash_screen_wait_delay();
 
+    /* Set reset for USBD, USB2, AHBDMA, and APBDMA. */
+    rst_enable(CARDEVICE_USBD);
+    rst_enable(CARDEVICE_USB2);
+    rst_enable(CARDEVICE_AHBDMA);
+    rst_enable(CARDEVICE_APBDMA);
+
     /* Return the memory address for booting CPU0. */
     return (uint32_t)exosphere_memaddr;
 }

fusee/fusee-secondary/src/package2.c

@@ -44,7 +44,7 @@ static inline size_t align_to_4(size_t s) {
     return ((s + 3) >> 2) << 2;
 }
 
-void package2_rebuild_and_copy(package2_header_t *package2, uint32_t target_firmware, void *emummc, size_t emummc_size) {
+void package2_rebuild_and_copy(package2_header_t *package2, uint32_t target_firmware, void *mesosphere, size_t mesosphere_size, void *emummc, size_t emummc_size) {
     package2_header_t *rebuilt_package2;
     size_t rebuilt_package2_size;
     void *kernel;
@@ -95,10 +95,28 @@ void package2_rebuild_and_copy(package2_header_t *package2, uint32_t target_firm
         fatal_error("Error: inappropriate kernel embedded ini context");
     }
 
+    /* Use mesosphere instead of Nintendo's kernel when present. */
+    const bool is_mesosphere = mesosphere != NULL && mesosphere_size != 0;
+    if (is_mesosphere) {
+        kernel = mesosphere;
+        kernel_size = mesosphere_size;
+
+        /* Patch mesosphere to use our rebuilt ini. */
+        *(volatile uint64_t *)((uintptr_t)mesosphere + 8) = (uint64_t)mesosphere_size;
+
+        /* Place the kernel section at the correct location. */
+        package2->metadata.section_offsets[PACKAGE2_SECTION_KERNEL] = 0x60000;
+        package2->metadata.entrypoint                               = 0x60000;
+
+        print(SCREEN_LOG_LEVEL_DEBUG, "Using Mesosphere...\n");
+    }
+
     print(SCREEN_LOG_LEVEL_DEBUG, "Rebuilding the INI1 section...\n");
     if (target_firmware < ATMOSPHERE_TARGET_FIRMWARE_8_0_0) {
         package2_get_src_section((void *)&orig_ini1, package2, PACKAGE2_SECTION_INI1);
-    } else {
+    }
+
+    if (target_firmware >= ATMOSPHERE_TARGET_FIRMWARE_8_0_0 || is_mesosphere) {
         /* On 8.0.0, place INI1 right after kernelldr for our sanity. */
         package2->metadata.section_offsets[PACKAGE2_SECTION_INI1] = package2->metadata.section_offsets[PACKAGE2_SECTION_KERNEL] + kernel_size;
     }

fusee/fusee-secondary/src/package2.h

@@ -94,6 +94,6 @@ static inline uint8_t package2_meta_get_header_version(const package2_meta_t *me
     return (uint8_t)((metadata->ctr_dwords[1] ^ (metadata->ctr_dwords[1] >> 16) ^ (metadata->ctr_dwords[1] >> 24)) & 0xFF);
 }
 
-void package2_rebuild_and_copy(package2_header_t *package2, uint32_t target_firmware, void *emummc, size_t emummc_size);
+void package2_rebuild_and_copy(package2_header_t *package2, uint32_t target_firmware, void *mesosphere, size_t mesosphere_size, void *emummc, size_t emummc_size);
 
 #endif

libraries/.gitrepo

@@ -6,7 +6,7 @@
 [subrepo]
 	remote = https://github.com/Atmosphere-NX/Atmosphere-libs
 	branch = master
-	commit = cac5957d3f4b1417cf76a83cf704a14a254dd4dc
-	parent = 3726def6ecc547e64912ddb050737ebd296366e7
+	commit = 2d522dc6a12b2eb5eb3f103a8c5b5126ca301b1a
+	parent = 35fffade4ea4fdbba9cf2443c321e724c9e70c3d
 	method = merge
 	cmdver = 0.4.1

libraries/config/arch/arm64/arch.mk

@@ -5,7 +5,7 @@ endif
 include $(DEVKITPRO)/devkitA64/base_rules
 
 export ATMOSPHERE_DEFINES  += -DATMOSPHERE_ARCH_ARM64
-export ATMOSPHERE_SETTINGS += -march=armv8-a+crc+crypto -mtp=soft
+export ATMOSPHERE_SETTINGS += -mtp=soft
 export ATMOSPHERE_CFLAGS   +=
 export ATMOSPHERE_CXXFLAGS +=
 export ATMOSPHERE_ASFLAGS  +=

libraries/config/arch/arm64/cpu/cortex_a57/cpu.mk

@@ -1,5 +1,5 @@
 export ATMOSPHERE_DEFINES  += -DATMOSPHERE_CPU_ARM_CORTEX_A57
-export ATMOSPHERE_SETTINGS += -mtune=cortex-a57
+export ATMOSPHERE_SETTINGS += -march=armv8-a+crc+crypto -mtune=cortex-a57
 export ATMOSPHERE_CFLAGS   +=
 export ATMOSPHERE_CXXFLAGS +=
 export ATMOSPHERE_ASFLAGS  +=

libraries/config/common.mk

@@ -10,7 +10,7 @@ ifeq ($(strip $(ATMOSPHERE_BOARD)),)
 export ATMOSPHERE_BOARD := nx-hac-001
 
 ifeq ($(strip $(ATMOSPHERE_CPU)),)
-export ATMOSPHERE_CPU   := arm-cortex-a57
+export ATMOSPHERE_CPU            := arm-cortex-a57
 endif
 
 endif
@@ -28,40 +28,44 @@ export ATMOSPHERE_ASFLAGS  :=
 ifeq ($(ATMOSPHERE_BOARD),nx-hac-001)
 
 ifeq ($(ATMOSPHERE_CPU),arm-cortex-a57)
-export ATMOSPHERE_ARCH_DIR   := arch/arm64
-export ATMOSPHERE_BOARD_DIR  := board/nintendo/nx
-export ATMOSPHERE_OS_DIR     := os/horizon
+export ATMOSPHERE_ARCH_DIR   := arm64
+export ATMOSPHERE_BOARD_DIR  := nintendo/nx
+export ATMOSPHERE_OS_DIR     := horizon
 
 export ATMOSPHERE_ARCH_NAME  := arm64
 export ATMOSPHERE_BOARD_NAME := nintendo_nx
 export ATMOSPHERE_OS_NAME    := horizon
+
+export ATMOSPHERE_CPU_EXTENSIONS := arm_crypto_extension aarch64_crypto_extension
 else ifeq ($(ATMOSPHERE_CPU),arm7tdmi)
-export ATMOSPHERE_ARCH_DIR   := arch/arm
-export ATMOSPHERE_BOARD_DIR  := board/nintendo/nx_bpmp
-export ATMOSPHERE_OS_DIR     := os/horizon
+export ATMOSPHERE_ARCH_DIR   := arm
+export ATMOSPHERE_BOARD_DIR  := nintendo/nx_bpmp
+export ATMOSPHERE_OS_DIR     := horizon
 
 export ATMOSPHERE_ARCH_NAME  := arm
 export ATMOSPHERE_BOARD_NAME := nintendo_nx
 export ATMOSPHERE_OS_NAME    := horizon
+
+export ATMOSPHERE_CPU_EXTENSIONS :=
 endif
 
 endif
 
 ifeq ($(ATMOSPHERE_CPU),arm-cortex-a57)
-export ATMOSPHERE_CPU_DIR    := arch/arm64/cpu/cortex_a57
+export ATMOSPHERE_CPU_DIR    := cortex_a57
 export ATMOSPHERE_CPU_NAME   := arm_cortex_a57
 endif
 
 ifeq ($(ATMOSPHERE_CPU),arm7tdmi)
-export ATMOSPHERE_CPU_DIR    := arch/arm/cpu/arm7tdmi
+export ATMOSPHERE_CPU_DIR    := arm7tdmi
 export ATMOSPHERE_CPU_NAME   := arm7tdmi
 endif
 
 
-export ATMOSPHERE_ARCH_MAKE_DIR  := $(ATMOSPHERE_CONFIG_MAKE_DIR)/$(ATMOSPHERE_ARCH_DIR)
-export ATMOSPHERE_BOARD_MAKE_DIR := $(ATMOSPHERE_CONFIG_MAKE_DIR)/$(ATMOSPHERE_BOARD_DIR)
-export ATMOSPHERE_OS_MAKE_DIR    := $(ATMOSPHERE_CONFIG_MAKE_DIR)/$(ATMOSPHERE_OS_DIR)
-export ATMOSPHERE_CPU_MAKE_DIR   := $(ATMOSPHERE_CONFIG_MAKE_DIR)/$(ATMOSPHERE_CPU_DIR)
+export ATMOSPHERE_ARCH_MAKE_DIR  := $(ATMOSPHERE_CONFIG_MAKE_DIR)/arch/$(ATMOSPHERE_ARCH_DIR)
+export ATMOSPHERE_BOARD_MAKE_DIR := $(ATMOSPHERE_CONFIG_MAKE_DIR)/board/$(ATMOSPHERE_BOARD_DIR)
+export ATMOSPHERE_OS_MAKE_DIR    := $(ATMOSPHERE_CONFIG_MAKE_DIR)/os/$(ATMOSPHERE_OS_DIR)
+export ATMOSPHERE_CPU_MAKE_DIR   := $(ATMOSPHERE_ARCH_MAKE_DIR)/cpu/$(ATMOSPHERE_CPU_DIR)
 
 export ATMOSPHERE_LIBRARY_DIR := lib_$(ATMOSPHERE_BOARD_NAME)_$(ATMOSPHERE_ARCH_NAME)
 export ATMOSPHERE_BUILD_DIR := build_$(ATMOSPHERE_BOARD_NAME)_$(ATMOSPHERE_ARCH_NAME)
@@ -102,12 +106,32 @@ BUILD        := build
 DATA         := data
 INCLUDES     := include
 
-GENERAL_SOURCE_DIRS=$1 $(foreach d,$(filter-out $1/arch $1/board $1,$(wildcard $1/*)),$(if $(wildcard $d/.),$(call DIR_WILDCARD,$d) $d,))
-SPECIFIC_SOURCE_DIRS=$(if $(wildcard $1/$2/.*),$1/$2 $(call DIR_WILDCARD,$1/$2),)
-ALL_SOURCE_DIRS=$(call GENERAL_SOURCE_DIRS,$1) $(call SPECIFIC_SOURCE_DIRS,$1,$(ATMOSPHERE_ARCH_DIR)) $(call SPECIFIC_SOURCE_DIRS,$1,$(ATMOSPHERE_BOARD_DIR)) $(call SPECIFIC_SOURCE_DIRS,$1,$(ATMOSPHERE_OS_DIR))
+GENERAL_SOURCE_DIRS=$1 $(foreach d,$(filter-out $1/arch $1/board $1/os $1/cpu $1,$(wildcard $1/*)),$(if $(wildcard $d/.),$(call DIR_WILDCARD,$d) $d,))
+SPECIFIC_SOURCE_DIRS=$(if $(wildcard $1/$2/$3/.*),$1/$2/$3 $(call DIR_WILDCARD,$1/$2/$3),$(if $(wildcard $1/$2/generic/.*), $1/$2/generic $(call DIR_WILDCARD,$1/$2/generic),))
+UNFILTERED_SOURCE_DIRS=$1 $(foreach d,$(wildcard $1/*),$(if $(wildcard $d/.),$(call DIR_WILDCARD,$d) $d,))
+
+ALL_SOURCE_DIRS=$(call GENERAL_SOURCE_DIRS,$1) \
+                $(call SPECIFIC_SOURCE_DIRS,$1,arch,$(ATMOSPHERE_ARCH_DIR)) \
+                $(call SPECIFIC_SOURCE_DIRS,$1,board,$(ATMOSPHERE_BOARD_DIR)) \
+                $(call SPECIFIC_SOURCE_DIRS,$1,os,$(ATMOSPHERE_OS_DIR)) \
+                $(call SPECIFIC_SOURCE_DIRS,$1,cpu,$(ATMOSPHERE_ARCH_DIR)/$(ATMOSPHERE_CPU_DIR))
 
 SOURCES      ?= $(call ALL_SOURCE_DIRS,source)
 
+FIND_SPECIFIC_SOURCE_FILES= $(notdir $(wildcard $1/*.$2.$3.$4)) $(filter-out $(subst .$2.$3.,.$2.generic.,$(notdir $(wildcard $1/*.$2.$3.$4))),$(notdir $(wildcard $1/*.$2.generic.$4)))
+
+FIND_SPECIFIC_SOURCE_FILES_EX=$(foreach ext,$3,$(notdir $(wildcard $1/*.$2.$(ext).$4))) $(filter-out $(foreach ext,$3,$(subst .$2.$(ext).,.$2.generic.,$(notdir $(wildcard $1/*.$2.$(ext).$4)))),$(notdir $(wildcard $1/*.$2.generic.$4)))
+
+FIND_SOURCE_FILES=$(foreach dir,$1,$(filter-out $(notdir $(wildcard $(dir)/*.arch.*.$2)) \
+                                                    $(notdir $(wildcard $(dir)/*.board.*.$2)) \
+                                                    $(notdir $(wildcard $(dir)/*.os.*.$2)) \
+                                                    $(notdir $(wildcard $(dir)/.cpu.*.$2)), \
+                                                $(notdir $(wildcard $(dir)/*.$2)))) \
+                  $(foreach dir,$1,$(call FIND_SPECIFIC_SOURCE_FILES,$(dir),arch,$(ATMOSPHERE_ARCH_NAME),$2)) \
+                  $(foreach dir,$1,$(call FIND_SPECIFIC_SOURCE_FILES,$(dir),board,$(ATMOSPHERE_BOARD_NAME),$2)) \
+                  $(foreach dir,$1,$(call FIND_SPECIFIC_SOURCE_FILES,$(dir),os,$(ATMOSPHERE_OS_NAME),$2)) \
+                  $(foreach dir,$1,$(call FIND_SPECIFIC_SOURCE_FILES_EX,$(dir),cpu,$(ATMOSPHERE_CPU_NAME) $(ATMOSPHERE_CPU_EXTENSIONS),$2))
+
 #---------------------------------------------------------------------------------
 # Rules for compiling pre-compiled headers
 #---------------------------------------------------------------------------------

libraries/config/templates/exosphere.mk

@@ -8,7 +8,7 @@ include  $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/../common.mk
 #---------------------------------------------------------------------------------
 ifeq ($(strip $(ATMOSPHERE_ARCH_NAME)),arm64)
 DEFINES     := $(ATMOSPHERE_DEFINES) -DATMOSPHERE_IS_EXOSPHERE
-SETTINGS    := $(ATMOSPHERE_SETTINGS) -mgeneral-regs-only -ffixed-x18 -Os -Werror -fno-non-call-exceptions
+SETTINGS    := $(ATMOSPHERE_SETTINGS) -mgeneral-regs-only -ffixed-x18 -Os -Wextra -Werror -fno-non-call-exceptions
 CFLAGS      := $(ATMOSPHERE_CFLAGS) $(SETTINGS) $(DEFINES) $(INCLUDE)
 CXXFLAGS    := $(CFLAGS) $(ATMOSPHERE_CXXFLAGS) -fno-use-cxa-atexit
 ASFLAGS     := $(ATMOSPHERE_ASFLAGS) $(SETTINGS)

libraries/config/templates/mesosphere.mk

@@ -7,10 +7,10 @@ include  $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/../common.mk
 # options for code generation
 #---------------------------------------------------------------------------------
 export DEFINES     := $(ATMOSPHERE_DEFINES) -DATMOSPHERE_IS_MESOSPHERE
-export SETTINGS    := $(ATMOSPHERE_SETTINGS) -O2 -mgeneral-regs-only -ffixed-x18 -Werror -fno-non-call-exceptions
+export SETTINGS    := $(ATMOSPHERE_SETTINGS) -O2 -mgeneral-regs-only -ffixed-x18 -Wextra -Werror -fno-non-call-exceptions
 export CFLAGS      := $(ATMOSPHERE_CFLAGS) $(SETTINGS) $(DEFINES) $(INCLUDE)
 export CXXFLAGS    := $(CFLAGS) $(ATMOSPHERE_CXXFLAGS) -fno-use-cxa-atexit
-export ASFLAGS     := $(ATMOSPHERE_ASFLAGS) $(SETTINGS) $(DEFINES)
+export ASFLAGS     := $(ATMOSPHERE_ASFLAGS) $(SETTINGS) $(DEFINES) $(INCLUDE)
 
 export LDFLAGS	=	-specs=$(TOPDIR)/$(notdir $(TOPDIR)).specs -fno-asynchronous-unwind-tables -fno-unwind-tables -nostdlib -nostartfiles -g $(SETTINGS) -Wl,-Map,$(notdir $*.map) -Wl,-z,relro,-z,now
 

libraries/libexosphere/arm.mk

@@ -14,7 +14,7 @@ include  $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/../config/common.mk
 #---------------------------------------------------------------------------------
 
 DEFINES     := $(ATMOSPHERE_DEFINES) -DATMOSPHERE_IS_EXOSPHERE
-SETTINGS    := $(ATMOSPHERE_SETTINGS) -Os -Werror -flto -fno-non-call-exceptions
+SETTINGS    := $(ATMOSPHERE_SETTINGS) -Os -Wextra -Werror -flto -fno-non-call-exceptions
 CFLAGS      := $(ATMOSPHERE_CFLAGS) $(SETTINGS) $(DEFINES) $(INCLUDE)
 CXXFLAGS    := $(CFLAGS) $(ATMOSPHERE_CXXFLAGS) -fno-use-cxa-atexit
 ASFLAGS     := $(ATMOSPHERE_ASFLAGS) $(SETTINGS)
@@ -39,23 +39,9 @@ ifneq ($(BUILD),$(notdir $(CURDIR)))
 export VPATH	:=	$(foreach dir,$(SOURCES),$(CURDIR)/$(dir)) $(CURDIR)/include \
 			$(foreach dir,$(DATA),$(CURDIR)/$(dir))
 
-CFILES      :=	$(foreach dir,$(SOURCES),$(filter-out $(notdir $(wildcard $(dir)/*.arch.*.c)) $(notdir $(wildcard $(dir)/*.board.*.c)) $(notdir $(wildcard $(dir)/*.os.*.c)), \
-                                                      $(notdir $(wildcard $(dir)/*.c))))
-CFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.arch.$(ATMOSPHERE_ARCH_NAME).c)))
-CFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.board.$(ATMOSPHERE_BOARD_NAME).c)))
-CFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.os.$(ATMOSPHERE_OS_NAME).c)))
-
-CPPFILES    :=	$(foreach dir,$(SOURCES),$(filter-out $(notdir $(wildcard $(dir)/*.arch.*.cpp)) $(notdir $(wildcard $(dir)/*.board.*.cpp)) $(notdir $(wildcard $(dir)/*.os.*.cpp)), \
-                                                      $(notdir $(wildcard $(dir)/*.cpp))))
-CPPFILES    +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.arch.$(ATMOSPHERE_ARCH_NAME).cpp)))
-CPPFILES    +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.board.$(ATMOSPHERE_BOARD_NAME).cpp)))
-CPPFILES    +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.os.$(ATMOSPHERE_OS_NAME).cpp)))
-
-SFILES      :=	$(foreach dir,$(SOURCES),$(filter-out $(notdir $(wildcard $(dir)/*.arch.*.s)) $(notdir $(wildcard $(dir)/*.board.*.s)) $(notdir $(wildcard $(dir)/*.os.*.s)), \
-                                                      $(notdir $(wildcard $(dir)/*.s))))
-SFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.arch.$(ATMOSPHERE_ARCH_NAME).s)))
-SFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.board.$(ATMOSPHERE_BOARD_NAME).s)))
-SFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.os.$(ATMOSPHERE_OS_NAME).s)))
+CFILES      :=	$(call FIND_SOURCE_FILES,$(SOURCES),c)
+CPPFILES    :=	$(call FIND_SOURCE_FILES,$(SOURCES),cpp)
+SFILES      :=	$(call FIND_SOURCE_FILES,$(SOURCES),s)
 
 #---------------------------------------------------------------------------------
 # use CXX for linking C++ projects, CC for standard C

libraries/libexosphere/arm64.mk

@@ -14,7 +14,7 @@ include  $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/../config/common.mk
 #---------------------------------------------------------------------------------
 
 DEFINES     := $(ATMOSPHERE_DEFINES) -DATMOSPHERE_IS_EXOSPHERE
-SETTINGS    := $(ATMOSPHERE_SETTINGS) -mgeneral-regs-only -ffixed-x18 -Os -Werror -fno-non-call-exceptions
+SETTINGS    := $(ATMOSPHERE_SETTINGS) -mgeneral-regs-only -ffixed-x18 -Os -Wextra -Werror -fno-non-call-exceptions
 CFLAGS      := $(ATMOSPHERE_CFLAGS) $(SETTINGS) $(DEFINES) $(INCLUDE)
 CXXFLAGS    := $(CFLAGS) $(ATMOSPHERE_CXXFLAGS) -fno-use-cxa-atexit
 ASFLAGS     := $(ATMOSPHERE_ASFLAGS) $(SETTINGS)
@@ -39,23 +39,9 @@ ifneq ($(BUILD),$(notdir $(CURDIR)))
 export VPATH	:=	$(foreach dir,$(SOURCES),$(CURDIR)/$(dir)) $(CURDIR)/include \
 			$(foreach dir,$(DATA),$(CURDIR)/$(dir))
 
-CFILES      :=	$(foreach dir,$(SOURCES),$(filter-out $(notdir $(wildcard $(dir)/*.arch.*.c)) $(notdir $(wildcard $(dir)/*.board.*.c)) $(notdir $(wildcard $(dir)/*.os.*.c)), \
-                                                      $(notdir $(wildcard $(dir)/*.c))))
-CFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.arch.$(ATMOSPHERE_ARCH_NAME).c)))
-CFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.board.$(ATMOSPHERE_BOARD_NAME).c)))
-CFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.os.$(ATMOSPHERE_OS_NAME).c)))
-
-CPPFILES    :=	$(foreach dir,$(SOURCES),$(filter-out $(notdir $(wildcard $(dir)/*.arch.*.cpp)) $(notdir $(wildcard $(dir)/*.board.*.cpp)) $(notdir $(wildcard $(dir)/*.os.*.cpp)), \
-                                                      $(notdir $(wildcard $(dir)/*.cpp))))
-CPPFILES    +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.arch.$(ATMOSPHERE_ARCH_NAME).cpp)))
-CPPFILES    +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.board.$(ATMOSPHERE_BOARD_NAME).cpp)))
-CPPFILES    +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.os.$(ATMOSPHERE_OS_NAME).cpp)))
-
-SFILES      :=	$(foreach dir,$(SOURCES),$(filter-out $(notdir $(wildcard $(dir)/*.arch.*.s)) $(notdir $(wildcard $(dir)/*.board.*.s)) $(notdir $(wildcard $(dir)/*.os.*.s)), \
-                                                      $(notdir $(wildcard $(dir)/*.s))))
-SFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.arch.$(ATMOSPHERE_ARCH_NAME).s)))
-SFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.board.$(ATMOSPHERE_BOARD_NAME).s)))
-SFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.os.$(ATMOSPHERE_OS_NAME).s)))
+CFILES      :=	$(call FIND_SOURCE_FILES,$(SOURCES),c)
+CPPFILES    :=	$(call FIND_SOURCE_FILES,$(SOURCES),cpp)
+SFILES      :=	$(call FIND_SOURCE_FILES,$(SOURCES),s)
 
 #---------------------------------------------------------------------------------
 # use CXX for linking C++ projects, CC for standard C

libraries/libexosphere/include/exosphere/secmon/secmon_monitor_context.hpp

@@ -89,7 +89,13 @@ namespace ams::secmon {
 
     constexpr inline const SecureMonitorConfiguration DefaultSecureMonitorConfiguration = {
         .target_firmware = ams::TargetFirmware_Current,
+        .key_generation  = {},
+        .hardware_type   = {},
+        .soc_type        = {},
+        .hardware_state  = {},
+        .pad_0B          = {},
         .flags           = SecureMonitorConfigurationFlag_Default,
+        .reserved        = {},
     };
 
 }

libraries/libexosphere/include/exosphere/tegra/tegra_clkrst.hpp

@@ -203,12 +203,12 @@ DEFINE_CLK_RST_REG_BIT_ENUM(RST_CPUG_CMPLX_CLR_CLR_NONCPURESET, 29, DISABLE, ENA
     HANDLER(H, MEM,           1,  0)    \
     HANDLER(H, AHBDMA,        1,  1)    \
     HANDLER(H, APBDMA,        1,  2)    \
-    HANDLER(H, USB2,          1, 26)    \
     HANDLER(H, PMC,           1,  6)    \
     HANDLER(H, FUSE,          1,  7)    \
     HANDLER(H, KFUSE,         1,  8)    \
     HANDLER(H, I2C5,          1, 15)    \
     HANDLER(H, EMC,           1, 25)    \
+    HANDLER(H, USB2,          1, 26)    \
     HANDLER(U, CSITE,         2,  9)    \
     HANDLER(U, IRAMA,         2, 20)    \
     HANDLER(U, IRAMB,         2, 21)    \

libraries/libexosphere/source/crypto/crypto_aes_impl_security_engine.cpp

@@ -33,6 +33,8 @@ namespace ams::crypto::impl {
     template<size_t KeySize>
     void AesImpl<KeySize>::Initialize(const void *key, size_t key_size, bool is_encrypt) {
         static_assert(IsSupportedKeySize(KeySize));
+        AMS_ASSERT(key_size == sizeof(int));
+        AMS_UNUSED(is_encrypt);
 
         /* Set the security engine keyslot. */
         this->slot = *static_cast<const int *>(key);
@@ -50,9 +52,11 @@ namespace ams::crypto::impl {
         } else if constexpr (KeySize == 24) {
             /* Aes 192. */
             /* TODO: se::EncryptAes192(dst, dst_size, this->slot, src, src_size); */
+            AMS_UNUSED(dst, dst_size, src, src_size);
         } else if constexpr (KeySize == 32) {
             /* Aes 256. */
             /* TODO: se::EncryptAes256(dst, dst_size, this->slot, src, src_size); */
+            AMS_UNUSED(dst, dst_size, src, src_size);
         } else {
             /* Invalid key size. */
             static_assert(!std::is_same<AesImpl<KeySize>, AesImpl<KeySize>>::value);
@@ -71,9 +75,11 @@ namespace ams::crypto::impl {
         } else if constexpr (KeySize == 24) {
             /* Aes 192. */
             /* TODO: se::DecryptAes192(dst, dst_size, this->slot, src, src_size); */
+            AMS_UNUSED(dst, dst_size, src, src_size);
         } else if constexpr (KeySize == 32) {
             /* Aes 256. */
             /* TODO: se::DecryptAes256(dst, dst_size, this->slot, src, src_size); */
+            AMS_UNUSED(dst, dst_size, src, src_size);
         } else {
             /* Invalid key size. */
             static_assert(!std::is_same<AesImpl<KeySize>, AesImpl<KeySize>>::value);

libraries/libexosphere/source/i2c/i2c_api.cpp

@@ -88,6 +88,8 @@ namespace ams::i2c {
         }
 
         bool Write(uintptr_t base_address, Port port, int address, const void *src, size_t src_size, bool unused) {
+            AMS_UNUSED(port, unused);
+
             /* Ensure we don't write too much. */
             u32 data = 0;
             if (src_size > MaxTransferSize) {
@@ -125,6 +127,8 @@ namespace ams::i2c {
         }
 
         bool Read(uintptr_t base_address, Port port, void *dst, size_t dst_size, int address, bool unused) {
+            AMS_UNUSED(port, unused);
+
             /* Ensure we don't read too much. */
             if (dst_size > MaxTransferSize) {
                 return false;

libraries/libexosphere/source/wdt/wdt_api.cpp

@@ -81,7 +81,8 @@ namespace ams::wdt {
                 /* Enable the counters. */
                 reg::Write(registers + 0x188, 0x1);
 
-                while (true) { /* ... */ }
+                /* Wait forever until the reboot takes. */
+                AMS_INFINITE_LOOP();
             }
         #endif
 

libraries/libmesosphere/Makefile

@@ -9,10 +9,10 @@ include  $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/../config/common.mk
 PRECOMPILED_HEADERS := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/include/mesosphere.hpp
 
 DEFINES     := $(ATMOSPHERE_DEFINES) -DATMOSPHERE_IS_MESOSPHERE
-SETTINGS    := $(ATMOSPHERE_SETTINGS) -O2 -mgeneral-regs-only -ffixed-x18 -Werror -fno-non-call-exceptions
+SETTINGS    := $(ATMOSPHERE_SETTINGS) -O2 -mgeneral-regs-only -ffixed-x18 -Wextra -Werror -fno-non-call-exceptions
 CFLAGS      := $(ATMOSPHERE_CFLAGS) $(SETTINGS) $(DEFINES) $(INCLUDE)
 CXXFLAGS    := $(CFLAGS) $(ATMOSPHERE_CXXFLAGS) -fno-use-cxa-atexit -flto
-ASFLAGS     := $(ATMOSPHERE_ASFLAGS) $(SETTINGS)
+ASFLAGS     := $(ATMOSPHERE_ASFLAGS) $(SETTINGS) $(DEFINES) $(INCLUDE)
 
 SOURCES     += $(call ALL_SOURCE_DIRS,../libvapours/source)
 
@@ -34,23 +34,9 @@ ifneq ($(BUILD),$(notdir $(CURDIR)))
 export VPATH	:=	$(foreach dir,$(SOURCES),$(CURDIR)/$(dir)) $(CURDIR)/include \
 			$(foreach dir,$(DATA),$(CURDIR)/$(dir))
 
-CFILES      :=	$(foreach dir,$(SOURCES),$(filter-out $(notdir $(wildcard $(dir)/*.arch.*.c)) $(notdir $(wildcard $(dir)/*.board.*.c)) $(notdir $(wildcard $(dir)/*.os.*.c)), \
-                                                      $(notdir $(wildcard $(dir)/*.c))))
-CFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.arch.$(ATMOSPHERE_ARCH_NAME).c)))
-CFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.board.$(ATMOSPHERE_BOARD_NAME).c)))
-CFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.os.$(ATMOSPHERE_OS_NAME).c)))
-
-CPPFILES    :=	$(foreach dir,$(SOURCES),$(filter-out $(notdir $(wildcard $(dir)/*.arch.*.cpp)) $(notdir $(wildcard $(dir)/*.board.*.cpp)) $(notdir $(wildcard $(dir)/*.os.*.cpp)), \
-                                                      $(notdir $(wildcard $(dir)/*.cpp))))
-CPPFILES    +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.arch.$(ATMOSPHERE_ARCH_NAME).cpp)))
-CPPFILES    +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.board.$(ATMOSPHERE_BOARD_NAME).cpp)))
-CPPFILES    +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.os.$(ATMOSPHERE_OS_NAME).cpp)))
-
-SFILES      :=	$(foreach dir,$(SOURCES),$(filter-out $(notdir $(wildcard $(dir)/*.arch.*.s)) $(notdir $(wildcard $(dir)/*.board.*.s)) $(notdir $(wildcard $(dir)/*.os.*.s)), \
-                                                      $(notdir $(wildcard $(dir)/*.s))))
-SFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.arch.$(ATMOSPHERE_ARCH_NAME).s)))
-SFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.board.$(ATMOSPHERE_BOARD_NAME).s)))
-SFILES      +=  $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.os.$(ATMOSPHERE_OS_NAME).s)))
+CFILES      :=	$(call FIND_SOURCE_FILES,$(SOURCES),c)
+CPPFILES    :=	$(call FIND_SOURCE_FILES,$(SOURCES),cpp)
+SFILES      :=	$(call FIND_SOURCE_FILES,$(SOURCES),s)
 
 #---------------------------------------------------------------------------------
 # use CXX for linking C++ projects, CC for standard C

libraries/libmesosphere/include/mesosphere.hpp

@@ -27,6 +27,9 @@
 #include <mesosphere/kern_initial_process.hpp>
 #include <mesosphere/kern_k_exception_context.hpp>
 
+/* Tracing functionality. */
+#include <mesosphere/kern_k_trace.hpp>
+
 /* Core pre-initialization includes. */
 #include <mesosphere/kern_select_cpu.hpp>
 #include <mesosphere/kern_select_system_control.hpp>

libraries/libmesosphere/include/mesosphere/arch/arm/kern_generic_interrupt_controller.hpp

@@ -0,0 +1,279 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <vapours.hpp>
+#include <mesosphere/kern_select_cpu.hpp>
+#include <mesosphere/kern_k_typed_address.hpp>
+
+namespace ams::kern::arch::arm {
+
+    struct GicDistributor {
+        u32 ctlr;
+        u32 typer;
+        u32 iidr;
+        u32 reserved_0x0c;
+        u32 statusr;
+        u32 reserved_0x14[3];
+        u32 impldef_0x20[8];
+        u32 setspi_nsr;
+        u32 reserved_0x44;
+        u32 clrspi_nsr;
+        u32 reserved_0x4c;
+        u32 setspi_sr;
+        u32 reserved_0x54;
+        u32 clrspi_sr;
+        u32 reserved_0x5c[9];
+        u32 igroupr[32];
+        u32 isenabler[32];
+        u32 icenabler[32];
+        u32 ispendr[32];
+        u32 icpendr[32];
+        u32 isactiver[32];
+        u32 icactiver[32];
+        union {
+            u8  bytes[1020];
+            u32 words[255];
+        } ipriorityr;
+        u32 _0x7fc;
+        union {
+            u8  bytes[1020];
+            u32 words[255];
+        } itargetsr;
+        u32 _0xbfc;
+        u32 icfgr[64];
+        u32 igrpmodr[32];
+        u32 _0xd80[32];
+        u32 nsacr[64];
+        u32 sgir;
+        u32 _0xf04[3];
+        u32 cpendsgir[4];
+        u32 spendsgir[4];
+        u32 reserved_0xf30[52];
+
+        static constexpr size_t SgirCpuTargetListShift = 16;
+
+        enum SgirTargetListFilter : u32 {
+            SgirTargetListFilter_CpuTargetList = (0 << 24),
+            SgirTargetListFilter_Others        = (1 << 24),
+            SgirTargetListFilter_Self          = (2 << 24),
+            SgirTargetListFilter_Reserved      = (3 << 24),
+        };
+    };
+    static_assert(util::is_pod<GicDistributor>::value);
+    static_assert(sizeof(GicDistributor) == 0x1000);
+
+    struct GicCpuInterface {
+        u32 ctlr;
+        u32 pmr;
+        u32 bpr;
+        u32 iar;
+        u32 eoir;
+        u32 rpr;
+        u32 hppir;
+        u32 abpr;
+        u32 aiar;
+        u32 aeoir;
+        u32 ahppir;
+        u32 statusr;
+        u32 reserved_30[4];
+        u32 impldef_40[36];
+        u32 apr[4];
+        u32 nsapr[4];
+        u32 reserved_f0[3];
+        u32 iidr;
+        u32 reserved_100[960];
+        u32 dir;
+        u32 _0x1004[1023];
+    };
+    static_assert(util::is_pod<GicCpuInterface>::value);
+    static_assert(sizeof(GicCpuInterface) == 0x2000);
+
+    struct KInterruptController {
+        NON_COPYABLE(KInterruptController);
+        NON_MOVEABLE(KInterruptController);
+        public:
+            static constexpr s32 NumSoftwareInterrupts = 16;
+            static constexpr s32 NumLocalInterrupts = NumSoftwareInterrupts + 16;
+            static constexpr s32 NumGlobalInterrupts = 988;
+            static constexpr s32 NumInterrupts = NumLocalInterrupts + NumGlobalInterrupts;
+            static constexpr s32 NumPriorityLevels = 4;
+        public:
+            struct LocalState {
+                u32 isenabler[NumLocalInterrupts / 32];
+                u32 ipriorityr[NumLocalInterrupts / 4];
+                u32 itargetsr[NumLocalInterrupts / 4];
+                u32 icfgr[NumLocalInterrupts / 16];
+            };
+
+            struct GlobalState {
+                u32 isenabler[NumGlobalInterrupts / 32];
+                u32 ipriorityr[NumGlobalInterrupts / 4];
+                u32 itargetsr[NumGlobalInterrupts / 4];
+                u32 icfgr[NumGlobalInterrupts / 16];
+            };
+
+            enum PriorityLevel : u8 {
+                PriorityLevel_High  = 0,
+                PriorityLevel_Low   = NumPriorityLevels - 1,
+
+                PriorityLevel_Timer     = 1,
+                PriorityLevel_Scheduler = 2,
+            };
+        private:
+            static inline u32 s_mask[cpu::NumCores];
+        private:
+            volatile GicDistributor  *gicd;
+            volatile GicCpuInterface *gicc;
+        public:
+            constexpr KInterruptController() : gicd(nullptr), gicc(nullptr) { /* ... */ }
+
+            void Initialize(s32 core_id);
+            void Finalize(s32 core_id);
+
+            void SaveCoreLocal(LocalState *state) const;
+            void SaveGlobal(GlobalState *state) const;
+            void RestoreCoreLocal(const LocalState *state) const;
+            void RestoreGlobal(const GlobalState *state) const;
+        public:
+            u32 GetIrq() const {
+                return this->gicc->iar;
+            }
+
+            static constexpr s32 ConvertRawIrq(u32 irq) {
+                return (irq == 0x3FF) ? -1 : (irq & 0x3FF);
+            }
+
+            void Enable(s32 irq) const {
+                this->gicd->isenabler[irq / BITSIZEOF(u32)] = (1u << (irq % BITSIZEOF(u32)));
+            }
+
+            void Disable(s32 irq) const {
+                this->gicd->icenabler[irq / BITSIZEOF(u32)] = (1u << (irq % BITSIZEOF(u32)));
+            }
+
+            void Clear(s32 irq) const {
+                this->gicd->icpendr[irq / BITSIZEOF(u32)] = (1u << (irq % BITSIZEOF(u32)));
+            }
+
+            void SetTarget(s32 irq, s32 core_id) const {
+                this->gicd->itargetsr.bytes[irq] = this->gicd->itargetsr.bytes[irq] | GetGicMask(core_id);
+            }
+
+            void ClearTarget(s32 irq, s32 core_id) const {
+                this->gicd->itargetsr.bytes[irq] = this->gicd->itargetsr.bytes[irq] & ~GetGicMask(core_id);
+            }
+
+            void SetPriorityLevel(s32 irq, s32 level) const {
+                MESOSPHERE_ASSERT(PriorityLevel_High <= level && level <= PriorityLevel_Low);
+                this->gicd->ipriorityr.bytes[irq] = ToGicPriorityValue(level);
+            }
+
+            s32 GetPriorityLevel(s32 irq) const {
+                return FromGicPriorityValue(this->gicd->ipriorityr.bytes[irq]);
+            }
+
+            void SetPriorityLevel(s32 level) const {
+                MESOSPHERE_ASSERT(PriorityLevel_High <= level && level <= PriorityLevel_Low);
+                this->gicc->pmr = ToGicPriorityValue(level);
+            }
+
+            void SetEdge(s32 irq) const {
+                u32 cfg = this->gicd->icfgr[irq / (BITSIZEOF(u32) / 2)];
+                cfg &= ~(0x3 << (2 * (irq % (BITSIZEOF(u32) / 2))));
+                cfg |=  (0x2 << (2 * (irq % (BITSIZEOF(u32) / 2))));
+                this->gicd->icfgr[irq / (BITSIZEOF(u32) / 2)] = cfg;
+            }
+
+            void SetLevel(s32 irq) const {
+                u32 cfg = this->gicd->icfgr[irq / (BITSIZEOF(u32) / 2)];
+                cfg &= ~(0x3 << (2 * (irq % (BITSIZEOF(u32) / 2))));
+                cfg |=  (0x0 << (2 * (irq % (BITSIZEOF(u32) / 2))));
+                this->gicd->icfgr[irq / (BITSIZEOF(u32) / 2)] = cfg;
+            }
+
+            void SendInterProcessorInterrupt(s32 irq, u64 core_mask) {
+                MESOSPHERE_ASSERT(IsSoftware(irq));
+                this->gicd->sgir = GetCpuTargetListMask(irq, core_mask);
+            }
+
+            void SendInterProcessorInterrupt(s32 irq) {
+                MESOSPHERE_ASSERT(IsSoftware(irq));
+                this->gicd->sgir = GicDistributor::SgirTargetListFilter_Others | irq;
+            }
+
+            void EndOfInterrupt(u32 irq) const {
+                this->gicc->eoir = irq;
+            }
+
+            bool IsInterruptDefined(s32 irq) const {
+                const s32 num_interrupts = std::min(32 + 32 * (this->gicd->typer & 0x1F), static_cast<u32>(NumInterrupts));
+                return (0 <= irq && irq < num_interrupts);
+            }
+        public:
+            static constexpr ALWAYS_INLINE bool IsSoftware(s32 id) {
+                MESOSPHERE_ASSERT(0 <= id && id < NumInterrupts);
+                return id < NumSoftwareInterrupts;
+            }
+
+            static constexpr ALWAYS_INLINE bool IsLocal(s32 id) {
+                MESOSPHERE_ASSERT(0 <= id && id < NumInterrupts);
+                return id < NumLocalInterrupts;
+            }
+
+            static constexpr ALWAYS_INLINE bool IsGlobal(s32 id) {
+                MESOSPHERE_ASSERT(0 <= id && id < NumInterrupts);
+                return NumLocalInterrupts <= id;
+            }
+
+            static constexpr size_t GetGlobalInterruptIndex(s32 id) {
+                MESOSPHERE_ASSERT(IsGlobal(id));
+                return id - NumLocalInterrupts;
+            }
+
+            static constexpr size_t GetLocalInterruptIndex(s32 id) {
+                MESOSPHERE_ASSERT(IsLocal(id));
+                return id;
+            }
+        private:
+            static constexpr size_t PriorityShift = BITSIZEOF(u8) - __builtin_ctz(NumPriorityLevels);
+            static_assert(PriorityShift < BITSIZEOF(u8));
+
+            static constexpr ALWAYS_INLINE u8 ToGicPriorityValue(s32 level) {
+                return (level << PriorityShift) | ((1 << PriorityShift) - 1);
+            }
+
+            static constexpr ALWAYS_INLINE s32 FromGicPriorityValue(u8 priority) {
+                return (priority >> PriorityShift) & (NumPriorityLevels - 1);
+            }
+
+            static constexpr ALWAYS_INLINE s32 GetCpuTargetListMask(s32 irq, u64 core_mask) {
+                MESOSPHERE_ASSERT(IsSoftware(irq));
+                MESOSPHERE_ASSERT(core_mask < (1ul << cpu::NumCores));
+                return GicDistributor::SgirTargetListFilter_CpuTargetList | irq | (static_cast<u16>(core_mask) << GicDistributor::SgirCpuTargetListShift);
+            }
+
+            static ALWAYS_INLINE s32 GetGicMask(s32 core_id) {
+                return s_mask[core_id];
+            }
+
+            ALWAYS_INLINE void SetGicMask(s32 core_id) const {
+                s_mask[core_id] = this->gicd->itargetsr.bytes[0];
+            }
+
+            NOINLINE void SetupInterruptLines(s32 core_id) const;
+    };
+
+}

libraries/libmesosphere/include/mesosphere/arch/arm/kern_k_interrupt_controller.hpp

@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <vapours.hpp>
+#include <mesosphere/kern_select_cpu.hpp>
+#include <mesosphere/kern_k_typed_address.hpp>
+
+#if 1
+
+    #include <mesosphere/arch/arm/kern_generic_interrupt_controller.hpp>
+
+#else
+
+    #error "Unknown board for KInterruptController"
+
+#endif

libraries/libmesosphere/include/mesosphere/arch/arm/kern_k_memory_region_device_types.inc

@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/* All architectures must define NumArchitectureDeviceRegions. */
+constexpr inline const auto NumArchitectureDeviceRegions = 3;
+
+constexpr inline const auto KMemoryRegionType_Uart                  = KMemoryRegionType_ArchDeviceBase.DeriveSparse(0, NumArchitectureDeviceRegions, 0);
+constexpr inline const auto KMemoryRegionType_InterruptCpuInterface = KMemoryRegionType_ArchDeviceBase.DeriveSparse(0, NumArchitectureDeviceRegions, 1).SetAttribute(KMemoryRegionAttr_NoUserMap);
+constexpr inline const auto KMemoryRegionType_InterruptDistributor  = KMemoryRegionType_ArchDeviceBase.DeriveSparse(0, NumArchitectureDeviceRegions, 2).SetAttribute(KMemoryRegionAttr_NoUserMap);
+static_assert(KMemoryRegionType_Uart                 .GetValue() == (0x1D));
+static_assert(KMemoryRegionType_InterruptCpuInterface.GetValue() == (0x2D | KMemoryRegionAttr_NoUserMap));
+static_assert(KMemoryRegionType_InterruptDistributor .GetValue() == (0x4D | KMemoryRegionAttr_NoUserMap));

libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp

@@ -74,6 +74,10 @@ namespace ams::kern::arch::arm64::init {
             static ALWAYS_INLINE void ClearNewPageTable(KPhysicalAddress address) {
                 ClearPhysicalMemory(address, PageSize);
             }
+        public:
+            static consteval size_t GetMaximumOverheadSize(size_t size) {
+                return (util::DivideUp(size, L1BlockSize) + util::DivideUp(size, L2BlockSize)) * PageSize;
+            }
         private:
             size_t NOINLINE GetBlockCount(KVirtualAddress virt_addr, size_t size, size_t block_size) {
                 const KVirtualAddress end_virt_addr = virt_addr + size;

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu.hpp

@@ -59,11 +59,6 @@ namespace ams::kern::arch::arm64::cpu {
         InstructionMemoryBarrier();
     }
 
-    ALWAYS_INLINE void InvalidateEntireInstructionCache() {
-        __asm__ __volatile__("ic iallu" ::: "memory");
-        EnsureInstructionConsistency();
-    }
-
     ALWAYS_INLINE void Yield() {
         __asm__ __volatile__("yield" ::: "memory");
     }
@@ -179,6 +174,7 @@ namespace ams::kern::arch::arm64::cpu {
     void ClearPageToZeroImpl(void *);
     void FlushEntireDataCacheSharedForInit();
     void FlushEntireDataCacheLocalForInit();
+    void InvalidateEntireInstructionCacheForInit();
     void StoreEntireCacheForInit();
 
     void FlushEntireDataCache();
@@ -188,6 +184,8 @@ namespace ams::kern::arch::arm64::cpu {
     Result FlushDataCache(const void *addr, size_t size);
     Result InvalidateInstructionCache(void *addr, size_t size);
 
+    void InvalidateEntireInstructionCache();
+
     ALWAYS_INLINE void ClearPageToZero(void *page) {
         MESOSPHERE_ASSERT(util::IsAligned(reinterpret_cast<uintptr_t>(page), PageSize));
         MESOSPHERE_ASSERT(page != nullptr);

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_debug.hpp

@@ -33,7 +33,7 @@ namespace ams::kern::arch::arm64 {
             explicit KDebug() { /* ... */ }
             virtual ~KDebug() { /* ... */ }
 
-            static void PostDestroy(uintptr_t arg) { /* ... */ }
+            static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
         public:
             virtual Result GetThreadContextImpl(ams::svc::ThreadContext *out, KThread *thread, u32 context_flags) override;
             virtual Result SetThreadContextImpl(const ams::svc::ThreadContext &ctx, KThread *thread, u32 context_flags) override;

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_interrupt_controller.hpp

@@ -18,261 +18,19 @@
 #include <mesosphere/kern_select_cpu.hpp>
 #include <mesosphere/kern_k_typed_address.hpp>
 
-namespace ams::kern::arch::arm64 {
+#if 1
 
-    struct GicDistributor {
-        u32 ctlr;
-        u32 typer;
-        u32 iidr;
-        u32 reserved_0x0c;
-        u32 statusr;
-        u32 reserved_0x14[3];
-        u32 impldef_0x20[8];
-        u32 setspi_nsr;
-        u32 reserved_0x44;
-        u32 clrspi_nsr;
-        u32 reserved_0x4c;
-        u32 setspi_sr;
-        u32 reserved_0x54;
-        u32 clrspi_sr;
-        u32 reserved_0x5c[9];
-        u32 igroupr[32];
-        u32 isenabler[32];
-        u32 icenabler[32];
-        u32 ispendr[32];
-        u32 icpendr[32];
-        u32 isactiver[32];
-        u32 icactiver[32];
-        union {
-            u8  bytes[1020];
-            u32 words[255];
-        } ipriorityr;
-        u32 _0x7fc;
-        union {
-            u8  bytes[1020];
-            u32 words[255];
-        } itargetsr;
-        u32 _0xbfc;
-        u32 icfgr[64];
-        u32 igrpmodr[32];
-        u32 _0xd80[32];
-        u32 nsacr[64];
-        u32 sgir;
-        u32 _0xf04[3];
-        u32 cpendsgir[4];
-        u32 spendsgir[4];
-        u32 reserved_0xf30[52];
+    #include <mesosphere/arch/arm/kern_generic_interrupt_controller.hpp>
+    namespace ams::kern::arch::arm64 {
 
-        static constexpr size_t SgirCpuTargetListShift = 16;
+        using ams::kern::arch::arm::GicDistributor;
+        using ams::kern::arch::arm::GicCpuInterface;
+        using ams::kern::arch::arm::KInterruptController;
 
-        enum SgirTargetListFilter : u32 {
-            SgirTargetListFilter_CpuTargetList = (0 << 24),
-            SgirTargetListFilter_Others        = (1 << 24),
-            SgirTargetListFilter_Self          = (2 << 24),
-            SgirTargetListFilter_Reserved      = (3 << 24),
-        };
-    };
-    static_assert(util::is_pod<GicDistributor>::value);
-    static_assert(sizeof(GicDistributor) == 0x1000);
+    }
 
-    struct GicCpuInterface {
-        u32 ctlr;
-        u32 pmr;
-        u32 bpr;
-        u32 iar;
-        u32 eoir;
-        u32 rpr;
-        u32 hppir;
-        u32 abpr;
-        u32 aiar;
-        u32 aeoir;
-        u32 ahppir;
-        u32 statusr;
-        u32 reserved_30[4];
-        u32 impldef_40[36];
-        u32 apr[4];
-        u32 nsapr[4];
-        u32 reserved_f0[3];
-        u32 iidr;
-        u32 reserved_100[960];
-        u32 dir;
-        u32 _0x1004[1023];
-    };
-    static_assert(util::is_pod<GicCpuInterface>::value);
-    static_assert(sizeof(GicCpuInterface) == 0x2000);
+#else
 
-    struct KInterruptController {
-        NON_COPYABLE(KInterruptController);
-        NON_MOVEABLE(KInterruptController);
-        public:
-            static constexpr s32 NumSoftwareInterrupts = 16;
-            static constexpr s32 NumLocalInterrupts = NumSoftwareInterrupts + 16;
-            static constexpr s32 NumGlobalInterrupts = 988;
-            static constexpr s32 NumInterrupts = NumLocalInterrupts + NumGlobalInterrupts;
-            static constexpr s32 NumPriorityLevels = 4;
-        public:
-            struct LocalState {
-                u32 isenabler[NumLocalInterrupts / 32];
-                u32 ipriorityr[NumLocalInterrupts / 4];
-                u32 itargetsr[NumLocalInterrupts / 4];
-                u32 icfgr[NumLocalInterrupts / 16];
-            };
+    #error "Unknown board for KInterruptController"
 
-            struct GlobalState {
-                u32 isenabler[NumGlobalInterrupts / 32];
-                u32 ipriorityr[NumGlobalInterrupts / 4];
-                u32 itargetsr[NumGlobalInterrupts / 4];
-                u32 icfgr[NumGlobalInterrupts / 16];
-            };
-
-            enum PriorityLevel : u8 {
-                PriorityLevel_High  = 0,
-                PriorityLevel_Low   = NumPriorityLevels - 1,
-
-                PriorityLevel_Timer     = 1,
-                PriorityLevel_Scheduler = 2,
-            };
-        private:
-            static inline u32 s_mask[cpu::NumCores];
-        private:
-            volatile GicDistributor  *gicd;
-            volatile GicCpuInterface *gicc;
-        public:
-            constexpr KInterruptController() : gicd(nullptr), gicc(nullptr) { /* ... */ }
-
-            void Initialize(s32 core_id);
-            void Finalize(s32 core_id);
-
-            void SaveCoreLocal(LocalState *state) const;
-            void SaveGlobal(GlobalState *state) const;
-            void RestoreCoreLocal(const LocalState *state) const;
-            void RestoreGlobal(const GlobalState *state) const;
-        public:
-            u32 GetIrq() const {
-                return this->gicc->iar;
-            }
-
-            static constexpr s32 ConvertRawIrq(u32 irq) {
-                return (irq == 0x3FF) ? -1 : (irq & 0x3FF);
-            }
-
-            void Enable(s32 irq) const {
-                this->gicd->isenabler[irq / BITSIZEOF(u32)] = (1u << (irq % BITSIZEOF(u32)));
-            }
-
-            void Disable(s32 irq) const {
-                this->gicd->icenabler[irq / BITSIZEOF(u32)] = (1u << (irq % BITSIZEOF(u32)));
-            }
-
-            void Clear(s32 irq) const {
-                this->gicd->icpendr[irq / BITSIZEOF(u32)] = (1u << (irq % BITSIZEOF(u32)));
-            }
-
-            void SetTarget(s32 irq, s32 core_id) const {
-                this->gicd->itargetsr.bytes[irq] = this->gicd->itargetsr.bytes[irq] | GetGicMask(core_id);
-            }
-
-            void ClearTarget(s32 irq, s32 core_id) const {
-                this->gicd->itargetsr.bytes[irq] = this->gicd->itargetsr.bytes[irq] & ~GetGicMask(core_id);
-            }
-
-            void SetPriorityLevel(s32 irq, s32 level) const {
-                MESOSPHERE_ASSERT(PriorityLevel_High <= level && level <= PriorityLevel_Low);
-                this->gicd->ipriorityr.bytes[irq] = ToGicPriorityValue(level);
-            }
-
-            s32 GetPriorityLevel(s32 irq) const {
-                return FromGicPriorityValue(this->gicd->ipriorityr.bytes[irq]);
-            }
-
-            void SetPriorityLevel(s32 level) const {
-                MESOSPHERE_ASSERT(PriorityLevel_High <= level && level <= PriorityLevel_Low);
-                this->gicc->pmr = ToGicPriorityValue(level);
-            }
-
-            void SetEdge(s32 irq) const {
-                u32 cfg = this->gicd->icfgr[irq / (BITSIZEOF(u32) / 2)];
-                cfg &= ~(0x3 << (2 * (irq % (BITSIZEOF(u32) / 2))));
-                cfg |=  (0x2 << (2 * (irq % (BITSIZEOF(u32) / 2))));
-                this->gicd->icfgr[irq / (BITSIZEOF(u32) / 2)] = cfg;
-            }
-
-            void SetLevel(s32 irq) const {
-                u32 cfg = this->gicd->icfgr[irq / (BITSIZEOF(u32) / 2)];
-                cfg &= ~(0x3 << (2 * (irq % (BITSIZEOF(u32) / 2))));
-                cfg |=  (0x0 << (2 * (irq % (BITSIZEOF(u32) / 2))));
-                this->gicd->icfgr[irq / (BITSIZEOF(u32) / 2)] = cfg;
-            }
-
-            void SendInterProcessorInterrupt(s32 irq, u64 core_mask) {
-                MESOSPHERE_ASSERT(IsSoftware(irq));
-                this->gicd->sgir = GetCpuTargetListMask(irq, core_mask);
-            }
-
-            void SendInterProcessorInterrupt(s32 irq) {
-                MESOSPHERE_ASSERT(IsSoftware(irq));
-                this->gicd->sgir = GicDistributor::SgirTargetListFilter_Others | irq;
-            }
-
-            void EndOfInterrupt(u32 irq) const {
-                this->gicc->eoir = irq;
-            }
-
-            bool IsInterruptDefined(s32 irq) const {
-                const s32 num_interrupts = std::min(32 + 32 * (this->gicd->typer & 0x1F), static_cast<u32>(NumInterrupts));
-                return (0 <= irq && irq < num_interrupts);
-            }
-        public:
-            static constexpr ALWAYS_INLINE bool IsSoftware(s32 id) {
-                MESOSPHERE_ASSERT(0 <= id && id < NumInterrupts);
-                return id < NumSoftwareInterrupts;
-            }
-
-            static constexpr ALWAYS_INLINE bool IsLocal(s32 id) {
-                MESOSPHERE_ASSERT(0 <= id && id < NumInterrupts);
-                return id < NumLocalInterrupts;
-            }
-
-            static constexpr ALWAYS_INLINE bool IsGlobal(s32 id) {
-                MESOSPHERE_ASSERT(0 <= id && id < NumInterrupts);
-                return NumLocalInterrupts <= id;
-            }
-
-            static constexpr size_t GetGlobalInterruptIndex(s32 id) {
-                MESOSPHERE_ASSERT(IsGlobal(id));
-                return id - NumLocalInterrupts;
-            }
-
-            static constexpr size_t GetLocalInterruptIndex(s32 id) {
-                MESOSPHERE_ASSERT(IsLocal(id));
-                return id;
-            }
-        private:
-            static constexpr size_t PriorityShift = BITSIZEOF(u8) - __builtin_ctz(NumPriorityLevels);
-            static_assert(PriorityShift < BITSIZEOF(u8));
-
-            static constexpr ALWAYS_INLINE u8 ToGicPriorityValue(s32 level) {
-                return (level << PriorityShift) | ((1 << PriorityShift) - 1);
-            }
-
-            static constexpr ALWAYS_INLINE s32 FromGicPriorityValue(u8 priority) {
-                return (priority >> PriorityShift) & (NumPriorityLevels - 1);
-            }
-
-            static constexpr ALWAYS_INLINE s32 GetCpuTargetListMask(s32 irq, u64 core_mask) {
-                MESOSPHERE_ASSERT(IsSoftware(irq));
-                MESOSPHERE_ASSERT(core_mask < (1ul << cpu::NumCores));
-                return GicDistributor::SgirTargetListFilter_CpuTargetList | irq | (static_cast<u16>(core_mask) << GicDistributor::SgirCpuTargetListShift);
-            }
-
-            static ALWAYS_INLINE s32 GetGicMask(s32 core_id) {
-                return s_mask[core_id];
-            }
-
-            ALWAYS_INLINE void SetGicMask(s32 core_id) const {
-                s_mask[core_id] = this->gicd->itargetsr.bytes[0];
-            }
-
-            NOINLINE void SetupInterruptLines(s32 core_id) const;
-    };
-}
+#endif

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_memory_region_device_types.inc

@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/* All architectures must define NumArchitectureDeviceRegions. */
+constexpr inline const auto NumArchitectureDeviceRegions = 3;
+
+constexpr inline const auto KMemoryRegionType_Uart                  = KMemoryRegionType_ArchDeviceBase.DeriveSparse(0, NumArchitectureDeviceRegions, 0);
+constexpr inline const auto KMemoryRegionType_InterruptCpuInterface = KMemoryRegionType_ArchDeviceBase.DeriveSparse(0, NumArchitectureDeviceRegions, 1).SetAttribute(KMemoryRegionAttr_NoUserMap);
+constexpr inline const auto KMemoryRegionType_InterruptDistributor  = KMemoryRegionType_ArchDeviceBase.DeriveSparse(0, NumArchitectureDeviceRegions, 2).SetAttribute(KMemoryRegionAttr_NoUserMap);
+static_assert(KMemoryRegionType_Uart                 .GetValue() == (0x1D));
+static_assert(KMemoryRegionType_InterruptCpuInterface.GetValue() == (0x2D | KMemoryRegionAttr_NoUserMap));
+static_assert(KMemoryRegionType_InterruptDistributor .GetValue() == (0x4D | KMemoryRegionAttr_NoUserMap));

libraries/libmesosphere/include/mesosphere/board/generic/kern_k_device_page_table.hpp

@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_k_page_group.hpp>
+#include <mesosphere/kern_k_memory_manager.hpp>
+#include <mesosphere/kern_select_page_table.hpp>
+
+namespace ams::kern::board::generic {
+
+    using KDeviceVirtualAddress = u64;
+
+    class KDevicePageTable {
+        public:
+            constexpr KDevicePageTable() { /* ... */ }
+
+            Result ALWAYS_INLINE Initialize(u64 space_address, u64 space_size) { return ams::kern::svc::ResultNotImplemented(); }
+            void ALWAYS_INLINE Finalize() { /* ... */ }
+
+            Result ALWAYS_INLINE Attach(ams::svc::DeviceName device_name, u64 space_address, u64 space_size) { return ams::kern::svc::ResultNotImplemented(); }
+            Result ALWAYS_INLINE Detach(ams::svc::DeviceName device_name) { return ams::kern::svc::ResultNotImplemented(); }
+
+            Result ALWAYS_INLINE Map(size_t *out_mapped_size, const KPageGroup &pg, KDeviceVirtualAddress device_address, ams::svc::MemoryPermission device_perm, bool refresh_mappings) { return ams::kern::svc::ResultNotImplemented(); }
+            Result ALWAYS_INLINE Unmap(const KPageGroup &pg, KDeviceVirtualAddress device_address) { return ams::kern::svc::ResultNotImplemented(); }
+        public:
+            static ALWAYS_INLINE void Initialize() { /* ... */ }
+
+            static ALWAYS_INLINE void Lock() { /* ... */ }
+            static ALWAYS_INLINE void Unlock() { /* ... */ }
+            static ALWAYS_INLINE void Sleep() { /* ... */ }
+            static ALWAYS_INLINE void Wakeup() { /* ... */ }
+    };
+
+}

libraries/libmesosphere/include/mesosphere/board/nintendo/nx/kern_k_device_page_table.hpp

@@ -36,11 +36,13 @@ namespace ams::kern::board::nintendo::nx {
             u32 hs_detached_value;
         private:
             static ALWAYS_INLINE bool IsHeapVirtualAddress(KVirtualAddress addr) {
-                return KMemoryLayout::IsHeapVirtualAddress(nullptr, addr);
+                const KMemoryRegion *hint = nullptr;
+                return KMemoryLayout::IsHeapVirtualAddress(hint, addr);
             }
 
             static ALWAYS_INLINE bool IsHeapPhysicalAddress(KPhysicalAddress addr) {
-                return KMemoryLayout::IsHeapPhysicalAddress(nullptr, addr);
+                const KMemoryRegion *hint = nullptr;
+                return KMemoryLayout::IsHeapPhysicalAddress(hint, addr);
             }
 
             static ALWAYS_INLINE KVirtualAddress GetHeapVirtualAddress(KPhysicalAddress addr) {

libraries/libmesosphere/include/mesosphere/board/nintendo/nx/kern_k_memory_layout.hpp

@@ -18,6 +18,7 @@
 
 namespace ams::kern {
 
-    constexpr inline size_t MainMemorySize = 4_GB;
+    constexpr inline size_t MainMemorySize    = 4_GB;
+    constexpr inline size_t MainMemorySizeMax = 8_GB;
 
 }

libraries/libmesosphere/include/mesosphere/board/nintendo/nx/kern_k_memory_region_device_types.inc

@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/* All architectures must define NumBoardDeviceRegions. */
+constexpr inline const auto NumBoardDeviceRegions = 6;
+                                                                                                /* UNUSED: .Derive(NumBoardDeviceRegions, 0); */
+constexpr inline const auto KMemoryRegionType_MemoryController          = KMemoryRegionType_BoardDeviceBase.Derive(NumBoardDeviceRegions, 1).SetAttribute(KMemoryRegionAttr_NoUserMap);
+constexpr inline const auto KMemoryRegionType_MemoryController1         = KMemoryRegionType_BoardDeviceBase.Derive(NumBoardDeviceRegions, 2).SetAttribute(KMemoryRegionAttr_NoUserMap);
+constexpr inline const auto KMemoryRegionType_MemoryController0         = KMemoryRegionType_BoardDeviceBase.Derive(NumBoardDeviceRegions, 3).SetAttribute(KMemoryRegionAttr_NoUserMap);
+constexpr inline const auto KMemoryRegionType_PowerManagementController = KMemoryRegionType_BoardDeviceBase.Derive(NumBoardDeviceRegions, 4).DeriveTransition();
+constexpr inline const auto KMemoryRegionType_LegacyLpsDevices          = KMemoryRegionType_BoardDeviceBase.Derive(NumBoardDeviceRegions, 5);
+static_assert(KMemoryRegionType_MemoryController         .GetValue() == (0x55  | KMemoryRegionAttr_NoUserMap));
+static_assert(KMemoryRegionType_MemoryController1        .GetValue() == (0x65  | KMemoryRegionAttr_NoUserMap));
+static_assert(KMemoryRegionType_MemoryController0        .GetValue() == (0x95  | KMemoryRegionAttr_NoUserMap));
+static_assert(KMemoryRegionType_PowerManagementController.GetValue() == (0x1A5));
+
+static_assert(KMemoryRegionType_LegacyLpsDevices.GetValue() == 0xC5);
+
+constexpr inline const auto NumLegacyLpsDevices = 7;
+constexpr inline const auto KMemoryRegionType_LegacyLpsExceptionVectors = KMemoryRegionType_LegacyLpsDevices.Derive(NumLegacyLpsDevices, 0);
+constexpr inline const auto KMemoryRegionType_LegacyLpsIram             = KMemoryRegionType_LegacyLpsDevices.Derive(NumLegacyLpsDevices, 1);
+constexpr inline const auto KMemoryRegionType_LegacyLpsFlowController   = KMemoryRegionType_LegacyLpsDevices.Derive(NumLegacyLpsDevices, 2);
+constexpr inline const auto KMemoryRegionType_LegacyLpsPrimaryICtlr     = KMemoryRegionType_LegacyLpsDevices.Derive(NumLegacyLpsDevices, 3);
+constexpr inline const auto KMemoryRegionType_LegacyLpsSemaphore        = KMemoryRegionType_LegacyLpsDevices.Derive(NumLegacyLpsDevices, 4);
+constexpr inline const auto KMemoryRegionType_LegacyLpsAtomics          = KMemoryRegionType_LegacyLpsDevices.Derive(NumLegacyLpsDevices, 5);
+constexpr inline const auto KMemoryRegionType_LegacyLpsClkRst           = KMemoryRegionType_LegacyLpsDevices.Derive(NumLegacyLpsDevices, 6);
+static_assert(KMemoryRegionType_LegacyLpsExceptionVectors.GetValue() == 0x3C5);
+static_assert(KMemoryRegionType_LegacyLpsIram            .GetValue() == 0x5C5);
+static_assert(KMemoryRegionType_LegacyLpsFlowController  .GetValue() == 0x6C5);
+static_assert(KMemoryRegionType_LegacyLpsPrimaryICtlr    .GetValue() == 0x9C5);
+static_assert(KMemoryRegionType_LegacyLpsSemaphore       .GetValue() == 0xAC5);
+static_assert(KMemoryRegionType_LegacyLpsAtomics         .GetValue() == 0xCC5);
+static_assert(KMemoryRegionType_LegacyLpsClkRst          .GetValue() == 0x11C5);

libraries/libmesosphere/include/mesosphere/board/nintendo/nx/kern_k_system_control.hpp

@@ -39,7 +39,7 @@ namespace ams::kern::board::nintendo::nx {
             /* Initialization. */
             static NOINLINE void InitializePhase1();
             static NOINLINE void InitializePhase2();
-            static NOINLINE u32 GetInitialProcessBinaryPool();
+            static NOINLINE u32 GetCreateProcessMemoryPool();
 
             /* Randomness. */
             static void GenerateRandomBytes(void *dst, size_t size);

libraries/libmesosphere/include/mesosphere/init/kern_init_arguments_select.hpp

@@ -26,6 +26,5 @@ namespace ams::kern::init {
 
     KPhysicalAddress GetInitArgumentsAddress(s32 core_id);
     void SetInitArguments(s32 core_id, KPhysicalAddress address, uintptr_t arg);
-    void StoreInitArguments();
 
 }

libraries/libmesosphere/include/mesosphere/kern_build_config.hpp

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+
+#if 1 || defined(AMS_BUILD_FOR_AUDITING)
+#define MESOSPHERE_BUILD_FOR_AUDITING
+#endif
+
+#if defined(MESOSPHERE_BUILD_FOR_AUDITING) || defined(AMS_BUILD_FOR_DEBUGGING)
+#define MESOSPHERE_BUILD_FOR_DEBUGGING
+#endif
+
+#ifdef  MESOSPHERE_BUILD_FOR_DEBUGGING
+#define MESOSPHERE_ENABLE_ASSERTIONS
+#define MESOSPHERE_ENABLE_DEBUG_PRINT
+#endif
+
+#define MESOSPHERE_BUILD_FOR_TRACING

libraries/libmesosphere/include/mesosphere/kern_common.hpp

@@ -15,26 +15,19 @@
  */
 #pragma once
 #include <vapours.hpp>
+#include <mesosphere/kern_build_config.hpp>
+#include <mesosphere/svc/kern_svc_results.hpp>
 
 namespace ams::kern {
 
     constexpr size_t PageSize = 4_KB;
 
+#ifdef ATMOSPHERE_BOARD_NINTENDO_NX
     ams::TargetFirmware GetTargetFirmware();
+#else
+    consteval ALWAYS_INLINE ams::TargetFirmware GetTargetFirmware() {
+        return ams::TargetFirmware_Current;
+    }
+#endif
 
 }
-
-#if 1 || defined(AMS_BUILD_FOR_AUDITING)
-#define MESOSPHERE_BUILD_FOR_AUDITING
-#endif
-
-#if defined(MESOSPHERE_BUILD_FOR_AUDITING) || defined(AMS_BUILD_FOR_DEBUGGING)
-#define MESOSPHERE_BUILD_FOR_DEBUGGING
-#endif
-
-#ifdef  MESOSPHERE_BUILD_FOR_DEBUGGING
-#define MESOSPHERE_ENABLE_ASSERTIONS
-#define MESOSPHERE_ENABLE_DEBUG_PRINT
-#endif
-
-#include <mesosphere/svc/kern_svc_results.hpp>

libraries/libmesosphere/include/mesosphere/kern_initial_process.hpp

@@ -20,7 +20,7 @@
 namespace ams::kern {
 
     constexpr u32 InitialProcessBinaryMagic      = util::FourCC<'I','N','I','1'>::Code;
-    constexpr size_t InitialProcessBinarySizeMax = 0xC00000;
+    constexpr size_t InitialProcessBinarySizeMax = 12_MB;
 
     struct InitialProcessBinaryHeader {
         u32 magic;
@@ -34,5 +34,6 @@ namespace ams::kern {
 
     u64 GetInitialProcessIdMin();
     u64 GetInitialProcessIdMax();
+    size_t GetInitialProcessesSecureMemorySize();
 
 }

libraries/libmesosphere/include/mesosphere/kern_k_capabilities.hpp

@@ -354,6 +354,10 @@ namespace ams::kern {
             constexpr bool CanForceDebug() const {
                 return this->debug_capabilities.Get<DebugFlags::ForceDebug>();
             }
+
+            constexpr u32 GetIntendedKernelMajorVersion() const { return this->intended_kernel_version.Get<KernelVersion::MajorVersion>(); }
+            constexpr u32 GetIntendedKernelMinorVersion() const { return this->intended_kernel_version.Get<KernelVersion::MinorVersion>(); }
+            constexpr u32 GetIntendedKernelVersion() const { return ams::svc::EncodeKernelVersion(this->GetIntendedKernelMajorVersion(), this->GetIntendedKernelMinorVersion()); }
     };
 
 }

libraries/libmesosphere/include/mesosphere/kern_k_client_session.hpp

@@ -35,7 +35,7 @@ namespace ams::kern {
             }
 
             virtual void Destroy() override;
-            static void PostDestroy(uintptr_t arg) { /* ... */ }
+            static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
 
             constexpr KSession *GetParent() const { return this->parent; }
 

libraries/libmesosphere/include/mesosphere/kern_k_code_memory.hpp

@@ -46,7 +46,7 @@ namespace ams::kern {
             Result UnmapFromOwner(KProcessAddress address, size_t size);
 
             virtual bool IsInitialized() const override { return this->is_initialized; }
-            static void PostDestroy(uintptr_t arg) { /* ... */ }
+            static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
 
             KProcess *GetOwner() const { return this->owner; }
             KProcessAddress GetSourceAddress() { return this->address; }

libraries/libmesosphere/include/mesosphere/kern_k_device_address_space.hpp

@@ -37,7 +37,7 @@ namespace ams::kern {
             virtual void Finalize() override;
 
             virtual bool IsInitialized() const override { return this->is_initialized; }
-            static void PostDestroy(uintptr_t arg) { /* ... */ }
+            static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
 
             Result Attach(ams::svc::DeviceName device_name);
             Result Detach(ams::svc::DeviceName device_name);

libraries/libmesosphere/include/mesosphere/kern_k_dynamic_page_manager.hpp

@@ -46,9 +46,9 @@ namespace ams::kern {
                 /* We need to have positive size. */
                 R_UNLESS(sz > 0, svc::ResultOutOfMemory());
 
-                /* Calculate metadata overhead. */
-                const size_t metadata_size    = KPageBitmap::CalculateMetadataOverheadSize(sz / sizeof(PageBuffer));
-                const size_t allocatable_size = sz - metadata_size;
+                /* Calculate management overhead. */
+                const size_t management_size  = KPageBitmap::CalculateManagementOverheadSize(sz / sizeof(PageBuffer));
+                const size_t allocatable_size = sz - management_size;
 
                 /* Set tracking fields. */
                 this->address = memory;
@@ -56,12 +56,12 @@ namespace ams::kern {
                 this->count   = allocatable_size / sizeof(PageBuffer);
                 R_UNLESS(this->count > 0, svc::ResultOutOfMemory());
 
-                /* Clear the metadata region. */
-                u64 *metadata_ptr = GetPointer<u64>(this->address + allocatable_size);
-                std::memset(metadata_ptr, 0, metadata_size);
+                /* Clear the management region. */
+                u64 *management_ptr = GetPointer<u64>(this->address + allocatable_size);
+                std::memset(management_ptr, 0, management_size);
 
                 /* Initialize the bitmap. */
-                this->page_bitmap.Initialize(metadata_ptr, this->count);
+                this->page_bitmap.Initialize(management_ptr, this->count);
 
                 /* Free the pages to the bitmap. */
                 std::memset(GetPointer<PageBuffer>(this->address), 0, this->count * sizeof(PageBuffer));

libraries/libmesosphere/include/mesosphere/kern_k_interrupt_event.hpp

@@ -41,7 +41,7 @@ namespace ams::kern {
 
             virtual bool IsInitialized() const override { return this->is_initialized; }
 
-            static void PostDestroy(uintptr_t arg) { /* ... */ }
+            static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
 
             constexpr s32 GetInterruptId() const { return this->interrupt_id; }
     };

libraries/libmesosphere/include/mesosphere/kern_k_light_client_session.hpp

@@ -35,7 +35,7 @@ namespace ams::kern {
             }
 
             virtual void Destroy() override;
-            static void PostDestroy(uintptr_t arg) { /* ... */ }
+            static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
 
             constexpr const KLightSession *GetParent() const { return this->parent; }
 

libraries/libmesosphere/include/mesosphere/kern_k_light_server_session.hpp

@@ -41,7 +41,7 @@ namespace ams::kern {
             }
 
             virtual void Destroy() override;
-            static void PostDestroy(uintptr_t arg) { /* ... */ }
+            static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
 
             constexpr const KLightSession *GetParent() const { return this->parent; }
 

libraries/libmesosphere/include/mesosphere/kern_k_memory_block.hpp

@@ -197,6 +197,7 @@ namespace ams::kern {
                 .perm             = static_cast<ams::svc::MemoryPermission>(this->perm & KMemoryPermission_UserMask),
                 .ipc_refcount     = this->ipc_lock_count,
                 .device_refcount  = this->device_use_count,
+                .padding          = {},
             };
         }
 
@@ -396,6 +397,9 @@ namespace ams::kern {
             }
 
             constexpr void ShareToDevice(KMemoryPermission new_perm) {
+                /* New permission isn't used. */
+                MESOSPHERE_UNUSED(new_perm);
+
                 /* We must either be shared or have a zero lock count. */
                 MESOSPHERE_ASSERT((this->attribute & KMemoryAttribute_DeviceShared) == KMemoryAttribute_DeviceShared || this->device_use_count == 0);
 
@@ -407,6 +411,9 @@ namespace ams::kern {
             }
 
             constexpr void UnshareToDevice(KMemoryPermission new_perm) {
+                /* New permission isn't used. */
+                MESOSPHERE_UNUSED(new_perm);
+
                 /* We must be shared. */
                 MESOSPHERE_ASSERT((this->attribute & KMemoryAttribute_DeviceShared) == KMemoryAttribute_DeviceShared);
 
@@ -439,6 +446,9 @@ namespace ams::kern {
             }
 
             constexpr void UnlockForIpc(KMemoryPermission new_perm) {
+                /* New permission isn't used. */
+                MESOSPHERE_UNUSED(new_perm);
+
                 /* We must be locked. */
                 MESOSPHERE_ASSERT((this->attribute & KMemoryAttribute_IpcLocked) == KMemoryAttribute_IpcLocked);
 

libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp

@@ -16,9 +16,10 @@
 #pragma once
 #include <mesosphere/kern_common.hpp>
 #include <mesosphere/init/kern_init_page_table_select.hpp>
+#include <mesosphere/kern_k_memory_region.hpp>
 
 #if defined(ATMOSPHERE_BOARD_NINTENDO_NX)
-    #include <mesosphere/board/nintendo/nx/kern_k_memory_layout.board.nintendo_nx.hpp>
+    #include <mesosphere/board/nintendo/nx/kern_k_memory_layout.hpp>
 #else
     #error "Unknown board for KMemoryLayout"
 #endif
@@ -39,657 +40,136 @@ namespace ams::kern {
     constexpr size_t KernelPhysicalAddressSpaceLast = KernelPhysicalAddressSpaceEnd - 1ul;
     constexpr size_t KernelPhysicalAddressSpaceSize = KernelPhysicalAddressSpaceEnd - KernelPhysicalAddressSpaceBase;
 
-    enum KMemoryRegionType : u32 {
-        KMemoryRegionAttr_CarveoutProtected = 0x04000000,
-        KMemoryRegionAttr_DidKernelMap      = 0x08000000,
-        KMemoryRegionAttr_ShouldKernelMap   = 0x10000000,
-        KMemoryRegionAttr_UserReadOnly      = 0x20000000,
-        KMemoryRegionAttr_NoUserMap         = 0x40000000,
-        KMemoryRegionAttr_LinearMapped      = 0x80000000,
+    constexpr size_t KernelPageTableHeapSize    = init::KInitialPageTable::GetMaximumOverheadSize(kern::MainMemorySizeMax);
+    constexpr size_t KernelInitialPageHeapSize  = 128_KB;
 
-        KMemoryRegionType_None        = 0,
-        KMemoryRegionType_Kernel      = 1,
-        KMemoryRegionType_Dram        = 2,
-        KMemoryRegionType_CoreLocal   = 4,
+    constexpr size_t KernelSlabHeapDataSize           = 5_MB;
+    constexpr size_t KernelSlabHeapGapsSize           = 2_MB - 64_KB;
+    constexpr size_t KernelSlabHeapGapsSizeDeprecated = 2_MB;
+    constexpr size_t KernelSlabHeapSize               = KernelSlabHeapDataSize + KernelSlabHeapGapsSize;
 
-        KMemoryRegionType_VirtualKernelPtHeap       = 0x2A,
-        KMemoryRegionType_VirtualKernelTraceBuffer  = 0x4A,
-        KMemoryRegionType_VirtualKernelInitPt       = 0x19A,
+    /* NOTE: This is calculated from KThread slab counts, assuming KThread size <= 0x860. */
+    constexpr size_t KernelSlabHeapAdditionalSize     = 0x68000;
 
-        KMemoryRegionType_VirtualDramMetadataPool        = 0x29A,
-        KMemoryRegionType_VirtualDramManagedPool         = 0x31A,
-        KMemoryRegionType_VirtualDramApplicationPool     = 0x271A,
-        KMemoryRegionType_VirtualDramAppletPool          = 0x1B1A,
-        KMemoryRegionType_VirtualDramSystemPool          = 0x2B1A,
-        KMemoryRegionType_VirtualDramSystemNonSecurePool = 0x331A,
-
-        KMemoryRegionType_Uart                      = 0x1D,
-        KMemoryRegionType_InterruptDistributor      = 0x4D | KMemoryRegionAttr_NoUserMap,
-        KMemoryRegionType_InterruptCpuInterface     = 0x2D | KMemoryRegionAttr_NoUserMap,
-
-        KMemoryRegionType_MemoryController          = 0x55,
-        KMemoryRegionType_MemoryController0         = 0x95,
-        KMemoryRegionType_MemoryController1         = 0x65,
-        KMemoryRegionType_PowerManagementController = 0x1A5,
-
-        KMemoryRegionType_KernelAutoMap = KMemoryRegionType_Kernel | KMemoryRegionAttr_ShouldKernelMap,
-
-        KMemoryRegionType_KernelTemp   = 0x31,
-
-        KMemoryRegionType_KernelCode   = 0x19,
-        KMemoryRegionType_KernelStack  = 0x29,
-        KMemoryRegionType_KernelMisc   = 0x49,
-        KMemoryRegionType_KernelSlab   = 0x89,
-
-        KMemoryRegionType_KernelMiscMainStack      = 0xB49,
-        KMemoryRegionType_KernelMiscMappedDevice   = 0xD49,
-        KMemoryRegionType_KernelMiscIdleStack      = 0x1349,
-        KMemoryRegionType_KernelMiscUnknownDebug   = 0x1549,
-        KMemoryRegionType_KernelMiscExceptionStack = 0x2349,
-
-        KMemoryRegionType_DramLinearMapped  = KMemoryRegionType_Dram  | KMemoryRegionAttr_LinearMapped,
-
-        KMemoryRegionType_DramReservedEarly       = 0x16   | KMemoryRegionAttr_NoUserMap,
-        KMemoryRegionType_DramPoolPartition       = 0x26   | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_LinearMapped,
-        KMemoryRegionType_DramMetadataPool        = 0x166  | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_LinearMapped | KMemoryRegionAttr_CarveoutProtected,
-
-        KMemoryRegionType_DramNonKernel           = 0x1A6  | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_LinearMapped,
-
-        KMemoryRegionType_DramApplicationPool     = 0x7A6  | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_LinearMapped,
-        KMemoryRegionType_DramAppletPool          = 0xBA6  | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_LinearMapped,
-        KMemoryRegionType_DramSystemNonSecurePool = 0xDA6  | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_LinearMapped,
-        KMemoryRegionType_DramSystemPool          = 0x13A6 | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_LinearMapped | KMemoryRegionAttr_CarveoutProtected,
-
-
-
-        KMemoryRegionType_DramKernel        = 0xE   | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_CarveoutProtected,
-        KMemoryRegionType_DramKernelCode    = 0xCE  | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_CarveoutProtected,
-        KMemoryRegionType_DramKernelSlab    = 0x14E | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_CarveoutProtected,
-        KMemoryRegionType_DramKernelPtHeap  = 0x24E | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_CarveoutProtected | KMemoryRegionAttr_LinearMapped,
-        KMemoryRegionType_DramKernelInitPt  = 0x44E | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_CarveoutProtected | KMemoryRegionAttr_LinearMapped,
-
-        /* These regions aren't normally mapped in retail kernel. */
-        KMemoryRegionType_KernelTraceBuffer = 0xA6  | KMemoryRegionAttr_UserReadOnly | KMemoryRegionAttr_LinearMapped,
-        KMemoryRegionType_OnMemoryBootImage = 0x156,
-        KMemoryRegionType_DTB               = 0x256,
-    };
-
-    constexpr ALWAYS_INLINE KMemoryRegionType GetTypeForVirtualLinearMapping(u32 type_id) {
-        if (type_id == (type_id | KMemoryRegionType_KernelTraceBuffer)) {
-            return KMemoryRegionType_VirtualKernelTraceBuffer;
-        } else if (type_id == (type_id | KMemoryRegionType_DramKernelPtHeap)) {
-            return KMemoryRegionType_VirtualKernelPtHeap;
-        } else {
-            return KMemoryRegionType_Dram;
-        }
-    }
-
-    class KMemoryRegion : public util::IntrusiveRedBlackTreeBaseNode<KMemoryRegion> {
-        NON_COPYABLE(KMemoryRegion);
-        NON_MOVEABLE(KMemoryRegion);
-        private:
-            uintptr_t address;
-            uintptr_t pair_address;
-            size_t region_size;
-            u32 attributes;
-            u32 type_id;
-        public:
-            static constexpr ALWAYS_INLINE int Compare(const KMemoryRegion &lhs, const KMemoryRegion &rhs) {
-                if (lhs.GetAddress() < rhs.GetAddress()) {
-                    return -1;
-                } else if (lhs.GetAddress() <= rhs.GetLastAddress()) {
-                    return 0;
-                } else {
-                    return 1;
-                }
-            }
-        public:
-            constexpr ALWAYS_INLINE KMemoryRegion() : address(0), pair_address(0), region_size(0), attributes(0), type_id(0) { /* ... */ }
-            constexpr ALWAYS_INLINE KMemoryRegion(uintptr_t a, size_t rs, uintptr_t p, u32 r, u32 t) :
-                address(a), pair_address(p), region_size(rs), attributes(r), type_id(t)
-            {
-                /* ... */
-            }
-            constexpr ALWAYS_INLINE KMemoryRegion(uintptr_t a, size_t rs, u32 r, u32 t) : KMemoryRegion(a, rs, std::numeric_limits<uintptr_t>::max(), r, t) { /* ... */ }
-
-            constexpr ALWAYS_INLINE uintptr_t GetAddress() const {
-                return this->address;
-            }
-
-            constexpr ALWAYS_INLINE uintptr_t GetPairAddress() const {
-                return this->pair_address;
-            }
-
-            constexpr ALWAYS_INLINE size_t GetSize() const {
-                return this->region_size;
-            }
-
-            constexpr ALWAYS_INLINE uintptr_t GetEndAddress() const {
-                return this->GetAddress() + this->GetSize();
-            }
-
-            constexpr ALWAYS_INLINE uintptr_t GetLastAddress() const {
-                return this->GetEndAddress() - 1;
-            }
-
-            constexpr ALWAYS_INLINE u32 GetAttributes() const {
-                return this->attributes;
-            }
-
-            constexpr ALWAYS_INLINE u32 GetType() const {
-                return this->type_id;
-            }
-
-            constexpr ALWAYS_INLINE void SetType(u32 type) {
-                MESOSPHERE_INIT_ABORT_UNLESS(this->CanDerive(type));
-                this->type_id = type;
-            }
-
-            constexpr ALWAYS_INLINE bool Contains(uintptr_t address) const {
-                return this->GetAddress() <= address && address <= this->GetLastAddress();
-            }
-
-            constexpr ALWAYS_INLINE bool IsDerivedFrom(u32 type) const {
-                return (this->GetType() | type) == this->GetType();
-            }
-
-            constexpr ALWAYS_INLINE bool HasTypeAttribute(KMemoryRegionType attr) const {
-                return (this->GetType() | attr) == this->GetType();
-            }
-
-            constexpr ALWAYS_INLINE bool CanDerive(u32 type) const {
-                return (this->GetType() | type) == type;
-            }
-
-            constexpr ALWAYS_INLINE void SetPairAddress(uintptr_t a) {
-                this->pair_address = a;
-            }
-
-            constexpr ALWAYS_INLINE void SetTypeAttribute(KMemoryRegionType attr) {
-                this->type_id |= attr;
-            }
-    };
-    static_assert(std::is_trivially_destructible<KMemoryRegion>::value);
-
-    class KMemoryRegionTree {
-        public:
-            struct DerivedRegionExtents {
-                const KMemoryRegion *first_region;
-                const KMemoryRegion *last_region;
-
-                constexpr DerivedRegionExtents() : first_region(nullptr), last_region(nullptr) { /* ... */ }
-
-                constexpr ALWAYS_INLINE uintptr_t GetAddress() const {
-                    return this->first_region->GetAddress();
-                }
-
-                constexpr ALWAYS_INLINE uintptr_t GetEndAddress() const {
-                    return this->last_region->GetEndAddress();
-                }
-
-                constexpr ALWAYS_INLINE size_t GetSize() const {
-                    return this->GetEndAddress() - this->GetAddress();
-                }
-
-                constexpr ALWAYS_INLINE uintptr_t GetLastAddress() const {
-                    return this->GetEndAddress() - 1;
-                }
-            };
-        private:
-            using TreeType = util::IntrusiveRedBlackTreeBaseTraits<KMemoryRegion>::TreeType<KMemoryRegion>;
-        public:
-            using value_type        = TreeType::value_type;
-            using size_type         = TreeType::size_type;
-            using difference_type   = TreeType::difference_type;
-            using pointer           = TreeType::pointer;
-            using const_pointer     = TreeType::const_pointer;
-            using reference         = TreeType::reference;
-            using const_reference   = TreeType::const_reference;
-            using iterator          = TreeType::iterator;
-            using const_iterator    = TreeType::const_iterator;
-        private:
-            TreeType tree;
-        public:
-            constexpr ALWAYS_INLINE KMemoryRegionTree() : tree() { /* ... */ }
-        public:
-            iterator FindContainingRegion(uintptr_t address) {
-                return this->find(KMemoryRegion(address, 1, 0, 0));
-            }
-
-            iterator FindFirstRegionByTypeAttr(u32 type_id, u32 attr = 0) {
-                for (auto it = this->begin(); it != this->end(); it++) {
-                    if (it->GetType() == type_id && it->GetAttributes() == attr) {
-                        return it;
-                    }
-                }
-                MESOSPHERE_INIT_ABORT();
-            }
-
-            iterator FindFirstRegionByType(u32 type_id) {
-                for (auto it = this->begin(); it != this->end(); it++) {
-                    if (it->GetType() == type_id) {
-                        return it;
-                    }
-                }
-                MESOSPHERE_INIT_ABORT();
-            }
-
-            iterator TryFindFirstRegionByType(u32 type_id) {
-                for (auto it = this->begin(); it != this->end(); it++) {
-                    if (it->GetType() == type_id) {
-                        return it;
-                    }
-                }
-
-                return this->end();
-            }
-
-            iterator FindFirstDerivedRegion(u32 type_id) {
-                for (auto it = this->begin(); it != this->end(); it++) {
-                    if (it->IsDerivedFrom(type_id)) {
-                        return it;
-                    }
-                }
-                MESOSPHERE_INIT_ABORT();
-            }
-
-            iterator TryFindFirstDerivedRegion(u32 type_id) {
-                for (auto it = this->begin(); it != this->end(); it++) {
-                    if (it->IsDerivedFrom(type_id)) {
-                        return it;
-                    }
-                }
-
-                return this->end();
-            }
-
-            DerivedRegionExtents GetDerivedRegionExtents(u32 type_id) const {
-                DerivedRegionExtents extents;
-
-                MESOSPHERE_INIT_ABORT_UNLESS(extents.first_region == nullptr);
-                MESOSPHERE_INIT_ABORT_UNLESS(extents.last_region  == nullptr);
-
-                for (auto it = this->cbegin(); it != this->cend(); it++) {
-                    if (it->IsDerivedFrom(type_id)) {
-                        if (extents.first_region == nullptr) {
-                            extents.first_region = std::addressof(*it);
-                        }
-                        extents.last_region = std::addressof(*it);
-                    }
-                }
-
-                MESOSPHERE_INIT_ABORT_UNLESS(extents.first_region != nullptr);
-                MESOSPHERE_INIT_ABORT_UNLESS(extents.last_region  != nullptr);
-
-                return extents;
-            }
-        public:
-            NOINLINE bool Insert(uintptr_t address, size_t size, u32 type_id, u32 new_attr = 0, u32 old_attr = 0);
-            NOINLINE KVirtualAddress GetRandomAlignedRegion(size_t size, size_t alignment, u32 type_id);
-
-            ALWAYS_INLINE KVirtualAddress GetRandomAlignedRegionWithGuard(size_t size, size_t alignment, u32 type_id, size_t guard_size) {
-                return this->GetRandomAlignedRegion(size + 2 * guard_size, alignment, type_id) + guard_size;
-            }
-        public:
-            /* Iterator accessors. */
-            iterator begin() {
-                return this->tree.begin();
-            }
-
-            const_iterator begin() const {
-                return this->tree.begin();
-            }
-
-            iterator end() {
-                return this->tree.end();
-            }
-
-            const_iterator end() const {
-                return this->tree.end();
-            }
-
-            const_iterator cbegin() const {
-                return this->begin();
-            }
-
-            const_iterator cend() const {
-                return this->end();
-            }
-
-            iterator iterator_to(reference ref) {
-                return this->tree.iterator_to(ref);
-            }
-
-            const_iterator iterator_to(const_reference ref) const {
-                return this->tree.iterator_to(ref);
-            }
-
-            /* Content management. */
-            bool empty() const {
-                return this->tree.empty();
-            }
-
-            reference back() {
-                return this->tree.back();
-            }
-
-            const_reference back() const {
-                return this->tree.back();
-            }
-
-            reference front() {
-                return this->tree.front();
-            }
-
-            const_reference front() const {
-                return this->tree.front();
-            }
-
-            /* GCC over-eagerly inlines this operation. */
-            NOINLINE iterator insert(reference ref) {
-                return this->tree.insert(ref);
-            }
-
-            NOINLINE iterator erase(iterator it) {
-                return this->tree.erase(it);
-            }
-
-            iterator find(const_reference ref) const {
-                return this->tree.find(ref);
-            }
-
-            iterator nfind(const_reference ref) const {
-                return this->tree.nfind(ref);
-            }
-    };
-
-    class KMemoryRegionAllocator {
-        NON_COPYABLE(KMemoryRegionAllocator);
-        NON_MOVEABLE(KMemoryRegionAllocator);
-        public:
-            static constexpr size_t MaxMemoryRegions = 1000;
-            friend class KMemoryLayout;
-        private:
-            KMemoryRegion region_heap[MaxMemoryRegions];
-            size_t num_regions;
-        private:
-            constexpr ALWAYS_INLINE KMemoryRegionAllocator() : region_heap(), num_regions() { /* ... */ }
-        public:
-            ALWAYS_INLINE KMemoryRegion *Allocate() {
-                /* Ensure we stay within the bounds of our heap. */
-                MESOSPHERE_INIT_ABORT_UNLESS(this->num_regions < MaxMemoryRegions);
-
-                return &this->region_heap[this->num_regions++];
-            }
-
-            template<typename... Args>
-            ALWAYS_INLINE KMemoryRegion *Create(Args&&... args) {
-                KMemoryRegion *region = this->Allocate();
-                new (region) KMemoryRegion(std::forward<Args>(args)...);
-                return region;
-            }
-    };
+    constexpr size_t KernelResourceSize               = KernelPageTableHeapSize + KernelInitialPageHeapSize + KernelSlabHeapSize;
 
     class KMemoryLayout {
         private:
             static /* constinit */ inline uintptr_t s_linear_phys_to_virt_diff;
             static /* constinit */ inline uintptr_t s_linear_virt_to_phys_diff;
-            static /* constinit */ inline KMemoryRegionAllocator s_region_allocator;
             static /* constinit */ inline KMemoryRegionTree s_virtual_tree;
             static /* constinit */ inline KMemoryRegionTree s_physical_tree;
             static /* constinit */ inline KMemoryRegionTree s_virtual_linear_tree;
             static /* constinit */ inline KMemoryRegionTree s_physical_linear_tree;
         private:
-            static ALWAYS_INLINE auto GetVirtualLinearExtents(const KMemoryRegionTree::DerivedRegionExtents physical) {
-                return KMemoryRegion(GetInteger(GetLinearVirtualAddress(physical.GetAddress())), physical.GetSize(), 0, KMemoryRegionType_None);
+            template<typename AddressType> requires IsKTypedAddress<AddressType>
+            static ALWAYS_INLINE bool IsTypedAddress(const KMemoryRegion *&region, AddressType address, KMemoryRegionTree &tree, KMemoryRegionType type) {
+                /* Check if the cached region already contains the address. */
+                if (region != nullptr && region->Contains(GetInteger(address))) {
+                    return true;
+                }
+
+                /* Find the containing region, and update the cache. */
+                if (const KMemoryRegion *found = tree.Find(GetInteger(address)); found != nullptr && found->IsDerivedFrom(type)) {
+                    region = found;
+                    return true;
+                } else {
+                    return false;
+                }
+            }
+
+            template<typename AddressType> requires IsKTypedAddress<AddressType>
+            static ALWAYS_INLINE bool IsTypedAddress(const KMemoryRegion *&region, AddressType address, size_t size, KMemoryRegionTree &tree, KMemoryRegionType type) {
+                /* Get the end of the checked region. */
+                const uintptr_t last_address = GetInteger(address) + size - 1;
+
+                /* Walk the tree to verify the region is correct. */
+                const KMemoryRegion *cur = (region != nullptr && region->Contains(GetInteger(address))) ? region : tree.Find(GetInteger(address));
+                while (cur != nullptr && cur->IsDerivedFrom(type)) {
+                    if (last_address <= cur->GetLastAddress()) {
+                        region = cur;
+                        return true;
+                    }
+
+                    cur = cur->GetNext();
+                }
+                return false;
+            }
+
+            template<typename AddressType> requires IsKTypedAddress<AddressType>
+            static ALWAYS_INLINE const KMemoryRegion *Find(AddressType address, const KMemoryRegionTree &tree) {
+                return tree.Find(GetInteger(address));
+            }
+
+            static ALWAYS_INLINE KMemoryRegion &Dereference(KMemoryRegion *region) {
+                MESOSPHERE_INIT_ABORT_UNLESS(region != nullptr);
+                return *region;
+            }
+
+            static ALWAYS_INLINE const KMemoryRegion &Dereference(const KMemoryRegion *region) {
+                MESOSPHERE_INIT_ABORT_UNLESS(region != nullptr);
+                return *region;
+            }
+
+            static ALWAYS_INLINE KVirtualAddress GetStackTopAddress(s32 core_id, KMemoryRegionType type) {
+                return Dereference(GetVirtualMemoryRegionTree().FindByTypeAndAttribute(type, static_cast<u32>(core_id))).GetEndAddress();
             }
         public:
-            static ALWAYS_INLINE KMemoryRegionAllocator &GetMemoryRegionAllocator()        { return s_region_allocator; }
-            static ALWAYS_INLINE KMemoryRegionTree      &GetVirtualMemoryRegionTree()      { return s_virtual_tree; }
-            static ALWAYS_INLINE KMemoryRegionTree      &GetPhysicalMemoryRegionTree()     { return s_physical_tree; }
-            static ALWAYS_INLINE KMemoryRegionTree      &GetVirtualLinearMemoryRegionTree()  { return s_virtual_linear_tree; }
-            static ALWAYS_INLINE KMemoryRegionTree      &GetPhysicalLinearMemoryRegionTree() { return s_physical_linear_tree; }
+            static ALWAYS_INLINE KMemoryRegionTree &GetVirtualMemoryRegionTree()        { return s_virtual_tree; }
+            static ALWAYS_INLINE KMemoryRegionTree &GetPhysicalMemoryRegionTree()       { return s_physical_tree; }
+            static ALWAYS_INLINE KMemoryRegionTree &GetVirtualLinearMemoryRegionTree()  { return s_virtual_linear_tree; }
+            static ALWAYS_INLINE KMemoryRegionTree &GetPhysicalLinearMemoryRegionTree() { return s_physical_linear_tree; }
 
-            static ALWAYS_INLINE KMemoryRegionTree::iterator GetEnd(KVirtualAddress) {
-                return GetVirtualLinearMemoryRegionTree().end();
-            }
+            static ALWAYS_INLINE KVirtualAddress GetLinearVirtualAddress(KPhysicalAddress address)  { return GetInteger(address) + s_linear_phys_to_virt_diff; }
+            static ALWAYS_INLINE KPhysicalAddress GetLinearPhysicalAddress(KVirtualAddress address) { return GetInteger(address) + s_linear_virt_to_phys_diff; }
 
-            static ALWAYS_INLINE KMemoryRegionTree::iterator GetEnd(KPhysicalAddress) {
-                return GetPhysicalMemoryRegionTree().end();
-            }
+            static NOINLINE const KMemoryRegion *Find(KVirtualAddress address)  { return Find(address, GetVirtualMemoryRegionTree()); }
+            static NOINLINE const KMemoryRegion *Find(KPhysicalAddress address) { return Find(address, GetPhysicalMemoryRegionTree()); }
 
-            static NOINLINE KMemoryRegionTree::iterator FindContainingRegion(KVirtualAddress address) {
-                return GetVirtualMemoryRegionTree().FindContainingRegion(GetInteger(address));
-            }
+            static NOINLINE const KMemoryRegion *FindLinear(KVirtualAddress address)  { return Find(address, GetVirtualLinearMemoryRegionTree());  }
+            static NOINLINE const KMemoryRegion *FindLinear(KPhysicalAddress address) { return Find(address, GetPhysicalLinearMemoryRegionTree()); }
 
-            static NOINLINE KMemoryRegionTree::iterator FindContainingRegion(KPhysicalAddress address) {
-                return GetPhysicalMemoryRegionTree().FindContainingRegion(GetInteger(address));
-            }
+            static NOINLINE KVirtualAddress GetMainStackTopAddress(s32 core_id)      { return GetStackTopAddress(core_id, KMemoryRegionType_KernelMiscMainStack); }
+            static NOINLINE KVirtualAddress GetIdleStackTopAddress(s32 core_id)      { return GetStackTopAddress(core_id, KMemoryRegionType_KernelMiscIdleStack); }
+            static NOINLINE KVirtualAddress GetExceptionStackTopAddress(s32 core_id) { return GetStackTopAddress(core_id, KMemoryRegionType_KernelMiscExceptionStack); }
 
-            static ALWAYS_INLINE KVirtualAddress GetLinearVirtualAddress(KPhysicalAddress address) {
-                return GetInteger(address) + s_linear_phys_to_virt_diff;
-            }
+            static NOINLINE KVirtualAddress GetSlabRegionAddress()      { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_KernelSlab)).GetAddress(); }
+            static NOINLINE KVirtualAddress GetCoreLocalRegionAddress() { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_CoreLocalRegion)).GetAddress(); }
 
-            static ALWAYS_INLINE KPhysicalAddress GetLinearPhysicalAddress(KVirtualAddress address) {
-                return GetInteger(address) + s_linear_virt_to_phys_diff;
-            }
+            static NOINLINE const KMemoryRegion &GetDeviceRegion(KMemoryRegionType type) { return Dereference(GetPhysicalMemoryRegionTree().FindFirstDerived(type)); }
+            static KPhysicalAddress GetDevicePhysicalAddress(KMemoryRegionType type) { return GetDeviceRegion(type).GetAddress(); }
+            static KVirtualAddress  GetDeviceVirtualAddress(KMemoryRegionType type)  { return GetDeviceRegion(type).GetPairAddress(); }
 
-            static NOINLINE KVirtualAddress GetMainStackTopAddress(s32 core_id) {
-                return GetVirtualMemoryRegionTree().FindFirstRegionByTypeAttr(KMemoryRegionType_KernelMiscMainStack, static_cast<u32>(core_id))->GetEndAddress();
-            }
+            static NOINLINE const KMemoryRegion &GetPoolManagementRegion() { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_VirtualDramPoolManagement)); }
+            static NOINLINE const KMemoryRegion &GetPageTableHeapRegion()  { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_VirtualDramKernelPtHeap)); }
+            static NOINLINE const KMemoryRegion &GetKernelStackRegion()    { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_KernelStack)); }
+            static NOINLINE const KMemoryRegion &GetTempRegion()           { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_KernelTemp)); }
+            static NOINLINE const KMemoryRegion &GetCoreLocalRegion()      { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_CoreLocalRegion)); }
 
-            static NOINLINE KVirtualAddress GetIdleStackTopAddress(s32 core_id) {
-                return GetVirtualMemoryRegionTree().FindFirstRegionByTypeAttr(KMemoryRegionType_KernelMiscIdleStack, static_cast<u32>(core_id))->GetEndAddress();
-            }
+            static NOINLINE const KMemoryRegion &GetKernelTraceBufferRegion() { return Dereference(GetVirtualLinearMemoryRegionTree().FindByType(KMemoryRegionType_VirtualDramKernelTraceBuffer)); }
 
-            static NOINLINE KVirtualAddress GetExceptionStackTopAddress(s32 core_id) {
-                return GetVirtualMemoryRegionTree().FindFirstRegionByTypeAttr(KMemoryRegionType_KernelMiscExceptionStack, static_cast<u32>(core_id))->GetEndAddress();
-            }
+            static NOINLINE const KMemoryRegion &GetVirtualLinearRegion(KVirtualAddress address) { return Dereference(FindLinear(address)); }
 
-            static NOINLINE KVirtualAddress GetSlabRegionAddress() {
-                return GetVirtualMemoryRegionTree().FindFirstRegionByType(KMemoryRegionType_KernelSlab)->GetAddress();
-            }
+            static NOINLINE const KMemoryRegion *GetPhysicalKernelTraceBufferRegion() { return GetPhysicalMemoryRegionTree().FindFirstDerived(KMemoryRegionType_KernelTraceBuffer); }
+            static NOINLINE const KMemoryRegion *GetPhysicalOnMemoryBootImageRegion() { return GetPhysicalMemoryRegionTree().FindFirstDerived(KMemoryRegionType_OnMemoryBootImage); }
+            static NOINLINE const KMemoryRegion *GetPhysicalDTBRegion()               { return GetPhysicalMemoryRegionTree().FindFirstDerived(KMemoryRegionType_DTB); }
 
-            static NOINLINE KVirtualAddress GetCoreLocalRegionAddress() {
-                return GetVirtualMemoryRegionTree().FindFirstRegionByType(KMemoryRegionType_CoreLocal)->GetAddress();
-            }
+            static NOINLINE bool IsHeapPhysicalAddress(const KMemoryRegion *&region, KPhysicalAddress address) { return IsTypedAddress(region, address, GetPhysicalLinearMemoryRegionTree(), KMemoryRegionType_DramUserPool); }
+            static NOINLINE bool IsHeapVirtualAddress(const KMemoryRegion *&region, KVirtualAddress address)   { return IsTypedAddress(region, address, GetVirtualLinearMemoryRegionTree(),  KMemoryRegionType_VirtualDramUserPool); }
 
-            static NOINLINE KVirtualAddress GetInterruptDistributorAddress() {
-                return GetPhysicalMemoryRegionTree().FindFirstDerivedRegion(KMemoryRegionType_InterruptDistributor)->GetPairAddress();
-            }
+            static NOINLINE bool IsHeapPhysicalAddress(const KMemoryRegion *&region, KPhysicalAddress address, size_t size) { return IsTypedAddress(region, address, size, GetPhysicalLinearMemoryRegionTree(), KMemoryRegionType_DramUserPool); }
+            static NOINLINE bool IsHeapVirtualAddress(const KMemoryRegion *&region, KVirtualAddress address, size_t size)   { return IsTypedAddress(region, address, size, GetVirtualLinearMemoryRegionTree(),  KMemoryRegionType_VirtualDramUserPool); }
 
-            static NOINLINE KVirtualAddress GetInterruptCpuInterfaceAddress() {
-                return GetPhysicalMemoryRegionTree().FindFirstDerivedRegion(KMemoryRegionType_InterruptCpuInterface)->GetPairAddress();
-            }
-
-            static NOINLINE KVirtualAddress GetUartAddress() {
-                return GetPhysicalMemoryRegionTree().FindFirstDerivedRegion(KMemoryRegionType_Uart)->GetPairAddress();
-            }
-
-            static NOINLINE KMemoryRegion &GetMemoryControllerRegion() {
-                return *GetPhysicalMemoryRegionTree().FindFirstDerivedRegion(KMemoryRegionType_MemoryController);
-            }
-
-            static NOINLINE KMemoryRegion &GetMetadataPoolRegion() {
-                return *GetVirtualMemoryRegionTree().FindFirstRegionByType(KMemoryRegionType_VirtualDramMetadataPool);
-            }
-
-            static NOINLINE KMemoryRegion &GetPageTableHeapRegion() {
-                return *GetVirtualMemoryRegionTree().FindFirstRegionByType(KMemoryRegionType_VirtualKernelPtHeap);
-            }
-
-            static NOINLINE KMemoryRegion &GetKernelStackRegion() {
-                return *GetVirtualMemoryRegionTree().FindFirstRegionByType(KMemoryRegionType_KernelStack);
-            }
-
-            static NOINLINE KMemoryRegion &GetTempRegion() {
-                return *GetVirtualMemoryRegionTree().FindFirstRegionByType(KMemoryRegionType_KernelTemp);
-            }
-
-            static NOINLINE KMemoryRegion &GetVirtualLinearRegion(KVirtualAddress address) {
-                return *GetVirtualLinearMemoryRegionTree().FindContainingRegion(GetInteger(address));
-            }
-
-            static NOINLINE const KMemoryRegion *TryGetKernelTraceBufferRegion() {
-                auto &tree = GetPhysicalMemoryRegionTree();
-                if (KMemoryRegionTree::const_iterator it = tree.TryFindFirstDerivedRegion(KMemoryRegionType_KernelTraceBuffer); it != tree.end()) {
-                    return std::addressof(*it);
-                } else {
-                    return nullptr;
-                }
-            }
-
-            static NOINLINE const KMemoryRegion *TryGetOnMemoryBootImageRegion() {
-                auto &tree = GetPhysicalMemoryRegionTree();
-                if (KMemoryRegionTree::const_iterator it = tree.TryFindFirstDerivedRegion(KMemoryRegionType_OnMemoryBootImage); it != tree.end()) {
-                    return std::addressof(*it);
-                } else {
-                    return nullptr;
-                }
-            }
-
-            static NOINLINE const KMemoryRegion *TryGetDTBRegion() {
-                auto &tree = GetPhysicalMemoryRegionTree();
-                if (KMemoryRegionTree::const_iterator it = tree.TryFindFirstDerivedRegion(KMemoryRegionType_DTB); it != tree.end()) {
-                    return std::addressof(*it);
-                } else {
-                    return nullptr;
-                }
-            }
-
-            static NOINLINE bool IsHeapPhysicalAddress(const KMemoryRegion **out, KPhysicalAddress address, const KMemoryRegion *hint = nullptr) {
-                auto &tree = GetPhysicalLinearMemoryRegionTree();
-                KMemoryRegionTree::const_iterator it = tree.end();
-                if (hint != nullptr) {
-                    it = tree.iterator_to(*hint);
-                }
-                if (it == tree.end() || !it->Contains(GetInteger(address))) {
-                    it = tree.FindContainingRegion(GetInteger(address));
-                }
-                if (it != tree.end() && it->IsDerivedFrom(KMemoryRegionType_DramNonKernel)) {
-                    if (out) {
-                        *out = std::addressof(*it);
-                    }
-                    return true;
-                }
-                return false;
-            }
-
-            static NOINLINE bool IsHeapPhysicalAddress(const KMemoryRegion **out, KPhysicalAddress address, size_t size, const KMemoryRegion *hint = nullptr) {
-                auto &tree = GetPhysicalLinearMemoryRegionTree();
-                KMemoryRegionTree::const_iterator it = tree.end();
-                if (hint != nullptr) {
-                    it = tree.iterator_to(*hint);
-                }
-                if (it == tree.end() || !it->Contains(GetInteger(address))) {
-                    it = tree.FindContainingRegion(GetInteger(address));
-                }
-                if (it != tree.end() && it->IsDerivedFrom(KMemoryRegionType_DramNonKernel)) {
-                    const uintptr_t last_address = GetInteger(address) + size - 1;
-                    do {
-                        if (last_address <= it->GetLastAddress()) {
-                            if (out) {
-                                *out = std::addressof(*it);
-                            }
-                            return true;
-                        }
-                        it++;
-                    } while (it != tree.end() && it->IsDerivedFrom(KMemoryRegionType_DramNonKernel));
-                }
-                return false;
-            }
-
-            static NOINLINE bool IsLinearMappedPhysicalAddress(const KMemoryRegion **out, KPhysicalAddress address, const KMemoryRegion *hint = nullptr) {
-                auto &tree = GetPhysicalLinearMemoryRegionTree();
-                KMemoryRegionTree::const_iterator it = tree.end();
-                if (hint != nullptr) {
-                    it = tree.iterator_to(*hint);
-                }
-                if (it == tree.end() || !it->Contains(GetInteger(address))) {
-                    it = tree.FindContainingRegion(GetInteger(address));
-                }
-                if (it != tree.end() && it->IsDerivedFrom(KMemoryRegionAttr_LinearMapped)) {
-                    if (out) {
-                        *out = std::addressof(*it);
-                    }
-                    return true;
-                }
-                return false;
-            }
-
-            static NOINLINE bool IsLinearMappedPhysicalAddress(const KMemoryRegion **out, KPhysicalAddress address, size_t size, const KMemoryRegion *hint = nullptr) {
-                auto &tree = GetPhysicalLinearMemoryRegionTree();
-                KMemoryRegionTree::const_iterator it = tree.end();
-                if (hint != nullptr) {
-                    it = tree.iterator_to(*hint);
-                }
-                if (it == tree.end() || !it->Contains(GetInteger(address))) {
-                    it = tree.FindContainingRegion(GetInteger(address));
-                }
-                if (it != tree.end() && it->IsDerivedFrom(KMemoryRegionAttr_LinearMapped)) {
-                    const uintptr_t last_address = GetInteger(address) + size - 1;
-                    do {
-                        if (last_address <= it->GetLastAddress()) {
-                            if (out) {
-                                *out = std::addressof(*it);
-                            }
-                            return true;
-                        }
-                        it++;
-                    } while (it != tree.end() && it->IsDerivedFrom(KMemoryRegionAttr_LinearMapped));
-                }
-                return false;
-            }
-
-            static NOINLINE bool IsHeapVirtualAddress(const KMemoryRegion **out, KVirtualAddress address, const KMemoryRegion *hint = nullptr) {
-                auto &tree = GetVirtualLinearMemoryRegionTree();
-                KMemoryRegionTree::const_iterator it = tree.end();
-                if (hint != nullptr) {
-                    it = tree.iterator_to(*hint);
-                }
-                if (it == tree.end() || !it->Contains(GetInteger(address))) {
-                    it = tree.FindContainingRegion(GetInteger(address));
-                }
-                if (it != tree.end() && it->IsDerivedFrom(KMemoryRegionType_VirtualDramManagedPool)) {
-                    if (out) {
-                        *out = std::addressof(*it);
-                    }
-                    return true;
-                }
-                return false;
-            }
-
-            static NOINLINE bool IsHeapVirtualAddress(const KMemoryRegion **out, KVirtualAddress address, size_t size, const KMemoryRegion *hint = nullptr) {
-                auto &tree = GetVirtualLinearMemoryRegionTree();
-                KMemoryRegionTree::const_iterator it = tree.end();
-                if (hint != nullptr) {
-                    it = tree.iterator_to(*hint);
-                }
-                if (it == tree.end() || !it->Contains(GetInteger(address))) {
-                    it = tree.FindContainingRegion(GetInteger(address));
-                }
-                if (it != tree.end() && it->IsDerivedFrom(KMemoryRegionType_VirtualDramManagedPool)) {
-                    const uintptr_t last_address = GetInteger(address) + size - 1;
-                    do {
-                        if (last_address <= it->GetLastAddress()) {
-                            if (out) {
-                                *out = std::addressof(*it);
-                            }
-                            return true;
-                        }
-                        it++;
-                    } while (it != tree.end() && it->IsDerivedFrom(KMemoryRegionType_VirtualDramManagedPool));
-                }
-                return false;
-            }
+            static NOINLINE bool IsLinearMappedPhysicalAddress(const KMemoryRegion *&region, KPhysicalAddress address)  { return IsTypedAddress(region, address, GetPhysicalLinearMemoryRegionTree(), static_cast<KMemoryRegionType>(KMemoryRegionAttr_LinearMapped)); }
+            static NOINLINE bool IsLinearMappedPhysicalAddress(const KMemoryRegion *&region, KPhysicalAddress address, size_t size) { return IsTypedAddress(region, address, size, GetPhysicalLinearMemoryRegionTree(), static_cast<KMemoryRegionType>(KMemoryRegionAttr_LinearMapped)); }
 
             static NOINLINE std::tuple<size_t, size_t> GetTotalAndKernelMemorySizes() {
                 size_t total_size = 0, kernel_size = 0;
-                for (auto it = GetPhysicalMemoryRegionTree().cbegin(); it != GetPhysicalMemoryRegionTree().cend(); it++) {
-                    if (it->IsDerivedFrom(KMemoryRegionType_Dram)) {
-                        total_size += it->GetSize();
-                        if (!it->IsDerivedFrom(KMemoryRegionType_DramNonKernel)) {
-                            kernel_size += it->GetSize();
+                for (const auto &region : GetPhysicalMemoryRegionTree()) {
+                    if (region.IsDerivedFrom(KMemoryRegionType_Dram)) {
+                        total_size += region.GetSize();
+                        if (!region.IsDerivedFrom(KMemoryRegionType_DramUserPool)) {
+                            kernel_size += region.GetSize();
                         }
                     }
                 }
@@ -697,86 +177,39 @@ namespace ams::kern {
             }
 
             static void InitializeLinearMemoryRegionTrees(KPhysicalAddress aligned_linear_phys_start, KVirtualAddress linear_virtual_start);
+            static size_t GetResourceRegionSizeForInit();
 
-            static NOINLINE auto GetKernelRegionExtents() {
-                return GetVirtualMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_Kernel);
+            static NOINLINE auto GetKernelRegionExtents()      { return GetVirtualMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_Kernel); }
+            static NOINLINE auto GetKernelCodeRegionExtents()  { return GetVirtualMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_KernelCode); }
+            static NOINLINE auto GetKernelStackRegionExtents() { return GetVirtualMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_KernelStack); }
+            static NOINLINE auto GetKernelMiscRegionExtents()  { return GetVirtualMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_KernelMisc); }
+            static NOINLINE auto GetKernelSlabRegionExtents()  { return GetVirtualMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_KernelSlab); }
+
+
+            static NOINLINE auto GetLinearRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionAttr_LinearMapped); }
+
+            static NOINLINE auto GetLinearRegionVirtualExtents()  {
+                auto physical = GetLinearRegionPhysicalExtents();
+                return KMemoryRegion(GetInteger(GetLinearVirtualAddress(physical.GetAddress())), physical.GetSize(), 0, KMemoryRegionType_None);
             }
 
-            static NOINLINE auto GetKernelCodeRegionExtents() {
-                return GetVirtualMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_KernelCode);
-            }
+            static NOINLINE auto GetMainMemoryPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_Dram); }
+            static NOINLINE auto GetCarveoutRegionExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionAttr_CarveoutProtected); }
 
-            static NOINLINE auto GetKernelStackRegionExtents() {
-                return GetVirtualMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_KernelStack);
-            }
+            static NOINLINE auto GetKernelRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramKernelBase); }
+            static NOINLINE auto GetKernelCodeRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramKernelCode); }
+            static NOINLINE auto GetKernelSlabRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramKernelSlab); }
+            static NOINLINE auto GetKernelPageTableHeapRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramKernelPtHeap); }
+            static NOINLINE auto GetKernelInitPageTableRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramKernelInitPt); }
 
-            static NOINLINE auto GetKernelMiscRegionExtents() {
-                return GetVirtualMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_KernelMisc);
-            }
+            static NOINLINE auto GetKernelPoolManagementRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramPoolManagement); }
+            static NOINLINE auto GetKernelPoolPartitionRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramPoolPartition); }
+            static NOINLINE auto GetKernelSystemPoolRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramSystemPool); }
+            static NOINLINE auto GetKernelSystemNonSecurePoolRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramSystemNonSecurePool); }
+            static NOINLINE auto GetKernelAppletPoolRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramAppletPool); }
+            static NOINLINE auto GetKernelApplicationPoolRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramApplicationPool); }
 
-            static NOINLINE auto GetKernelSlabRegionExtents() {
-                return GetVirtualMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_KernelSlab);
-            }
-
-            static NOINLINE const KMemoryRegion &GetCoreLocalRegion() {
-                return *GetVirtualMemoryRegionTree().FindFirstRegionByType(KMemoryRegionType_CoreLocal);
-            }
-
-            static NOINLINE auto GetLinearRegionPhysicalExtents() {
-                return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionAttr_LinearMapped);
-            }
-
-            static NOINLINE auto GetLinearRegionExtents() {
-                return GetVirtualLinearExtents(GetLinearRegionPhysicalExtents());
-            }
-
-            static NOINLINE auto GetCarveoutRegionExtents() {
-                return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionAttr_CarveoutProtected);
-            }
-
-            static NOINLINE auto GetKernelRegionPhysicalExtents() {
-                return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramKernel);
-            }
-
-            static NOINLINE auto GetKernelCodeRegionPhysicalExtents() {
-                return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramKernelCode);
-            }
-
-            static NOINLINE auto GetKernelSlabRegionPhysicalExtents() {
-                return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramKernelSlab);
-            }
-
-            static NOINLINE auto GetKernelPageTableHeapRegionPhysicalExtents() {
-                return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramKernelPtHeap);
-            }
-
-            static NOINLINE auto GetKernelInitPageTableRegionPhysicalExtents() {
-                return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramKernelInitPt);
-            }
-
-            static NOINLINE auto GetKernelPoolPartitionRegionPhysicalExtents() {
-                return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramPoolPartition);
-            }
-
-            static NOINLINE auto GetKernelMetadataPoolRegionPhysicalExtents() {
-                return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramMetadataPool);
-            }
-
-            static NOINLINE auto GetKernelSystemPoolRegionPhysicalExtents() {
-                return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramSystemPool);
-            }
-
-            static NOINLINE auto GetKernelSystemNonSecurePoolRegionPhysicalExtents() {
-                return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramSystemNonSecurePool);
-            }
-
-            static NOINLINE auto GetKernelAppletPoolRegionPhysicalExtents() {
-                return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramAppletPool);
-            }
-
-            static NOINLINE auto GetKernelApplicationPoolRegionPhysicalExtents() {
-                return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramApplicationPool);
-            }
+            static NOINLINE auto GetKernelTraceBufferRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_KernelTraceBuffer); }
     };
 
 

libraries/libmesosphere/include/mesosphere/kern_k_memory_manager.hpp

@@ -38,6 +38,7 @@ namespace ams::kern {
 
                 /* Aliases. */
                 Pool_Unsafe = Pool_Application,
+                Pool_Secure = Pool_System,
             };
 
             enum Direction {
@@ -54,7 +55,7 @@ namespace ams::kern {
                 private:
                     using RefCount = u16;
                 public:
-                    static size_t CalculateMetadataOverheadSize(size_t region_size);
+                    static size_t CalculateManagementOverheadSize(size_t region_size);
 
                     static constexpr size_t CalculateOptimizedProcessOverheadSize(size_t region_size) {
                         return (util::AlignUp((region_size / PageSize), BITSIZEOF(u64)) / BITSIZEOF(u64)) * sizeof(u64);
@@ -62,24 +63,24 @@ namespace ams::kern {
                 private:
                     KPageHeap heap;
                     RefCount *page_reference_counts;
-                    KVirtualAddress metadata_region;
+                    KVirtualAddress management_region;
                     Pool pool;
                     Impl *next;
                     Impl *prev;
                 public:
-                    Impl() : heap(), page_reference_counts(), metadata_region(), pool(), next(), prev() { /* ... */ }
+                    Impl() : heap(), page_reference_counts(), management_region(), pool(), next(), prev() { /* ... */ }
 
-                    size_t Initialize(const KMemoryRegion *region, Pool pool, KVirtualAddress metadata_region, KVirtualAddress metadata_region_end);
+                    size_t Initialize(const KMemoryRegion *region, Pool pool, KVirtualAddress management_region, KVirtualAddress management_region_end);
 
                     KVirtualAddress AllocateBlock(s32 index, bool random) { return this->heap.AllocateBlock(index, random); }
                     void Free(KVirtualAddress addr, size_t num_pages) { this->heap.Free(addr, num_pages); }
 
-                    void InitializeOptimizedMemory() { std::memset(GetVoidPointer(this->metadata_region), 0, CalculateOptimizedProcessOverheadSize(this->heap.GetSize())); }
+                    void InitializeOptimizedMemory() { std::memset(GetVoidPointer(this->management_region), 0, CalculateOptimizedProcessOverheadSize(this->heap.GetSize())); }
 
                     void TrackUnoptimizedAllocation(KVirtualAddress block, size_t num_pages);
-                    size_t TrackOptimizedAllocation(KVirtualAddress block, size_t num_pages);
+                    void TrackOptimizedAllocation(KVirtualAddress block, size_t num_pages);
 
-                    size_t ProcessOptimizedAllocation(bool *out_any_new, KVirtualAddress block, size_t num_pages, u8 fill_pattern);
+                    bool ProcessOptimizedAllocation(KVirtualAddress block, size_t num_pages, u8 fill_pattern);
 
                     constexpr Pool GetPool() const { return this->pool; }
                     constexpr size_t GetSize() const { return this->heap.GetSize(); }
@@ -87,15 +88,16 @@ namespace ams::kern {
 
                     size_t GetFreeSize() const { return this->heap.GetFreeSize(); }
 
+                    constexpr size_t GetPageOffset(KVirtualAddress address)      const { return this->heap.GetPageOffset(address); }
+                    constexpr size_t GetPageOffsetToEnd(KVirtualAddress address) const { return this->heap.GetPageOffsetToEnd(address); }
+
                     constexpr void SetNext(Impl *n) { this->next = n; }
                     constexpr void SetPrev(Impl *n) { this->prev = n; }
                     constexpr Impl *GetNext() const { return this->next; }
                     constexpr Impl *GetPrev() const { return this->prev; }
 
-                    void Open(KLightLock *pool_locks, KVirtualAddress address, size_t num_pages) {
-                        KScopedLightLock lk(pool_locks[this->pool]);
-
-                        size_t index = this->heap.GetPageOffset(address);
+                    void Open(KVirtualAddress address, size_t num_pages) {
+                        size_t index = this->GetPageOffset(address);
                         const size_t end = index + num_pages;
                         while (index < end) {
                             const RefCount ref_count = (++this->page_reference_counts[index]);
@@ -105,10 +107,8 @@ namespace ams::kern {
                         }
                     }
 
-                    void Close(KLightLock *pool_locks, KVirtualAddress address, size_t num_pages) {
-                        KScopedLightLock lk(pool_locks[this->pool]);
-
-                        size_t index = this->heap.GetPageOffset(address);
+                    void Close(KVirtualAddress address, size_t num_pages) {
+                        size_t index = this->GetPageOffset(address);
                         const size_t end = index + num_pages;
 
                         size_t free_start = 0;
@@ -173,7 +173,7 @@ namespace ams::kern {
                 /* ... */
             }
 
-            NOINLINE void Initialize(KVirtualAddress metadata_region, size_t metadata_region_size);
+            NOINLINE void Initialize(KVirtualAddress management_region, size_t management_region_size);
 
             NOINLINE Result InitializeOptimizedMemory(u64 process_id, Pool pool);
             NOINLINE void FinalizeOptimizedMemory(u64 process_id, Pool pool);
@@ -186,8 +186,13 @@ namespace ams::kern {
                 /* Repeatedly open references until we've done so for all pages. */
                 while (num_pages) {
                     auto &manager = this->GetManager(address);
-                    const size_t cur_pages = std::min(num_pages, (manager.GetEndAddress() - address) / PageSize);
-                    manager.Open(this->pool_locks, address, cur_pages);
+                    const size_t cur_pages = std::min(num_pages, manager.GetPageOffsetToEnd(address));
+
+                    {
+                        KScopedLightLock lk(this->pool_locks[manager.GetPool()]);
+                        manager.Open(address, cur_pages);
+                    }
+
                     num_pages -= cur_pages;
                     address += cur_pages * PageSize;
                 }
@@ -197,8 +202,13 @@ namespace ams::kern {
                 /* Repeatedly close references until we've done so for all pages. */
                 while (num_pages) {
                     auto &manager = this->GetManager(address);
-                    const size_t cur_pages = std::min(num_pages, (manager.GetEndAddress() - address) / PageSize);
-                    manager.Close(this->pool_locks, address, cur_pages);
+                    const size_t cur_pages = std::min(num_pages, manager.GetPageOffsetToEnd(address));
+
+                    {
+                        KScopedLightLock lk(this->pool_locks[manager.GetPool()]);
+                        manager.Close(address, cur_pages);
+                    }
+
                     num_pages -= cur_pages;
                     address += cur_pages * PageSize;
                 }
@@ -238,8 +248,8 @@ namespace ams::kern {
                 return total;
             }
         public:
-            static size_t CalculateMetadataOverheadSize(size_t region_size) {
-                return Impl::CalculateMetadataOverheadSize(region_size);
+            static size_t CalculateManagementOverheadSize(size_t region_size) {
+                return Impl::CalculateManagementOverheadSize(region_size);
             }
 
             static constexpr ALWAYS_INLINE u32 EncodeOption(Pool pool, Direction dir) {

libraries/libmesosphere/include/mesosphere/kern_k_memory_region.hpp

@@ -0,0 +1,318 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_k_memory_region_type.hpp>
+
+namespace ams::kern {
+
+    class KMemoryRegionTree;
+
+    class KMemoryRegion : public util::IntrusiveRedBlackTreeBaseNode<KMemoryRegion> {
+        NON_COPYABLE(KMemoryRegion);
+        NON_MOVEABLE(KMemoryRegion);
+        private:
+            friend class KMemoryRegionTree;
+        private:
+            uintptr_t address;
+            uintptr_t pair_address;
+            size_t region_size;
+            u32 attributes;
+            u32 type_id;
+        public:
+            static constexpr ALWAYS_INLINE int Compare(const KMemoryRegion &lhs, const KMemoryRegion &rhs) {
+                if (lhs.GetAddress() < rhs.GetAddress()) {
+                    return -1;
+                } else if (lhs.GetAddress() <= rhs.GetLastAddress()) {
+                    return 0;
+                } else {
+                    return 1;
+                }
+            }
+        public:
+            constexpr ALWAYS_INLINE KMemoryRegion() : address(0), pair_address(0), region_size(0), attributes(0), type_id(0) { /* ... */ }
+            constexpr ALWAYS_INLINE KMemoryRegion(uintptr_t a, size_t rs, uintptr_t p, u32 r, u32 t) :
+                address(a), pair_address(p), region_size(rs), attributes(r), type_id(t)
+            {
+                /* ... */
+            }
+            constexpr ALWAYS_INLINE KMemoryRegion(uintptr_t a, size_t rs, u32 r, u32 t) : KMemoryRegion(a, rs, std::numeric_limits<uintptr_t>::max(), r, t) { /* ... */ }
+        private:
+            constexpr ALWAYS_INLINE void Reset(uintptr_t a, uintptr_t rs, uintptr_t p, u32 r, u32 t) {
+                this->address      = a;
+                this->pair_address = p;
+                this->region_size  = rs;
+                this->attributes   = r;
+                this->type_id      = t;
+            }
+        public:
+            constexpr ALWAYS_INLINE uintptr_t GetAddress() const {
+                return this->address;
+            }
+
+            constexpr ALWAYS_INLINE uintptr_t GetPairAddress() const {
+                return this->pair_address;
+            }
+
+            constexpr ALWAYS_INLINE size_t GetSize() const {
+                return this->region_size;
+            }
+
+            constexpr ALWAYS_INLINE uintptr_t GetEndAddress() const {
+                return this->GetAddress() + this->GetSize();
+            }
+
+            constexpr ALWAYS_INLINE uintptr_t GetLastAddress() const {
+                return this->GetEndAddress() - 1;
+            }
+
+            constexpr ALWAYS_INLINE u32 GetAttributes() const {
+                return this->attributes;
+            }
+
+            constexpr ALWAYS_INLINE u32 GetType() const {
+                return this->type_id;
+            }
+
+            constexpr ALWAYS_INLINE void SetType(u32 type) {
+                MESOSPHERE_INIT_ABORT_UNLESS(this->CanDerive(type));
+                this->type_id = type;
+            }
+
+            constexpr ALWAYS_INLINE bool Contains(uintptr_t address) const {
+                return this->GetAddress() <= address && address <= this->GetLastAddress();
+            }
+
+            constexpr ALWAYS_INLINE bool IsDerivedFrom(u32 type) const {
+                return (this->GetType() | type) == this->GetType();
+            }
+
+            constexpr ALWAYS_INLINE bool HasTypeAttribute(KMemoryRegionAttr attr) const {
+                return (this->GetType() | attr) == this->GetType();
+            }
+
+            constexpr ALWAYS_INLINE bool CanDerive(u32 type) const {
+                return (this->GetType() | type) == type;
+            }
+
+            constexpr ALWAYS_INLINE void SetPairAddress(uintptr_t a) {
+                this->pair_address = a;
+            }
+
+            constexpr ALWAYS_INLINE void SetTypeAttribute(KMemoryRegionAttr attr) {
+                this->type_id |= attr;
+            }
+    };
+    static_assert(std::is_trivially_destructible<KMemoryRegion>::value);
+
+    class KMemoryRegionTree {
+        public:
+            struct DerivedRegionExtents {
+                const KMemoryRegion *first_region;
+                const KMemoryRegion *last_region;
+
+                constexpr DerivedRegionExtents() : first_region(nullptr), last_region(nullptr) { /* ... */ }
+
+                constexpr ALWAYS_INLINE uintptr_t GetAddress() const {
+                    return this->first_region->GetAddress();
+                }
+
+                constexpr ALWAYS_INLINE uintptr_t GetEndAddress() const {
+                    return this->last_region->GetEndAddress();
+                }
+
+                constexpr ALWAYS_INLINE size_t GetSize() const {
+                    return this->GetEndAddress() - this->GetAddress();
+                }
+
+                constexpr ALWAYS_INLINE uintptr_t GetLastAddress() const {
+                    return this->GetEndAddress() - 1;
+                }
+            };
+        private:
+            using TreeType = util::IntrusiveRedBlackTreeBaseTraits<KMemoryRegion>::TreeType<KMemoryRegion>;
+        public:
+            using value_type        = TreeType::value_type;
+            using size_type         = TreeType::size_type;
+            using difference_type   = TreeType::difference_type;
+            using pointer           = TreeType::pointer;
+            using const_pointer     = TreeType::const_pointer;
+            using reference         = TreeType::reference;
+            using const_reference   = TreeType::const_reference;
+            using iterator          = TreeType::iterator;
+            using const_iterator    = TreeType::const_iterator;
+        private:
+            TreeType tree;
+        public:
+            constexpr ALWAYS_INLINE KMemoryRegionTree() : tree() { /* ... */ }
+        public:
+            KMemoryRegion *FindModifiable(uintptr_t address) {
+                if (auto it = this->find(KMemoryRegion(address, 1, 0, 0)); it != this->end()) {
+                    return std::addressof(*it);
+                } else {
+                    return nullptr;
+                }
+            }
+
+            const KMemoryRegion *Find(uintptr_t address) const {
+                if (auto it = this->find(KMemoryRegion(address, 1, 0, 0)); it != this->cend()) {
+                    return std::addressof(*it);
+                } else {
+                    return nullptr;
+                }
+            }
+
+            const KMemoryRegion *FindByType(u32 type_id) const {
+                for (auto it = this->cbegin(); it != this->cend(); ++it) {
+                    if (it->GetType() == type_id) {
+                        return std::addressof(*it);
+                    }
+                }
+                return nullptr;
+            }
+
+            const KMemoryRegion *FindByTypeAndAttribute(u32 type_id, u32 attr) const {
+                for (auto it = this->cbegin(); it != this->cend(); ++it) {
+                    if (it->GetType() == type_id && it->GetAttributes() == attr) {
+                        return std::addressof(*it);
+                    }
+                }
+                return nullptr;
+            }
+
+            const KMemoryRegion *FindFirstDerived(u32 type_id) const {
+                for (auto it = this->cbegin(); it != this->cend(); it++) {
+                    if (it->IsDerivedFrom(type_id)) {
+                        return std::addressof(*it);
+                    }
+                }
+                return nullptr;
+            }
+
+            const KMemoryRegion *FindLastDerived(u32 type_id) const {
+                const KMemoryRegion *region = nullptr;
+                for (auto it = this->begin(); it != this->end(); it++) {
+                    if (it->IsDerivedFrom(type_id)) {
+                        region = std::addressof(*it);
+                    }
+                }
+                return region;
+            }
+
+
+            DerivedRegionExtents GetDerivedRegionExtents(u32 type_id) const {
+                DerivedRegionExtents extents;
+
+                MESOSPHERE_INIT_ABORT_UNLESS(extents.first_region == nullptr);
+                MESOSPHERE_INIT_ABORT_UNLESS(extents.last_region  == nullptr);
+
+                for (auto it = this->cbegin(); it != this->cend(); it++) {
+                    if (it->IsDerivedFrom(type_id)) {
+                        if (extents.first_region == nullptr) {
+                            extents.first_region = std::addressof(*it);
+                        }
+                        extents.last_region = std::addressof(*it);
+                    }
+                }
+
+                MESOSPHERE_INIT_ABORT_UNLESS(extents.first_region != nullptr);
+                MESOSPHERE_INIT_ABORT_UNLESS(extents.last_region  != nullptr);
+
+                return extents;
+            }
+        public:
+            NOINLINE void InsertDirectly(uintptr_t address, size_t size, u32 attr = 0, u32 type_id = 0);
+            NOINLINE bool Insert(uintptr_t address, size_t size, u32 type_id, u32 new_attr = 0, u32 old_attr = 0);
+
+            NOINLINE KVirtualAddress GetRandomAlignedRegion(size_t size, size_t alignment, u32 type_id);
+
+            ALWAYS_INLINE KVirtualAddress GetRandomAlignedRegionWithGuard(size_t size, size_t alignment, u32 type_id, size_t guard_size) {
+                return this->GetRandomAlignedRegion(size + 2 * guard_size, alignment, type_id) + guard_size;
+            }
+        public:
+            /* Iterator accessors. */
+            iterator begin() {
+                return this->tree.begin();
+            }
+
+            const_iterator begin() const {
+                return this->tree.begin();
+            }
+
+            iterator end() {
+                return this->tree.end();
+            }
+
+            const_iterator end() const {
+                return this->tree.end();
+            }
+
+            const_iterator cbegin() const {
+                return this->begin();
+            }
+
+            const_iterator cend() const {
+                return this->end();
+            }
+
+            iterator iterator_to(reference ref) {
+                return this->tree.iterator_to(ref);
+            }
+
+            const_iterator iterator_to(const_reference ref) const {
+                return this->tree.iterator_to(ref);
+            }
+
+            /* Content management. */
+            bool empty() const {
+                return this->tree.empty();
+            }
+
+            reference back() {
+                return this->tree.back();
+            }
+
+            const_reference back() const {
+                return this->tree.back();
+            }
+
+            reference front() {
+                return this->tree.front();
+            }
+
+            const_reference front() const {
+                return this->tree.front();
+            }
+
+            /* GCC over-eagerly inlines this operation. */
+            NOINLINE iterator insert(reference ref) {
+                return this->tree.insert(ref);
+            }
+
+            NOINLINE iterator erase(iterator it) {
+                return this->tree.erase(it);
+            }
+
+            iterator find(const_reference ref) const {
+                return this->tree.find(ref);
+            }
+
+            iterator nfind(const_reference ref) const {
+                return this->tree.nfind(ref);
+            }
+    };
+
+}

libraries/libmesosphere/include/mesosphere/kern_k_memory_region_type.hpp

@@ -0,0 +1,300 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+
+namespace ams::kern {
+
+    enum KMemoryRegionType : u32 {};
+
+    enum KMemoryRegionAttr : typename std::underlying_type<KMemoryRegionType>::type {
+        KMemoryRegionAttr_CarveoutProtected = 0x04000000,
+        KMemoryRegionAttr_DidKernelMap      = 0x08000000,
+        KMemoryRegionAttr_ShouldKernelMap   = 0x10000000,
+        KMemoryRegionAttr_UserReadOnly      = 0x20000000,
+        KMemoryRegionAttr_NoUserMap         = 0x40000000,
+        KMemoryRegionAttr_LinearMapped      = 0x80000000,
+    };
+
+    namespace impl {
+
+        consteval size_t BitsForDeriveSparse(size_t n) {
+            return n + 1;
+        }
+
+        consteval size_t BitsForDeriveDense(size_t n) {
+            AMS_ASSUME(n > 0);
+
+            size_t low = 0, high = 1;
+            for (size_t i = 0; i < n - 1; ++i) {
+                if ((++low) == high) {
+                    ++high;
+                    low = 0;
+                }
+            }
+            return high + 1;
+        }
+
+        class KMemoryRegionTypeValue {
+            private:
+                using ValueType = typename std::underlying_type<KMemoryRegionType>::type;
+            private:
+                ValueType value;
+                size_t next_bit;
+                bool finalized;
+                bool sparse_only;
+                bool dense_only;
+            private:
+                consteval KMemoryRegionTypeValue(ValueType v) : value(v), next_bit(0), finalized(false), sparse_only(false), dense_only(false) { /* ... */ }
+            public:
+                consteval KMemoryRegionTypeValue() : KMemoryRegionTypeValue(0) { /* ... */ }
+
+                consteval operator KMemoryRegionType() const { return static_cast<KMemoryRegionType>(this->value); }
+                consteval ValueType GetValue() const { return this->value; }
+
+                consteval const KMemoryRegionTypeValue &Finalize()      { this->finalized = true;   return *this; }
+                consteval const KMemoryRegionTypeValue &SetSparseOnly() { this->sparse_only = true; return *this; }
+                consteval const KMemoryRegionTypeValue &SetDenseOnly() { this->dense_only = true; return *this; }
+
+                consteval KMemoryRegionTypeValue &SetAttribute(KMemoryRegionAttr attr) { AMS_ASSUME(!this->finalized); this->value |= attr; return *this; }
+
+                consteval KMemoryRegionTypeValue DeriveInitial(size_t i, size_t next = BITSIZEOF(ValueType)) const {
+                    AMS_ASSUME(!this->finalized);
+                    AMS_ASSUME(!this->value);
+                    AMS_ASSUME(!this->next_bit);
+                    AMS_ASSUME(next > i);
+
+                    KMemoryRegionTypeValue new_type = *this;
+                    new_type.value    = (ValueType{1} << i);
+                    new_type.next_bit = next;
+                    return new_type;
+                }
+
+                consteval KMemoryRegionTypeValue DeriveAttribute(KMemoryRegionAttr attr) const {
+                    AMS_ASSUME(!this->finalized);
+
+                    KMemoryRegionTypeValue new_type = *this;
+                    new_type.value |= attr;
+                    return new_type;
+                }
+
+                consteval KMemoryRegionTypeValue DeriveTransition(size_t ofs = 0, size_t adv = 1) const {
+                    AMS_ASSUME(!this->finalized);
+                    AMS_ASSUME(ofs < adv);
+                    AMS_ASSUME(this->next_bit + adv <= BITSIZEOF(ValueType));
+
+                    KMemoryRegionTypeValue new_type = *this;
+                    new_type.value |= (ValueType{1} << (this->next_bit + ofs));
+                    new_type.next_bit += adv;
+                    return new_type;
+                }
+
+                consteval KMemoryRegionTypeValue DeriveSparse(size_t ofs, size_t n, size_t i) const {
+                    AMS_ASSUME(!this->finalized);
+                    AMS_ASSUME(!this->dense_only);
+                    AMS_ASSUME(this->next_bit + ofs + n + 1 <= BITSIZEOF(ValueType));
+                    AMS_ASSUME(i < n);
+
+                    KMemoryRegionTypeValue new_type = *this;
+                    new_type.value |= (ValueType{1} << (this->next_bit + ofs));
+                    new_type.value |= (ValueType{1} << (this->next_bit + ofs + 1 + i));
+                    new_type.next_bit += ofs + n + 1;
+                    return new_type;
+                }
+
+                consteval KMemoryRegionTypeValue Derive(size_t n, size_t i) const {
+                    AMS_ASSUME(!this->finalized);
+                    AMS_ASSUME(!this->sparse_only);
+                    AMS_ASSUME(this->next_bit + BitsForDeriveDense(n) <= BITSIZEOF(ValueType));
+                    AMS_ASSUME(i < n);
+
+                    size_t low = 0, high = 1;
+                    for (size_t j = 0; j < i; ++j) {
+                        if ((++low) == high) {
+                            ++high;
+                            low = 0;
+                        }
+                    }
+                    AMS_ASSUME(high < BitsForDeriveDense(n));
+
+
+                    KMemoryRegionTypeValue new_type = *this;
+                    new_type.value |= (ValueType{1} << (this->next_bit + low));
+                    new_type.value |= (ValueType{1} << (this->next_bit + high));
+                    new_type.next_bit += BitsForDeriveDense(n);
+                    return new_type;
+                }
+
+                consteval KMemoryRegionTypeValue Advance(size_t n) const {
+                    AMS_ASSUME(!this->finalized);
+                    AMS_ASSUME(this->next_bit + n <= BITSIZEOF(ValueType));
+
+                    KMemoryRegionTypeValue new_type = *this;
+                    new_type.next_bit += n;
+                    return new_type;
+                }
+
+                constexpr ALWAYS_INLINE bool IsAncestorOf(ValueType v) const {
+                    return (this->value | v) == v;
+                }
+        };
+
+    }
+
+    constexpr inline const auto KMemoryRegionType_None = impl::KMemoryRegionTypeValue();
+
+    constexpr inline const auto KMemoryRegionType_Kernel          = KMemoryRegionType_None.DeriveInitial(0, 2);
+    constexpr inline const auto KMemoryRegionType_Dram            = KMemoryRegionType_None.DeriveInitial(1, 2);
+    constexpr inline const auto KMemoryRegionType_CoreLocalRegion = KMemoryRegionType_None.DeriveInitial(2).Finalize();
+    static_assert(KMemoryRegionType_Kernel         .GetValue() == 0x1);
+    static_assert(KMemoryRegionType_Dram           .GetValue() == 0x2);
+    static_assert(KMemoryRegionType_CoreLocalRegion.GetValue() == 0x4);
+
+    constexpr inline const auto KMemoryRegionType_DramKernelBase   = KMemoryRegionType_Dram.DeriveSparse(0, 3, 0).SetAttribute(KMemoryRegionAttr_NoUserMap).SetAttribute(KMemoryRegionAttr_CarveoutProtected);
+    constexpr inline const auto KMemoryRegionType_DramReservedBase = KMemoryRegionType_Dram.DeriveSparse(0, 3, 1);
+    constexpr inline const auto KMemoryRegionType_DramHeapBase     = KMemoryRegionType_Dram.DeriveSparse(0, 3, 2).SetAttribute(KMemoryRegionAttr_LinearMapped);
+    static_assert(KMemoryRegionType_DramKernelBase  .GetValue() == (0xE  | KMemoryRegionAttr_CarveoutProtected | KMemoryRegionAttr_NoUserMap));
+    static_assert(KMemoryRegionType_DramReservedBase.GetValue() == (0x16));
+    static_assert(KMemoryRegionType_DramHeapBase    .GetValue() == (0x26 | KMemoryRegionAttr_LinearMapped));
+
+    constexpr inline const auto KMemoryRegionType_DramKernelCode   = KMemoryRegionType_DramKernelBase.DeriveSparse(0, 4, 0);
+    constexpr inline const auto KMemoryRegionType_DramKernelSlab   = KMemoryRegionType_DramKernelBase.DeriveSparse(0, 4, 1);
+    constexpr inline const auto KMemoryRegionType_DramKernelPtHeap = KMemoryRegionType_DramKernelBase.DeriveSparse(0, 4, 2).SetAttribute(KMemoryRegionAttr_LinearMapped);
+    constexpr inline const auto KMemoryRegionType_DramKernelInitPt = KMemoryRegionType_DramKernelBase.DeriveSparse(0, 4, 3).SetAttribute(KMemoryRegionAttr_LinearMapped);
+    static_assert(KMemoryRegionType_DramKernelCode  .GetValue() == (0xCE  | KMemoryRegionAttr_CarveoutProtected | KMemoryRegionAttr_NoUserMap));
+    static_assert(KMemoryRegionType_DramKernelSlab  .GetValue() == (0x14E | KMemoryRegionAttr_CarveoutProtected | KMemoryRegionAttr_NoUserMap));
+    static_assert(KMemoryRegionType_DramKernelPtHeap.GetValue() == (0x24E | KMemoryRegionAttr_CarveoutProtected | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_LinearMapped));
+    static_assert(KMemoryRegionType_DramKernelInitPt.GetValue() == (0x44E | KMemoryRegionAttr_CarveoutProtected | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_LinearMapped));
+
+
+    constexpr inline const auto KMemoryRegionType_DramReservedEarly = KMemoryRegionType_DramReservedBase.DeriveAttribute(KMemoryRegionAttr_NoUserMap);
+    static_assert(KMemoryRegionType_DramReservedEarly.GetValue() == (0x16 | KMemoryRegionAttr_NoUserMap));
+
+                                                                                              /* UNUSED: DeriveSparse(0, 3, 0); */
+    constexpr inline const auto KMemoryRegionType_OnMemoryBootImage = KMemoryRegionType_DramReservedBase.DeriveSparse(0, 3, 1);
+    constexpr inline const auto KMemoryRegionType_DTB               = KMemoryRegionType_DramReservedBase.DeriveSparse(0, 3, 2);
+    static_assert(KMemoryRegionType_OnMemoryBootImage.GetValue() == 0x156);
+    static_assert(KMemoryRegionType_DTB.GetValue() == 0x256);
+
+    constexpr inline const auto KMemoryRegionType_KernelTraceBuffer = KMemoryRegionType_DramHeapBase.DeriveTransition(1, 3).SetAttribute(KMemoryRegionAttr_UserReadOnly);
+    static_assert(KMemoryRegionType_KernelTraceBuffer.GetValue() == (0xA6 | KMemoryRegionAttr_LinearMapped | KMemoryRegionAttr_UserReadOnly));
+
+    constexpr inline const auto KMemoryRegionType_DramPoolPartition = KMemoryRegionType_DramHeapBase.DeriveAttribute(KMemoryRegionAttr_NoUserMap);
+    static_assert(KMemoryRegionType_DramPoolPartition.GetValue() == (0x26 | KMemoryRegionAttr_LinearMapped | KMemoryRegionAttr_NoUserMap));
+
+    constexpr inline const auto KMemoryRegionType_DramPoolManagement = KMemoryRegionType_DramPoolPartition.DeriveTransition(0, 2).DeriveTransition().SetAttribute(KMemoryRegionAttr_CarveoutProtected);
+    constexpr inline const auto KMemoryRegionType_DramUserPool       = KMemoryRegionType_DramPoolPartition.DeriveTransition(1, 2).DeriveTransition();
+    static_assert(KMemoryRegionType_DramPoolManagement.GetValue() == (0x166 | KMemoryRegionAttr_LinearMapped | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_CarveoutProtected));
+    static_assert(KMemoryRegionType_DramUserPool.GetValue()       == (0x1A6 | KMemoryRegionAttr_LinearMapped | KMemoryRegionAttr_NoUserMap));
+
+    constexpr inline const auto KMemoryRegionType_DramApplicationPool     = KMemoryRegionType_DramUserPool.Derive(4, 0);
+    constexpr inline const auto KMemoryRegionType_DramAppletPool          = KMemoryRegionType_DramUserPool.Derive(4, 1);
+    constexpr inline const auto KMemoryRegionType_DramSystemNonSecurePool = KMemoryRegionType_DramUserPool.Derive(4, 2);
+    constexpr inline const auto KMemoryRegionType_DramSystemPool          = KMemoryRegionType_DramUserPool.Derive(4, 3).SetAttribute(KMemoryRegionAttr_CarveoutProtected);
+    static_assert(KMemoryRegionType_DramApplicationPool    .GetValue()   == (0x7A6  | KMemoryRegionAttr_LinearMapped | KMemoryRegionAttr_NoUserMap));
+    static_assert(KMemoryRegionType_DramAppletPool         .GetValue()   == (0xBA6  | KMemoryRegionAttr_LinearMapped | KMemoryRegionAttr_NoUserMap));
+    static_assert(KMemoryRegionType_DramSystemNonSecurePool.GetValue()   == (0xDA6  | KMemoryRegionAttr_LinearMapped | KMemoryRegionAttr_NoUserMap));
+    static_assert(KMemoryRegionType_DramSystemPool         .GetValue()   == (0x13A6 | KMemoryRegionAttr_LinearMapped | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_CarveoutProtected));
+
+    constexpr inline const auto KMemoryRegionType_VirtualDramHeapBase          = KMemoryRegionType_Dram.DeriveSparse(1, 3, 0);
+    constexpr inline const auto KMemoryRegionType_VirtualDramKernelPtHeap      = KMemoryRegionType_Dram.DeriveSparse(1, 3, 1);
+    constexpr inline const auto KMemoryRegionType_VirtualDramKernelTraceBuffer = KMemoryRegionType_Dram.DeriveSparse(1, 3, 2);
+    static_assert(KMemoryRegionType_VirtualDramHeapBase         .GetValue() == 0x1A);
+    static_assert(KMemoryRegionType_VirtualDramKernelPtHeap     .GetValue() == 0x2A);
+    static_assert(KMemoryRegionType_VirtualDramKernelTraceBuffer.GetValue() == 0x4A);
+
+    constexpr inline const auto KMemoryRegionType_VirtualDramKernelInitPt   = KMemoryRegionType_VirtualDramHeapBase.Derive(3, 0);
+    constexpr inline const auto KMemoryRegionType_VirtualDramPoolManagement = KMemoryRegionType_VirtualDramHeapBase.Derive(3, 1);
+    constexpr inline const auto KMemoryRegionType_VirtualDramUserPool       = KMemoryRegionType_VirtualDramHeapBase.Derive(3, 2);
+    static_assert(KMemoryRegionType_VirtualDramKernelInitPt  .GetValue() == 0x19A);
+    static_assert(KMemoryRegionType_VirtualDramPoolManagement.GetValue() == 0x29A);
+    static_assert(KMemoryRegionType_VirtualDramUserPool      .GetValue() == 0x31A);
+
+    /* NOTE: For unknown reason, the pools are derived out-of-order here. */
+    /* It's worth eventually trying to understand why Nintendo made this choice. */
+                                                                                                             /* UNUSED: .Derive(6, 0); */
+                                                                                                             /* UNUSED: .Derive(6, 1); */
+    constexpr inline const auto KMemoryRegionType_VirtualDramAppletPool          = KMemoryRegionType_VirtualDramUserPool.Derive(6, 2);
+    constexpr inline const auto KMemoryRegionType_VirtualDramApplicationPool     = KMemoryRegionType_VirtualDramUserPool.Derive(6, 3);
+    constexpr inline const auto KMemoryRegionType_VirtualDramSystemNonSecurePool = KMemoryRegionType_VirtualDramUserPool.Derive(6, 4);
+    constexpr inline const auto KMemoryRegionType_VirtualDramSystemPool          = KMemoryRegionType_VirtualDramUserPool.Derive(6, 5);
+    static_assert(KMemoryRegionType_VirtualDramAppletPool         .GetValue()   == 0x1B1A);
+    static_assert(KMemoryRegionType_VirtualDramApplicationPool    .GetValue()   == 0x271A);
+    static_assert(KMemoryRegionType_VirtualDramSystemNonSecurePool.GetValue()   == 0x2B1A);
+    static_assert(KMemoryRegionType_VirtualDramSystemPool         .GetValue()   == 0x331A);
+
+    constexpr inline const auto KMemoryRegionType_ArchDeviceBase  = KMemoryRegionType_Kernel.DeriveTransition(0, 1).SetSparseOnly();
+    constexpr inline const auto KMemoryRegionType_BoardDeviceBase = KMemoryRegionType_Kernel.DeriveTransition(0, 2).SetDenseOnly();
+    static_assert(KMemoryRegionType_ArchDeviceBase .GetValue()   == 0x5);
+    static_assert(KMemoryRegionType_BoardDeviceBase.GetValue()   == 0x5);
+
+    #if   defined(ATMOSPHERE_ARCH_ARM64)
+        #include <mesosphere/arch/arm64/kern_k_memory_region_device_types.inc>
+    #elif defined(ATMOSPHERE_ARCH_ARM)
+        #include <mesosphere/arch/arm/kern_k_memory_region_device_types.inc>
+    #else
+        /* Default to no architecture devices. */
+        constexpr inline const auto NumArchitectureDeviceRegions = 0;
+    #endif
+    static_assert(NumArchitectureDeviceRegions >= 0);
+
+    #if defined(ATMOSPHERE_BOARD_NINTENDO_NX)
+        #include <mesosphere/board/nintendo/nx/kern_k_memory_region_device_types.inc>
+    #else
+        /* Default to no board devices. */
+        constexpr inline const auto NumBoardDeviceRegions = 0;
+    #endif
+    static_assert(NumBoardDeviceRegions >= 0);
+
+    constexpr inline const auto KMemoryRegionType_KernelCode  = KMemoryRegionType_Kernel.DeriveSparse(1, 4, 0);
+    constexpr inline const auto KMemoryRegionType_KernelStack = KMemoryRegionType_Kernel.DeriveSparse(1, 4, 1);
+    constexpr inline const auto KMemoryRegionType_KernelMisc  = KMemoryRegionType_Kernel.DeriveSparse(1, 4, 2);
+    constexpr inline const auto KMemoryRegionType_KernelSlab  = KMemoryRegionType_Kernel.DeriveSparse(1, 4, 3);
+    static_assert(KMemoryRegionType_KernelCode .GetValue() == 0x19);
+    static_assert(KMemoryRegionType_KernelStack.GetValue() == 0x29);
+    static_assert(KMemoryRegionType_KernelMisc .GetValue() == 0x49);
+    static_assert(KMemoryRegionType_KernelSlab .GetValue() == 0x89);
+
+    constexpr inline const auto KMemoryRegionType_KernelMiscDerivedBase = KMemoryRegionType_KernelMisc.DeriveTransition();
+    static_assert(KMemoryRegionType_KernelMiscDerivedBase.GetValue() == 0x149);
+
+                                                                                                         /* UNUSED: .Derive(7, 0); */
+    constexpr inline const auto KMemoryRegionType_KernelMiscMainStack      = KMemoryRegionType_KernelMiscDerivedBase.Derive(7, 1);
+    constexpr inline const auto KMemoryRegionType_KernelMiscMappedDevice   = KMemoryRegionType_KernelMiscDerivedBase.Derive(7, 2);
+    constexpr inline const auto KMemoryRegionType_KernelMiscIdleStack      = KMemoryRegionType_KernelMiscDerivedBase.Derive(7, 3);
+    constexpr inline const auto KMemoryRegionType_KernelMiscUnknownDebug   = KMemoryRegionType_KernelMiscDerivedBase.Derive(7, 4);
+                                                                                                         /* UNUSED: .Derive(7, 5); */
+    constexpr inline const auto KMemoryRegionType_KernelMiscExceptionStack = KMemoryRegionType_KernelMiscDerivedBase.Derive(7, 6);
+    static_assert(KMemoryRegionType_KernelMiscMainStack     .GetValue() == 0xB49);
+    static_assert(KMemoryRegionType_KernelMiscMappedDevice  .GetValue() == 0xD49);
+    static_assert(KMemoryRegionType_KernelMiscIdleStack     .GetValue() == 0x1349);
+    static_assert(KMemoryRegionType_KernelMiscUnknownDebug  .GetValue() == 0x1549);
+    static_assert(KMemoryRegionType_KernelMiscExceptionStack.GetValue() == 0x2349);
+
+    constexpr inline const auto KMemoryRegionType_KernelTemp = KMemoryRegionType_Kernel.Advance(2).Derive(2, 0);
+    static_assert(KMemoryRegionType_KernelTemp.GetValue() == 0x31);
+
+    constexpr ALWAYS_INLINE KMemoryRegionType GetTypeForVirtualLinearMapping(u32 type_id) {
+        if (KMemoryRegionType_KernelTraceBuffer.IsAncestorOf(type_id)) {
+            return KMemoryRegionType_VirtualDramKernelTraceBuffer;
+        } else if (KMemoryRegionType_DramKernelPtHeap.IsAncestorOf(type_id)) {
+            return KMemoryRegionType_VirtualDramKernelPtHeap;
+        } else {
+            return KMemoryRegionType_Dram;
+        }
+    }
+
+}

libraries/libmesosphere/include/mesosphere/kern_k_page_bitmap.hpp

@@ -254,7 +254,7 @@ namespace ams::kern {
                 }
             }
         public:
-            static constexpr size_t CalculateMetadataOverheadSize(size_t region_size) {
+            static constexpr size_t CalculateManagementOverheadSize(size_t region_size) {
                 size_t overhead_bits = 0;
                 for (s32 depth = GetRequiredDepth(region_size) - 1; depth >= 0; depth--) {
                     region_size = util::AlignUp(region_size, BITSIZEOF(u64)) / BITSIZEOF(u64);

libraries/libmesosphere/include/mesosphere/kern_k_page_heap.hpp

@@ -115,11 +115,11 @@ namespace ams::kern {
                         return this->heap_address + (offset << this->GetShift());
                     }
                 public:
-                    static constexpr size_t CalculateMetadataOverheadSize(size_t region_size, size_t cur_block_shift, size_t next_block_shift) {
+                    static constexpr size_t CalculateManagementOverheadSize(size_t region_size, size_t cur_block_shift, size_t next_block_shift) {
                         const size_t cur_block_size  = (u64(1) << cur_block_shift);
                         const size_t next_block_size = (u64(1) << next_block_shift);
                         const size_t align = (next_block_shift != 0) ? next_block_size : cur_block_size;
-                        return KPageBitmap::CalculateMetadataOverheadSize((align * 2 + util::AlignUp(region_size, align)) / cur_block_size);
+                        return KPageBitmap::CalculateManagementOverheadSize((align * 2 + util::AlignUp(region_size, align)) / cur_block_size);
                     }
             };
         private:
@@ -129,7 +129,7 @@ namespace ams::kern {
             size_t num_blocks;
             Block blocks[NumMemoryBlockPageShifts];
         private:
-            void Initialize(KVirtualAddress heap_address, size_t heap_size, KVirtualAddress metadata_address, size_t metadata_size, const size_t *block_shifts, size_t num_block_shifts);
+            void Initialize(KVirtualAddress heap_address, size_t heap_size, KVirtualAddress management_address, size_t management_size, const size_t *block_shifts, size_t num_block_shifts);
             size_t GetNumFreePages() const;
 
             void FreeBlock(KVirtualAddress block, s32 index);
@@ -142,8 +142,8 @@ namespace ams::kern {
             constexpr size_t GetPageOffset(KVirtualAddress block) const { return (block - this->GetAddress()) / PageSize; }
             constexpr size_t GetPageOffsetToEnd(KVirtualAddress block) const { return (this->GetEndAddress() - block) / PageSize; }
 
-            void Initialize(KVirtualAddress heap_address, size_t heap_size, KVirtualAddress metadata_address, size_t metadata_size) {
-                return Initialize(heap_address, heap_size, metadata_address, metadata_size, MemoryBlockPageShifts, NumMemoryBlockPageShifts);
+            void Initialize(KVirtualAddress heap_address, size_t heap_size, KVirtualAddress management_address, size_t management_size) {
+                return Initialize(heap_address, heap_size, management_address, management_size, MemoryBlockPageShifts, NumMemoryBlockPageShifts);
             }
 
             size_t GetFreeSize() const { return this->GetNumFreePages() * PageSize; }
@@ -155,10 +155,10 @@ namespace ams::kern {
             KVirtualAddress AllocateBlock(s32 index, bool random);
             void Free(KVirtualAddress addr, size_t num_pages);
         private:
-            static size_t CalculateMetadataOverheadSize(size_t region_size, const size_t *block_shifts, size_t num_block_shifts);
+            static size_t CalculateManagementOverheadSize(size_t region_size, const size_t *block_shifts, size_t num_block_shifts);
         public:
-            static size_t CalculateMetadataOverheadSize(size_t region_size) {
-                return CalculateMetadataOverheadSize(region_size, MemoryBlockPageShifts, NumMemoryBlockPageShifts);
+            static size_t CalculateManagementOverheadSize(size_t region_size) {
+                return CalculateManagementOverheadSize(region_size, MemoryBlockPageShifts, NumMemoryBlockPageShifts);
             }
     };
 

libraries/libmesosphere/include/mesosphere/kern_k_page_table_base.hpp

@@ -205,43 +205,43 @@ namespace ams::kern {
             bool IsLinearMappedPhysicalAddress(KPhysicalAddress phys_addr) {
                 MESOSPHERE_ASSERT(this->IsLockedByCurrentThread());
 
-                return KMemoryLayout::IsLinearMappedPhysicalAddress(std::addressof(this->cached_physical_linear_region), phys_addr, this->cached_physical_linear_region);
+                return KMemoryLayout::IsLinearMappedPhysicalAddress(this->cached_physical_linear_region, phys_addr);
             }
 
             bool IsLinearMappedPhysicalAddress(KPhysicalAddress phys_addr, size_t size) {
                 MESOSPHERE_ASSERT(this->IsLockedByCurrentThread());
 
-                return KMemoryLayout::IsLinearMappedPhysicalAddress(std::addressof(this->cached_physical_linear_region), phys_addr, size, this->cached_physical_linear_region);
+                return KMemoryLayout::IsLinearMappedPhysicalAddress(this->cached_physical_linear_region, phys_addr, size);
             }
 
             bool IsHeapPhysicalAddress(KPhysicalAddress phys_addr) {
                 MESOSPHERE_ASSERT(this->IsLockedByCurrentThread());
 
-                return KMemoryLayout::IsHeapPhysicalAddress(std::addressof(this->cached_physical_heap_region), phys_addr, this->cached_physical_heap_region);
-            }
-
-            bool IsHeapPhysicalAddressForFinalize(KPhysicalAddress phys_addr) {
-                MESOSPHERE_ASSERT(!this->IsLockedByCurrentThread());
-
-                return KMemoryLayout::IsHeapPhysicalAddress(std::addressof(this->cached_physical_heap_region), phys_addr, this->cached_physical_heap_region);
+                return KMemoryLayout::IsHeapPhysicalAddress(this->cached_physical_heap_region, phys_addr);
             }
 
             bool IsHeapPhysicalAddress(KPhysicalAddress phys_addr, size_t size) {
                 MESOSPHERE_ASSERT(this->IsLockedByCurrentThread());
 
-                return KMemoryLayout::IsHeapPhysicalAddress(std::addressof(this->cached_physical_heap_region), phys_addr, size, this->cached_physical_heap_region);
+                return KMemoryLayout::IsHeapPhysicalAddress(this->cached_physical_heap_region, phys_addr, size);
+            }
+
+            bool IsHeapPhysicalAddressForFinalize(KPhysicalAddress phys_addr) {
+                MESOSPHERE_ASSERT(!this->IsLockedByCurrentThread());
+
+                return KMemoryLayout::IsHeapPhysicalAddress(this->cached_physical_heap_region, phys_addr);
             }
 
             bool IsHeapVirtualAddress(KVirtualAddress virt_addr) {
                 MESOSPHERE_ASSERT(this->IsLockedByCurrentThread());
 
-                return KMemoryLayout::IsHeapVirtualAddress(std::addressof(this->cached_virtual_heap_region), virt_addr, this->cached_virtual_heap_region);
+                return KMemoryLayout::IsHeapVirtualAddress(this->cached_virtual_heap_region, virt_addr);
             }
 
             bool IsHeapVirtualAddress(KVirtualAddress virt_addr, size_t size) {
                 MESOSPHERE_ASSERT(this->IsLockedByCurrentThread());
 
-                return KMemoryLayout::IsHeapVirtualAddress(std::addressof(this->cached_virtual_heap_region), virt_addr, size, this->cached_virtual_heap_region);
+                return KMemoryLayout::IsHeapVirtualAddress(this->cached_virtual_heap_region, virt_addr, size);
             }
 
             bool ContainsPages(KProcessAddress addr, size_t num_pages) const {

libraries/libmesosphere/include/mesosphere/kern_k_port.hpp

@@ -44,7 +44,7 @@ namespace ams::kern {
             constexpr KPort() : server(), client(), name(), state(State::Invalid), is_light() { /* ... */ }
             virtual ~KPort() { /* ... */ }
 
-            static void PostDestroy(uintptr_t arg) { /* ... */ }
+            static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
 
             void Initialize(s32 max_sessions, bool is_light, uintptr_t name);
             void OnClientClosed();

libraries/libmesosphere/include/mesosphere/kern_k_process.hpp

@@ -123,6 +123,20 @@ namespace ams::kern {
 
             void StartTermination();
             void FinishTermination();
+
+            void PinThread(s32 core_id, KThread *thread) {
+                MESOSPHERE_ASSERT(0 <= core_id && core_id < static_cast<s32>(cpu::NumCores));
+                MESOSPHERE_ASSERT(thread != nullptr);
+                MESOSPHERE_ASSERT(this->pinned_threads[core_id] == nullptr);
+                this->pinned_threads[core_id] = thread;
+            }
+
+            void UnpinThread(s32 core_id, KThread *thread) {
+                MESOSPHERE_ASSERT(0 <= core_id && core_id < static_cast<s32>(cpu::NumCores));
+                MESOSPHERE_ASSERT(thread != nullptr);
+                MESOSPHERE_ASSERT(this->pinned_threads[core_id] == thread);
+                this->pinned_threads[core_id] = nullptr;
+            }
         public:
             KProcess() { /* ... */ }
             virtual ~KProcess() { /* ... */ }
@@ -207,20 +221,6 @@ namespace ams::kern {
                 return this->pinned_threads[core_id];
             }
 
-            void PinThread(s32 core_id, KThread *thread) {
-                MESOSPHERE_ASSERT(0 <= core_id && core_id < static_cast<s32>(cpu::NumCores));
-                MESOSPHERE_ASSERT(thread != nullptr);
-                MESOSPHERE_ASSERT(this->pinned_threads[core_id] == nullptr);
-                this->pinned_threads[core_id] = thread;
-            }
-
-            void UnpinThread(s32 core_id, KThread *thread) {
-                MESOSPHERE_ASSERT(0 <= core_id && core_id < static_cast<s32>(cpu::NumCores));
-                MESOSPHERE_ASSERT(thread != nullptr);
-                MESOSPHERE_ASSERT(this->pinned_threads[core_id] == thread);
-                this->pinned_threads[core_id] = nullptr;
-            }
-
             void CopySvcPermissionsTo(KThread::StackParameters &sp) {
                 this->capabilities.CopySvcPermissionsTo(sp);
             }
@@ -327,6 +327,7 @@ namespace ams::kern {
             Result SetActivity(ams::svc::ProcessActivity activity);
 
             void PinCurrentThread();
+            void UnpinCurrentThread();
 
             Result SignalToAddress(KProcessAddress address) {
                 return this->cond_var.SignalToAddress(address);
@@ -358,6 +359,8 @@ namespace ams::kern {
             static Result GetProcessList(s32 *out_num_processes, ams::kern::svc::KUserPointer<u64 *> out_process_ids, s32 max_out_count);
 
             static void Switch(KProcess *cur_process, KProcess *next_process) {
+                MESOSPHERE_UNUSED(cur_process);
+
                 /* Set the current process pointer. */
                 SetCurrentProcess(next_process);
 
@@ -372,11 +375,11 @@ namespace ams::kern {
             /* Overridden parent functions. */
             virtual bool IsInitialized() const override { return this->is_initialized; }
 
-            static void PostDestroy(uintptr_t arg) { /* ... */ }
+            static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
 
             virtual void Finalize() override;
 
-            virtual u64 GetId() const override { return this->GetProcessId(); }
+            virtual u64 GetId() const override final { return this->GetProcessId(); }
 
             virtual bool IsSignaled() const override {
                 MESOSPHERE_ASSERT_THIS();

libraries/libmesosphere/include/mesosphere/kern_k_resource_limit.hpp

@@ -35,7 +35,7 @@ namespace ams::kern {
             constexpr ALWAYS_INLINE KResourceLimit() : limit_values(), current_values(), current_hints(), lock(), waiter_count(), cond_var() { /* ... */ }
             virtual ~KResourceLimit() { /* ... */ }
 
-            static ALWAYS_INLINE void PostDestroy(uintptr_t arg) { /* ... */ }
+            static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
 
             void Initialize();
             virtual void Finalize() override;

libraries/libmesosphere/include/mesosphere/kern_k_scheduler.hpp

@@ -76,7 +76,7 @@ namespace ams::kern {
             }
 
             ALWAYS_INLINE void RequestScheduleOnInterrupt() {
-                SetSchedulerUpdateNeeded();
+                this->state.needs_scheduling = true;
 
                 if (CanSchedule()) {
                     this->ScheduleOnInterrupt();
@@ -100,11 +100,7 @@ namespace ams::kern {
             }
         private:
             /* Static private API. */
-            static ALWAYS_INLINE bool IsSchedulerUpdateNeeded() { return s_scheduler_update_needed; }
-            static ALWAYS_INLINE void SetSchedulerUpdateNeeded() { s_scheduler_update_needed = true; }
-            static ALWAYS_INLINE void ClearSchedulerUpdateNeeded() { s_scheduler_update_needed = false; }
             static ALWAYS_INLINE KSchedulerPriorityQueue &GetPriorityQueue() { return s_priority_queue; }
-
             static NOINLINE u64 UpdateHighestPriorityThreadsImpl();
 
             static NOINLINE void InterruptTaskThreadToRunnable();
@@ -113,6 +109,10 @@ namespace ams::kern {
             static ALWAYS_INLINE bool CanSchedule() { return GetCurrentThread().GetDisableDispatchCount() == 0; }
             static ALWAYS_INLINE bool IsSchedulerLockedByCurrentThread() { return s_scheduler_lock.IsLockedByCurrentThread(); }
 
+            static ALWAYS_INLINE bool IsSchedulerUpdateNeeded() { return s_scheduler_update_needed; }
+            static ALWAYS_INLINE void SetSchedulerUpdateNeeded() { s_scheduler_update_needed = true; }
+            static ALWAYS_INLINE void ClearSchedulerUpdateNeeded() { s_scheduler_update_needed = false; }
+
             static ALWAYS_INLINE void DisableScheduling() {
                 MESOSPHERE_ASSERT(GetCurrentThread().GetDisableDispatchCount() >= 0);
                 GetCurrentThread().DisableDispatch();
@@ -139,9 +139,6 @@ namespace ams::kern {
 
             static NOINLINE void ClearPreviousThread(KThread *thread);
 
-            static NOINLINE void PinCurrentThread(KProcess *cur_process);
-            static NOINLINE void UnpinCurrentThread(KProcess *cur_process);
-
             static NOINLINE void OnThreadStateChanged(KThread *thread, KThread::ThreadState old_state);
             static NOINLINE void OnThreadPriorityChanged(KThread *thread, s32 old_priority);
             static NOINLINE void OnThreadAffinityMaskChanged(KThread *thread, const KAffinityMask &old_affinity, s32 old_core);

libraries/libmesosphere/include/mesosphere/kern_k_session_request.hpp

@@ -174,7 +174,7 @@ namespace ams::kern {
                 }
             }
 
-            static void PostDestroy(uintptr_t arg) { /* ... */ }
+            static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
 
             constexpr KThread *GetThread() const { return this->thread; }
             constexpr KWritableEvent *GetEvent() const { return this->event; }

libraries/libmesosphere/include/mesosphere/kern_k_shared_memory.hpp

@@ -47,7 +47,7 @@ namespace ams::kern {
             virtual void Finalize() override;
 
             virtual bool IsInitialized() const override { return this->is_initialized; }
-            static void PostDestroy(uintptr_t arg) { /* ... */ }
+            static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
 
             Result Map(KProcessPageTable *table, KProcessAddress address, size_t size, KProcess *process, ams::svc::MemoryPermission map_perm);
             Result Unmap(KProcessPageTable *table, KProcessAddress address, size_t size, KProcess *process);

libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp

@@ -472,6 +472,7 @@ namespace ams::kern {
             void AddCpuTime(s32 core_id, s64 amount) {
                 this->cpu_time += amount;
                 /* TODO: Debug kernels track per-core tick counts. Should we? */
+                MESOSPHERE_UNUSED(core_id);
             }
 
             s64 GetCpuTime() const { return this->cpu_time; }
@@ -523,7 +524,7 @@ namespace ams::kern {
 
         public:
             /* Overridden parent functions. */
-            virtual u64 GetId() const override { return this->GetThreadId(); }
+            virtual u64 GetId() const override final { return this->GetThreadId(); }
 
             virtual bool IsInitialized() const override { return this->initialized; }
             virtual uintptr_t GetPostDestroyArgument() const override { return reinterpret_cast<uintptr_t>(this->parent) | (this->resource_limit_release_hint ? 1 : 0); }

libraries/libmesosphere/include/mesosphere/kern_k_trace.hpp

@@ -0,0 +1,114 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_k_spin_lock.hpp>
+
+namespace ams::kern {
+
+    #if defined(MESOSPHERE_BUILD_FOR_TRACING)
+        constexpr inline bool IsKTraceEnabled = true;
+    #else
+        constexpr inline bool IsKTraceEnabled = false;
+    #endif
+
+    constexpr inline size_t KTraceBufferSize = IsKTraceEnabled ? 16_MB : 0;
+
+    static_assert(IsKTraceEnabled || !IsKTraceEnabled);
+
+    class KTrace {
+        public:
+            enum Type {
+                Type_ThreadSwitch   =  1,
+
+                Type_SvcEntry0      =  3,
+                Type_SvcEntry1      =  4,
+                Type_SvcExit0       =  5,
+                Type_SvcExit1       =  6,
+                Type_Interrupt      =  7,
+
+                Type_ScheduleUpdate = 11,
+
+                Type_CoreMigration  = 14,
+            };
+        private:
+            static bool s_is_active;
+        public:
+            static void Initialize(KVirtualAddress address, size_t size);
+            static void Start();
+            static void Stop();
+
+            static void PushRecord(u8 type, u64 param0 = 0, u64 param1 = 0, u64 param2 = 0, u64 param3 = 0, u64 param4 = 0, u64 param5 = 0);
+
+            static ALWAYS_INLINE bool IsActive() { return s_is_active; }
+    };
+
+}
+
+#define MESOSPHERE_KTRACE_RESUME()                    \
+    ({                                                \
+        if constexpr (::ams::kern::IsKTraceEnabled) { \
+            ::ams::kern::KTrace::Start();             \
+        }                                             \
+    })
+
+#define MESOSPHERE_KTRACE_PAUSE()                     \
+    ({                                                \
+        if constexpr (::ams::kern::IsKTraceEnabled) { \
+            ::ams::kern::KTrace::Stop();              \
+        }                                             \
+    })
+
+#define MESOSPHERE_KTRACE_PUSH_RECORD(TYPE, ...)                       \
+    ({                                                                 \
+        if constexpr (::ams::kern::IsKTraceEnabled) {                  \
+            if (::ams::kern::KTrace::IsActive()) {                     \
+                ::ams::kern::KTrace::PushRecord(TYPE, ## __VA_ARGS__); \
+            }                                                          \
+        }                                                              \
+    })
+
+#define MESOSPHERE_KTRACE_THREAD_SWITCH(NEXT) \
+    MESOSPHERE_KTRACE_PUSH_RECORD(::ams::kern::KTrace::Type_ThreadSwitch, (NEXT)->GetId())
+
+#define MESOSPHERE_KTRACE_SVC_ENTRY(SVC_ID, PARAM0, PARAM1, PARAM2, PARAM3, PARAM4, PARAM5, PARAM6, PARAM7)                           \
+    ({                                                                                                                                \
+        if constexpr (::ams::kern::IsKTraceEnabled) {                                                                                 \
+            if (::ams::kern::KTrace::IsActive()) {                                                                                    \
+                ::ams::kern::KTrace::PushRecord(::ams::kern::KTrace::Type_SvcEntry0, SVC_ID, PARAM0, PARAM1, PARAM2, PARAM3, PARAM4); \
+                ::ams::kern::KTrace::PushRecord(::ams::kern::KTrace::Type_SvcEntry1, PARAM5, PARAM6, PARAM7);                         \
+            }                                                                                                                         \
+        }                                                                                                                             \
+    })
+
+#define MESOSPHERE_KTRACE_SVC_EXIT(SVC_ID, PARAM0, PARAM1, PARAM2, PARAM3, PARAM4, PARAM5, PARAM6, PARAM7)                           \
+    ({                                                                                                                               \
+        if constexpr (::ams::kern::IsKTraceEnabled) {                                                                                \
+            if (::ams::kern::KTrace::IsActive()) {                                                                                   \
+                ::ams::kern::KTrace::PushRecord(::ams::kern::KTrace::Type_SvcExit0, SVC_ID, PARAM0, PARAM1, PARAM2, PARAM3, PARAM4); \
+                ::ams::kern::KTrace::PushRecord(::ams::kern::KTrace::Type_SvcExit1, PARAM5, PARAM6, PARAM7);                         \
+            }                                                                                                                        \
+        }                                                                                                                            \
+    })
+
+#define MESOSPHERE_KTRACE_INTERRUPT(ID) \
+    MESOSPHERE_KTRACE_PUSH_RECORD(::ams::kern::KTrace::Type_Interrupt, ID)
+
+#define MESOSPHERE_KTRACE_SCHEDULE_UPDATE(CORE, PREV, NEXT) \
+    MESOSPHERE_KTRACE_PUSH_RECORD(::ams::kern::KTrace::Type_ScheduleUpdate, CORE, (PREV)->GetId(), (NEXT)->GetId())
+
+#define MESOSPHERE_KTRACE_CORE_MIGRATION(THREAD_ID, PREV, NEXT, REASON) \
+    MESOSPHERE_KTRACE_PUSH_RECORD(::ams::kern::KTrace::Type_CoreMigration,  THREAD_ID, PREV, NEXT, REASON)

libraries/libmesosphere/include/mesosphere/kern_k_typed_address.hpp

@@ -31,11 +31,11 @@ namespace ams::kern {
             template<typename U>
             constexpr ALWAYS_INLINE explicit KTypedAddress(U *ptr) : address(reinterpret_cast<uintptr_t>(ptr)) { /* ... */ }
 
+            /* Copy constructor. */
+            constexpr ALWAYS_INLINE KTypedAddress(const KTypedAddress &rhs) = default;
+
             /* Assignment operator. */
-            constexpr ALWAYS_INLINE KTypedAddress operator=(KTypedAddress rhs) {
-                this->address = rhs.address;
-                return *this;
-            }
+            constexpr ALWAYS_INLINE KTypedAddress &operator=(const KTypedAddress &rhs) = default;
 
             /* Arithmetic operators. */
             template<typename I>
@@ -180,6 +180,9 @@ namespace ams::kern {
 
 #endif
 
+    template<typename T>
+    concept IsKTypedAddress = std::same_as<T, KPhysicalAddress> || std::same_as<T, KVirtualAddress> || std::same_as<T, KProcessAddress>;
+
     template<typename T>
     constexpr inline T Null = [] {
         if constexpr (std::is_same<T, uintptr_t>::value) {
@@ -197,6 +200,26 @@ namespace ams::kern {
     static_assert(sizeof(KVirtualAddress)  == sizeof(uintptr_t));
     static_assert(sizeof(KProcessAddress)  == sizeof(uintptr_t));
 
+    static_assert(std::is_trivially_copyable<KPhysicalAddress>::value);
+    static_assert(std::is_trivially_copyable<KVirtualAddress>::value);
+    static_assert(std::is_trivially_copyable<KProcessAddress>::value);
+
+    static_assert(std::is_trivially_copy_constructible<KPhysicalAddress>::value);
+    static_assert(std::is_trivially_copy_constructible<KVirtualAddress>::value);
+    static_assert(std::is_trivially_copy_constructible<KProcessAddress>::value);
+
+    static_assert(std::is_trivially_move_constructible<KPhysicalAddress>::value);
+    static_assert(std::is_trivially_move_constructible<KVirtualAddress>::value);
+    static_assert(std::is_trivially_move_constructible<KProcessAddress>::value);
+
+    static_assert(std::is_trivially_copy_assignable<KPhysicalAddress>::value);
+    static_assert(std::is_trivially_copy_assignable<KVirtualAddress>::value);
+    static_assert(std::is_trivially_copy_assignable<KProcessAddress>::value);
+
+    static_assert(std::is_trivially_move_assignable<KPhysicalAddress>::value);
+    static_assert(std::is_trivially_move_assignable<KVirtualAddress>::value);
+    static_assert(std::is_trivially_move_assignable<KProcessAddress>::value);
+
     static_assert(std::is_trivially_destructible<KPhysicalAddress>::value);
     static_assert(std::is_trivially_destructible<KVirtualAddress>::value);
     static_assert(std::is_trivially_destructible<KProcessAddress>::value);
@@ -206,6 +229,10 @@ namespace ams::kern {
     static_assert(Null<KVirtualAddress>  == Null<uintptr_t>);
     static_assert(Null<KProcessAddress>  == Null<uintptr_t>);
 
+    /* Constructor/assignment validations. */
+    static_assert([]{ const KPhysicalAddress a(5); KPhysicalAddress b(a); return b; }() == KPhysicalAddress(5));
+    static_assert([]{ const KPhysicalAddress a(5); KPhysicalAddress b(10); b = a; return b; }() == KPhysicalAddress(5));
+
     /* Arithmetic validations. */
     static_assert(KPhysicalAddress(10) + 5 == KPhysicalAddress(15));
     static_assert(KPhysicalAddress(10) - 5 == KPhysicalAddress(5));

libraries/libmesosphere/include/mesosphere/kern_k_writable_event.hpp

@@ -32,7 +32,7 @@ namespace ams::kern {
 
             virtual void Destroy() override;
 
-            static void PostDestroy(uintptr_t arg) { /* ... */ }
+            static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
 
             void Initialize(KEvent *p);
             Result Signal();

libraries/libmesosphere/include/mesosphere/kern_panic.hpp

@@ -19,17 +19,12 @@
 
 namespace ams::kern {
 
-    template<typename... ArgTypes>
-    ALWAYS_INLINE void UnusedImpl(ArgTypes &&... args) {
-        (static_cast<void>(args), ...);
-    }
-
     NORETURN NOINLINE void Panic(const char *file, int line, const char *format, ...) __attribute__((format(printf, 3, 4)));
     NORETURN NOINLINE void Panic();
 
 }
 
-#define MESOSPHERE_UNUSED(...) ::ams::kern::UnusedImpl(__VA_ARGS__)
+#define MESOSPHERE_UNUSED(...) AMS_UNUSED(__VA_ARGS__)
 
 #ifdef MESOSPHERE_ENABLE_DEBUG_PRINT
 #define MESOSPHERE_PANIC(...) do { ::ams::kern::Panic(__FILE__, __LINE__, ## __VA_ARGS__); } while(0)
@@ -69,7 +64,7 @@ namespace ams::kern {
 #define MESOSPHERE_UNIMPLEMENTED() MESOSPHERE_PANIC("%s: Unimplemented\n", __PRETTY_FUNCTION__)
 
 #define MESOSPHERE_ABORT() MESOSPHERE_PANIC("Abort()\n");
-#define MESOSPHERE_INIT_ABORT() do { /* ... */ } while (true)
+#define MESOSPHERE_INIT_ABORT() AMS_INFINITE_LOOP()
 
 #define MESOSPHERE_ABORT_UNLESS(expr)               \
     ({                                              \

libraries/libmesosphere/include/mesosphere/kern_select_device_page_table.hpp

@@ -24,5 +24,10 @@
     }
 
 #else
-    #error "Unknown board for KDevicePageTable"
+    #include <mesosphere/board/generic/kern_k_device_page_table.hpp>
+
+    namespace ams::kern {
+        using ams::kern::board::generic::KDevicePageTable;
+    }
+
 #endif

libraries/libmesosphere/include/mesosphere/kern_select_interrupt_controller.hpp

@@ -24,6 +24,13 @@
         using ams::kern::arch::arm64::KInterruptController;
     }
 
+#elif defined(ATMOSPHERE_ARCH_ARM)
+
+    #include <mesosphere/arch/arm/kern_k_interrupt_controller.hpp>
+    namespace ams::kern {
+        using ams::kern::arch::arm::KInterruptController;
+    }
+
 #else
 
     #error "Unknown architecture for KInterruptController"

libraries/libmesosphere/source/arch/arm/kern_generic_interrupt_controller.inc

@@ -15,7 +15,7 @@
  */
 #include <mesosphere.hpp>
 
-namespace ams::kern::arch::arm64 {
+namespace ams::kern::arch::arm {
 
     void KInterruptController::SetupInterruptLines(s32 core_id) const {
         const size_t ITLines = (core_id == 0) ? 32 * ((this->gicd->typer & 0x1F) + 1) : NumLocalInterrupts;
@@ -39,8 +39,8 @@ namespace ams::kern::arch::arm64 {
 
     void KInterruptController::Initialize(s32 core_id) {
         /* Setup pointers to ARM mmio. */
-        this->gicd = GetPointer<volatile  GicDistributor>(KMemoryLayout::GetInterruptDistributorAddress());
-        this->gicc = GetPointer<volatile GicCpuInterface>(KMemoryLayout::GetInterruptCpuInterfaceAddress());
+        this->gicd = GetPointer<volatile GicDistributor >(KMemoryLayout::GetDeviceVirtualAddress(KMemoryRegionType_InterruptDistributor));
+        this->gicc = GetPointer<volatile GicCpuInterface>(KMemoryLayout::GetDeviceVirtualAddress(KMemoryRegionType_InterruptCpuInterface));
 
         /* Clear CTLRs. */
         this->gicc->ctlr = 0;

libraries/libmesosphere/source/arch/arm/kern_k_interrupt_controller.board.generic.cpp

@@ -0,0 +1,19 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#include <mesosphere.hpp>
+
+/* Include the common implementation. */
+#include "../arm/kern_generic_interrupt_controller.inc"

libraries/libmesosphere/source/arch/arm64/kern_cpu.cpp

@@ -37,6 +37,7 @@ namespace ams::kern::arch::arm64::cpu {
                 constexpr KThreadTerminationInterruptHandler() : KInterruptHandler() { /* ... */ }
 
                 virtual KInterruptTask *OnInterrupt(s32 interrupt_id) override {
+                    MESOSPHERE_UNUSED(interrupt_id);
                     return nullptr;
                 }
         };
@@ -68,6 +69,8 @@ namespace ams::kern::arch::arm64::cpu {
 
                 /* Nintendo misuses this per their own API, but it's functional. */
                 virtual KInterruptTask *OnInterrupt(s32 interrupt_id) override {
+                    MESOSPHERE_UNUSED(interrupt_id);
+
                     if (this->which < 0) {
                         this->counter = cpu::GetCycleCounter();
                     } else {
@@ -85,7 +88,7 @@ namespace ams::kern::arch::arm64::cpu {
             public:
                 enum class Operation {
                     Idle,
-                    InvalidateInstructionCache,
+                    InstructionMemoryBarrier,
                     StoreDataCache,
                     FlushDataCache,
                 };
@@ -145,49 +148,63 @@ namespace ams::kern::arch::arm64::cpu {
                 }
 
                 virtual KInterruptTask *OnInterrupt(s32 interrupt_id) override {
+                    MESOSPHERE_UNUSED(interrupt_id);
                     this->ProcessOperation();
                     return nullptr;
                 }
 
                 void RequestOperation(Operation op) {
                     KScopedLightLock lk(this->lock);
-                    MESOSPHERE_ABORT_UNLESS(this->operation == Operation::Idle);
-                    /* Send and wait for acknowledgement of request. */
-                    {
-                        KScopedLightLock cv_lk(this->cv_lock);
+
+                    /* Create core masks for us to use. */
+                    constexpr u64 AllCoresMask = (1ul << cpu::NumCores) - 1ul;
+                    const u64 other_cores_mask = AllCoresMask & ~(1ul << GetCurrentCoreId());
+
+                    if ((op == Operation::InstructionMemoryBarrier) || (Kernel::GetState() == Kernel::State::Initializing)) {
+                        /* Check that there's no on-going operation. */
+                        MESOSPHERE_ABORT_UNLESS(this->operation == Operation::Idle);
                         MESOSPHERE_ABORT_UNLESS(this->target_cores == 0);
 
                         /* Set operation. */
                         this->operation = op;
 
-                        /* Create core masks for us to use. */
-                        constexpr u64 AllCoresMask = (1ul << cpu::NumCores) - 1ul;
-                        const u64 other_cores_mask = AllCoresMask & ~(1ul << GetCurrentCoreId());
+                        /* For certain operations, we want to send an interrupt. */
+                        this->target_cores = other_cores_mask;
 
-                        if ((op == Operation::InvalidateInstructionCache) || (Kernel::GetState() == Kernel::State::Initializing)) {
-                            /* For certain operations, we want to send an interrupt. */
-                            this->target_cores = other_cores_mask;
-                            DataSynchronizationBarrier();
-                            const u64 target_mask = this->target_cores;
-                            DataSynchronizationBarrier();
-                            Kernel::GetInterruptManager().SendInterProcessorInterrupt(KInterruptName_CacheOperation, target_mask);
-                            this->ProcessOperation();
-                            while (this->target_cores != 0) {
-                                cpu::Yield();
-                            }
-                        } else {
-                            /* Request all cores. */
-                            this->target_cores = AllCoresMask;
+                        const u64 target_mask = this->target_cores;
+                        DataSynchronizationBarrier();
+                        Kernel::GetInterruptManager().SendInterProcessorInterrupt(KInterruptName_CacheOperation, target_mask);
 
-                            /* Use the condvar. */
-                            this->cv.Broadcast();
-                            while (this->target_cores != 0) {
-                                this->cv.Wait(std::addressof(this->cv_lock));
-                            }
+                        this->ProcessOperation();
+                        while (this->target_cores != 0) {
+                            cpu::Yield();
                         }
+
+                        /* Go idle again. */
+                        this->operation = Operation::Idle;
+                    } else {
+                        /* Lock condvar so that we can send and wait for acknowledgement of request. */
+                        KScopedLightLock cv_lk(this->cv_lock);
+
+                        /* Check that there's no on-going operation. */
+                        MESOSPHERE_ABORT_UNLESS(this->operation == Operation::Idle);
+                        MESOSPHERE_ABORT_UNLESS(this->target_cores == 0);
+
+                        /* Set operation. */
+                        this->operation = op;
+
+                        /* Request all cores. */
+                        this->target_cores = AllCoresMask;
+
+                        /* Use the condvar. */
+                        this->cv.Broadcast();
+                        while (this->target_cores != 0) {
+                            this->cv.Wait(std::addressof(this->cv_lock));
+                        }
+
+                        /* Go idle again. */
+                        this->operation = Operation::Idle;
                     }
-                    /* Go idle again. */
-                    this->operation = Operation::Idle;
                 }
         };
 
@@ -269,7 +286,7 @@ namespace ams::kern::arch::arm64::cpu {
             switch (this->operation) {
                 case Operation::Idle:
                     break;
-                case Operation::InvalidateInstructionCache:
+                case Operation::InstructionMemoryBarrier:
                     InstructionMemoryBarrier();
                     break;
                 case Operation::StoreDataCache:
@@ -281,6 +298,8 @@ namespace ams::kern::arch::arm64::cpu {
                     DataSynchronizationBarrier();
                     break;
             }
+
+            this->target_cores &= ~(1ul << GetCurrentCoreId());
         }
 
         ALWAYS_INLINE void SetEventLocally() {
@@ -323,6 +342,14 @@ namespace ams::kern::arch::arm64::cpu {
             return ResultSuccess();
         }
 
+        ALWAYS_INLINE void InvalidateEntireInstructionCacheLocalImpl() {
+            __asm__ __volatile__("ic iallu" ::: "memory");
+        }
+
+        ALWAYS_INLINE void InvalidateEntireInstructionCacheGlobalImpl() {
+            __asm__ __volatile__("ic ialluis" ::: "memory");
+        }
+
     }
 
     void FlushEntireDataCacheSharedForInit() {
@@ -333,11 +360,16 @@ namespace ams::kern::arch::arm64::cpu {
         return PerformCacheOperationBySetWayLocal<true>(FlushDataCacheLineBySetWayImpl);
     }
 
+    void InvalidateEntireInstructionCacheForInit() {
+        InvalidateEntireInstructionCacheLocalImpl();
+        EnsureInstructionConsistency();
+    }
+
     void StoreEntireCacheForInit() {
         PerformCacheOperationBySetWayLocal<true>(StoreDataCacheLineBySetWayImpl);
         PerformCacheOperationBySetWayShared<true>(StoreDataCacheLineBySetWayImpl);
         DataSynchronizationBarrierInnerShareable();
-        InvalidateEntireInstructionCache();
+        InvalidateEntireInstructionCacheForInit();
     }
 
     void FlushEntireDataCache() {
@@ -391,12 +423,23 @@ namespace ams::kern::arch::arm64::cpu {
 
         R_TRY(InvalidateInstructionCacheRange(start, end));
 
-        /* Request the interrupt helper to invalidate, too. */
-        g_cache_operation_handler.RequestOperation(KCacheHelperInterruptHandler::Operation::InvalidateInstructionCache);
+        /* Request the interrupt helper to perform an instruction memory barrier. */
+        g_cache_operation_handler.RequestOperation(KCacheHelperInterruptHandler::Operation::InstructionMemoryBarrier);
 
         return ResultSuccess();
     }
 
+    void InvalidateEntireInstructionCache() {
+        KScopedCoreMigrationDisable dm;
+
+        /* Invalidate the instruction cache on all cores. */
+        InvalidateEntireInstructionCacheGlobalImpl();
+        EnsureInstructionConsistency();
+
+        /* Request the interrupt helper to perform an instruction memory barrier. */
+        g_cache_operation_handler.RequestOperation(KCacheHelperInterruptHandler::Operation::InstructionMemoryBarrier);
+    }
+
     void InitializeInterruptThreads(s32 core_id) {
         /* Initialize the cache operation handler. */
         g_cache_operation_handler.Initialize(core_id);

libraries/libmesosphere/source/arch/arm64/kern_exception_handlers.cpp

@@ -511,7 +511,7 @@ namespace ams::kern::arch::arm64 {
                     KScopedSchedulerLock lk;
 
                     /* Pin the current thread. */
-                    KScheduler::PinCurrentThread(GetCurrentProcessPointer());
+                    GetCurrentProcess().PinCurrentThread();
 
                     /* Set the interrupt flag for the thread. */
                     GetCurrentThread().SetInterruptFlag();

libraries/libmesosphere/source/arch/arm64/kern_k_hardware_timer.cpp

@@ -24,6 +24,7 @@ namespace ams::kern::arch::arm64 {
                 constexpr KHardwareTimerInterruptTask() : KInterruptTask() { /* ... */ }
 
                 virtual KInterruptTask *OnInterrupt(s32 interrupt_id) override {
+                    MESOSPHERE_UNUSED(interrupt_id);
                     return this;
                 }
 

libraries/libmesosphere/source/arch/arm64/kern_k_interrupt_controller.board.generic.cpp

@@ -0,0 +1,19 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#include <mesosphere.hpp>
+
+/* Include the common implementation. */
+#include "../arm/kern_generic_interrupt_controller.inc"

libraries/libmesosphere/source/arch/arm64/kern_k_interrupt_manager.cpp

@@ -111,6 +111,9 @@ namespace ams::kern::arch::arm64 {
         const u32 raw_irq = this->interrupt_controller.GetIrq();
         const s32 irq = KInterruptController::ConvertRawIrq(raw_irq);
 
+        /* Trace the interrupt. */
+        MESOSPHERE_KTRACE_INTERRUPT(irq);
+
         /* If the IRQ is spurious, we don't need to reschedule. */
         if (irq < 0) {
             return false;
@@ -180,7 +183,7 @@ namespace ams::kern::arch::arm64 {
                 KScopedSchedulerLock sl;
 
                 /* Pin the current thread. */
-                KScheduler::PinCurrentThread(GetCurrentProcessPointer());
+                GetCurrentProcess().PinCurrentThread();
 
                 /* Set the interrupt flag for the thread. */
                 GetCurrentThread().SetInterruptFlag();

libraries/libmesosphere/source/arch/arm64/kern_k_page_table.cpp

@@ -160,6 +160,7 @@ namespace ams::kern::arch::arm64 {
 
     void KPageTable::Initialize(s32 core_id) {
         /* Nothing actually needed here. */
+        MESOSPHERE_UNUSED(core_id);
     }
 
     Result KPageTable::InitializeForKernel(void *table, KVirtualAddress start, KVirtualAddress end) {
@@ -181,7 +182,8 @@ namespace ams::kern::arch::arm64 {
     }
 
     Result KPageTable::InitializeForProcess(u32 id, ams::svc::CreateProcessFlag as_type, bool enable_aslr, bool from_back, KMemoryManager::Pool pool, KProcessAddress code_address, size_t code_size, KMemoryBlockSlabManager *mem_block_slab_manager, KBlockInfoManager *block_info_manager, KPageTableManager *pt_manager) {
-        /* Convert the address space type to a width. */
+        /* The input ID isn't actually used. */
+        MESOSPHERE_UNUSED(id);
 
         /* Get an ASID */
         this->asid = g_asid_manager.Reserve();
@@ -364,6 +366,9 @@ namespace ams::kern::arch::arm64 {
         MESOSPHERE_ASSERT(util::IsAligned(GetInteger(phys_addr), L1BlockSize));
         MESOSPHERE_ASSERT(util::IsAligned(num_pages * PageSize,  L1BlockSize));
 
+        /* Allocation is never needed for L1 block mapping. */
+        MESOSPHERE_UNUSED(page_list, reuse_ll);
+
         auto &impl = this->GetImpl();
 
         /* Iterate, mapping each block. */

libraries/libmesosphere/source/arch/arm64/kern_k_thread_context.cpp

@@ -281,6 +281,7 @@ namespace ams::kern::arch::arm64 {
     }
 
     void KThreadContext::OnThreadTerminating(const KThread *thread) {
+        MESOSPHERE_UNUSED(thread);
         /* ... */
     }
 

libraries/libmesosphere/source/arch/arm64/svc/kern_svc_handlers.cpp

@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#include <mesosphere.hpp>
+
+namespace ams::kern::svc {
+
+    void TraceSvcEntry(const u64 *data) {
+        MESOSPHERE_KTRACE_SVC_ENTRY(GetCurrentThread().GetSvcId(), data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7]);
+    }
+
+    void TraceSvcExit(const u64 *data) {
+        MESOSPHERE_KTRACE_SVC_EXIT(GetCurrentThread().GetSvcId(), data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7]);
+    }
+
+}

libraries/libmesosphere/source/arch/arm64/svc/kern_svc_handlers_asm.s

@@ -13,6 +13,7 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
+#include <mesosphere/kern_build_config.hpp>
 
 /* ams::kern::arch::arm64::SvcHandler64() */
 .section    .text._ZN3ams4kern4arch5arm6412SvcHandler64Ev, "ax", %progbits
@@ -83,6 +84,24 @@ _ZN3ams4kern4arch5arm6412SvcHandler64Ev:
     strb    w10, [sp, #(0x120 + 0x12)]
     strb    w8,  [sp, #(0x120 + 0x11)]
 
+    /* If we should, trace the svc entry. */
+#if defined(MESOSPHERE_BUILD_FOR_TRACING)
+    sub     sp, sp, #0x50
+    stp     x0,  x1,  [sp, #(8 *  0)]
+    stp     x2,  x3,  [sp, #(8 *  2)]
+    stp     x4,  x5,  [sp, #(8 *  4)]
+    stp     x6,  x7,  [sp, #(8 *  6)]
+    str     x11,      [sp, #(8 *  8)]
+    mov     x0, sp
+    bl      _ZN3ams4kern3svc13TraceSvcEntryEPKm
+    ldp     x0,  x1,  [sp, #(8 *  0)]
+    ldp     x2,  x3,  [sp, #(8 *  2)]
+    ldp     x4,  x5,  [sp, #(8 *  4)]
+    ldp     x6,  x7,  [sp, #(8 *  6)]
+    ldr     x11,      [sp, #(8 *  8)]
+    add     sp, sp, #0x50
+#endif
+
     /* Invoke the SVC handler. */
     msr     daifclr, #2
     blr     x11
@@ -163,6 +182,22 @@ _ZN3ams4kern4arch5arm6412SvcHandler64Ev:
     /* Clear our in-SVC note. */
     strb    wzr, [sp, #(0x120 + 0x12)]
 
+    /* If we should, trace the svc exit. */
+#if defined(MESOSPHERE_BUILD_FOR_TRACING)
+    sub     sp, sp, #0x40
+    stp     x0,  x1,  [sp, #(8 *  0)]
+    stp     x2,  x3,  [sp, #(8 *  2)]
+    stp     x4,  x5,  [sp, #(8 *  4)]
+    stp     x6,  x7,  [sp, #(8 *  6)]
+    mov     x0, sp
+    bl      _ZN3ams4kern3svc12TraceSvcExitEPKm
+    ldp     x0,  x1,  [sp, #(8 *  0)]
+    ldp     x2,  x3,  [sp, #(8 *  2)]
+    ldp     x4,  x5,  [sp, #(8 *  4)]
+    ldp     x6,  x7,  [sp, #(8 *  6)]
+    add     sp, sp, #0x40
+#endif
+
     /* Restore registers. */
     ldp     x30, x8,  [sp, #(8 * 30)]
     ldp     x9,  x10, [sp, #(8 * 32)]
@@ -259,6 +294,24 @@ _ZN3ams4kern4arch5arm6412SvcHandler32Ev:
     strb    w15, [sp, #(0x120 + 0x12)]
     strb    w16, [sp, #(0x120 + 0x11)]
 
+    /* If we should, trace the svc entry. */
+#if defined(MESOSPHERE_BUILD_FOR_TRACING)
+    sub     sp, sp, #0x50
+    stp     x0,  x1,  [sp, #(8 *  0)]
+    stp     x2,  x3,  [sp, #(8 *  2)]
+    stp     x4,  x5,  [sp, #(8 *  4)]
+    stp     x6,  x7,  [sp, #(8 *  6)]
+    str     x19,      [sp, #(8 *  8)]
+    mov     x0, sp
+    bl      _ZN3ams4kern3svc13TraceSvcEntryEPKm
+    ldp     x0,  x1,  [sp, #(8 *  0)]
+    ldp     x2,  x3,  [sp, #(8 *  2)]
+    ldp     x4,  x5,  [sp, #(8 *  4)]
+    ldp     x6,  x7,  [sp, #(8 *  6)]
+    ldr     x19,      [sp, #(8 *  8)]
+    add     sp, sp, #0x50
+#endif
+
     /* Invoke the SVC handler. */
     msr     daifclr, #2
     blr     x19
@@ -327,6 +380,22 @@ _ZN3ams4kern4arch5arm6412SvcHandler32Ev:
     /* Clear our in-SVC note. */
     strb    wzr, [sp, #(0x120 + 0x12)]
 
+    /* If we should, trace the svc exit. */
+#if defined(MESOSPHERE_BUILD_FOR_TRACING)
+    sub     sp, sp, #0x40
+    stp     x0,  x1,  [sp, #(8 *  0)]
+    stp     x2,  x3,  [sp, #(8 *  2)]
+    stp     x4,  x5,  [sp, #(8 *  4)]
+    stp     x6,  x7,  [sp, #(8 *  6)]
+    mov     x0, sp
+    bl      _ZN3ams4kern3svc12TraceSvcExitEPKm
+    ldp     x0,  x1,  [sp, #(8 *  0)]
+    ldp     x2,  x3,  [sp, #(8 *  2)]
+    ldp     x4,  x5,  [sp, #(8 *  4)]
+    ldp     x6,  x7,  [sp, #(8 *  6)]
+    add     sp, sp, #0x40
+#endif
+
     /* Restore registers. */
     ldp     x8,  x9,  [sp, #(8 *  8)]
     ldp     x10, x11, [sp, #(8 * 10)]

libraries/libmesosphere/source/arch/arm64/svc/kern_svc_tables.cpp

@@ -147,6 +147,13 @@ namespace ams::kern::svc {
                         if (table_64_from_32) { ::ams::kern::svc::PatchSvcTableEntry(table_64_from_32, svc::SvcId_QueryIoMapping, LegacyQueryIoMapping::Call64From32); }
                     }
 
+                    /* 6.0.0 broke the ABI for GetFutureThreadInfo, and renamed it to GetDebugFutureThreadInfo. */
+                    if (target_fw < TargetFirmware_6_0_0) {
+                        static_assert(svc::SvcId_GetDebugFutureThreadInfo == svc::SvcId_LegacyGetFutureThreadInfo);
+                        if (table_64)         { ::ams::kern::svc::PatchSvcTableEntry(table_64,         svc::SvcId_GetDebugFutureThreadInfo, LegacyGetFutureThreadInfo::Call64); }
+                        if (table_64_from_32) { ::ams::kern::svc::PatchSvcTableEntry(table_64_from_32, svc::SvcId_GetDebugFutureThreadInfo, LegacyGetFutureThreadInfo::Call64From32); }
+                    }
+
                     /* 3.0.0 broke the ABI for ContinueDebugEvent. */
                     if (target_fw < TargetFirmware_3_0_0) {
                         if (table_64)         { ::ams::kern::svc::PatchSvcTableEntry(table_64,         svc::SvcId_ContinueDebugEvent, LegacyContinueDebugEvent::Call64); }

libraries/libmesosphere/source/board/nintendo/nx/kern_atomics_registers.hpp

@@ -0,0 +1,21 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+
+#define ATOMICS_AP0_TRIGGER	   0x000
+#define ATOMICS_AP0_RESULT(id) (0xc00 + id * 4)
+
+#define TRIGGER_CMD_GET 4

libraries/libmesosphere/source/board/nintendo/nx/kern_bpmp_api.hpp

@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere.hpp>
+
+/* Message Flags */
+#define BPMP_MSG_DO_ACK            (1 << 0)
+#define BPMP_MSG_RING_DOORBELL  (1 << 1)
+
+/* Messages */
+#define MRQ_PING            0
+#define MRQ_ENABLE_SUSPEND  17
+#define MRQ_CPU_PMIC_SELECT 28
+
+/* BPMP Power states. */
+#define TEGRA_BPMP_PM_CC1 9
+#define TEGRA_BPMP_PM_CC4 12
+#define TEGRA_BPMP_PM_CC6 14
+#define TEGRA_BPMP_PM_CC7 15
+#define TEGRA_BPMP_PM_SC1 17
+#define TEGRA_BPMP_PM_SC2 18
+#define TEGRA_BPMP_PM_SC3 19
+#define TEGRA_BPMP_PM_SC4 20
+#define TEGRA_BPMP_PM_SC7 23
+
+/* Channel states. */
+#define CH_MASK(ch) (0x3u << ((ch) * 2))
+#define SL_SIGL(ch) (0x0u << ((ch) * 2))
+#define SL_QUED(ch) (0x1u << ((ch) * 2))
+#define MA_FREE(ch) (0x2u << ((ch) * 2))
+#define MA_ACKD(ch) (0x3u << ((ch) * 2))
+
+constexpr inline int MessageSize = 0x80;
+constexpr inline int MessageDataSizeMax = 0x78;
+
+struct MailboxData {
+    s32 code;
+    s32 flags;
+    u8 data[MessageDataSizeMax];
+};
+
+static_assert(ams::util::is_pod<MailboxData>::value);
+static_assert(sizeof(MailboxData) == MessageSize);
+
+struct ChannelData {
+    MailboxData *ib;
+    MailboxData *ob;
+};
+
+

libraries/libmesosphere/source/board/nintendo/nx/kern_clkrst_registers.hpp

@@ -0,0 +1,20 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+
+#define CLK_RST_CONTROLLER_CLK_SOURCE_CSITE 0x1D4
+#define CLK_RST_CONTROLLER_RST_DEV_L_SET    0x300
+#define CLK_RST_CONTROLLER_RST_DEV_L_CLR    0x304

libraries/libmesosphere/source/board/nintendo/nx/kern_evp_registers.hpp

@@ -0,0 +1,18 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+
+#define EVP_COP_RESET_VECTOR 0x200

libraries/libmesosphere/source/board/nintendo/nx/kern_flow_registers.hpp

@@ -0,0 +1,21 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+
+#define FLOW_CTLR_HALT_COP_EVENTS       0x004
+#define FLOW_CTLR_CC4_HVC_CONTROL       0x060
+#define FLOW_CTLR_CC4_RETENTION_CONTROL 0x064
+#define FLOW_CTLR_CC4_HVC_RETRY         0x08C

libraries/libmesosphere/source/board/nintendo/nx/kern_ictlr_registers.hpp

@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+
+#define ICTLR_REG_BASE(irq) ((((irq) - 32) >> 5) * 0x100)
+#define ICTLR_FIR_SET(irq)  (ICTLR_REG_BASE(irq) + 0x18)
+#define ICTLR_FIR_CLR(irq)  (ICTLR_REG_BASE(irq) + 0x1c)
+#define FIR_BIT(irq)        (1 << ((irq) & 0x1f))
+
+#define INT_GIC_BASE            (0)
+#define INT_PRI_BASE            (INT_GIC_BASE + 32)
+#define INT_SHR_SEM_OUTBOX_IBF  (INT_PRI_BASE + 6)

libraries/libmesosphere/source/board/nintendo/nx/kern_k_device_page_table.cpp

@@ -16,6 +16,10 @@
 #include <mesosphere.hpp>
 #include "kern_mc_registers.hpp"
 
+#if defined(MESOSPHERE_BUILD_FOR_DEBUGGING) || defined(MESOSPHERE_BUILD_FOR_AUDITING)
+#define MESOSPHERE_ENABLE_MEMORY_CONTROLLER_INTERRUPT
+#endif
+
 namespace ams::kern::board::nintendo::nx {
 
     namespace {
@@ -332,13 +336,13 @@ namespace ams::kern::board::nintendo::nx {
         };
 
         /* Globals. */
-        KLightLock g_lock;
-        u8 g_reserved_asid;
-        KPhysicalAddress   g_memory_controller_address;
-        KPhysicalAddress   g_reserved_table_phys_addr;
-        KDeviceAsidManager g_asid_manager;
-        u32 g_saved_page_tables[AsidCount];
-        u32 g_saved_asid_registers[ams::svc::DeviceName_Count];
+        constinit KLightLock g_lock;
+        constinit u8 g_reserved_asid;
+        constinit KPhysicalAddress   g_memory_controller_address;
+        constinit KPhysicalAddress   g_reserved_table_phys_addr;
+        constinit KDeviceAsidManager g_asid_manager;
+        constinit u32 g_saved_page_tables[AsidCount];
+        constinit u32 g_saved_asid_registers[ams::svc::DeviceName_Count];
 
         /* Memory controller access functionality. */
         void WriteMcRegister(size_t offset, u32 value) {
@@ -349,6 +353,237 @@ namespace ams::kern::board::nintendo::nx {
             return KSystemControl::ReadRegisterPrivileged(GetInteger(g_memory_controller_address) + offset);
         }
 
+        /* Memory controller interrupt functionality. */
+
+        constexpr const char * const MemoryControllerClientNames[138] = {
+            [  0] = "csr_ptcr (ptc)",
+            [  1] = "csr_display0a (dc)",
+            [  2] = "csr_display0ab (dcb)",
+            [  3] = "csr_display0b (dc)",
+            [  4] = "csr_display0bb (dcb)",
+            [  5] = "csr_display0c (dc)",
+            [  6] = "csr_display0cb (dcb)",
+            [  7] = "Unknown Client",
+            [  8] = "Unknown Client",
+            [  9] = "Unknown Client",
+            [ 10] = "Unknown Client",
+            [ 11] = "Unknown Client",
+            [ 12] = "Unknown Client",
+            [ 13] = "Unknown Client",
+            [ 14] = "csr_afir (afi)",
+            [ 15] = "csr_avpcarm7r (avpc)",
+            [ 16] = "csr_displayhc (dc)",
+            [ 17] = "csr_displayhcb (dcb)",
+            [ 18] = "Unknown Client",
+            [ 19] = "Unknown Client",
+            [ 20] = "Unknown Client",
+            [ 21] = "csr_hdar (hda)",
+            [ 22] = "csr_host1xdmar (hc)",
+            [ 23] = "csr_host1xr (hc)",
+            [ 24] = "Unknown Client",
+            [ 25] = "Unknown Client",
+            [ 26] = "Unknown Client",
+            [ 27] = "Unknown Client",
+            [ 28] = "csr_nvencsrd (nvenc)",
+            [ 29] = "csr_ppcsahbdmar (ppcs)",
+            [ 30] = "csr_ppcsahbslvr (ppcs)",
+            [ 31] = "csr_satar (sata)",
+            [ 32] = "Unknown Client",
+            [ 33] = "Unknown Client",
+            [ 34] = "Unknown Client",
+            [ 35] = "Unknown Client",
+            [ 36] = "Unknown Client",
+            [ 37] = "Unknown Client",
+            [ 38] = "Unknown Client",
+            [ 39] = "csr_mpcorer (cpu)",
+            [ 40] = "Unknown Client",
+            [ 41] = "Unknown Client",
+            [ 42] = "Unknown Client",
+            [ 43] = "csw_nvencswr (nvenc)",
+            [ 44] = "Unknown Client",
+            [ 45] = "Unknown Client",
+            [ 46] = "Unknown Client",
+            [ 47] = "Unknown Client",
+            [ 48] = "Unknown Client",
+            [ 49] = "csw_afiw (afi)",
+            [ 50] = "csw_avpcarm7w (avpc)",
+            [ 51] = "Unknown Client",
+            [ 52] = "Unknown Client",
+            [ 53] = "csw_hdaw (hda)",
+            [ 54] = "csw_host1xw (hc)",
+            [ 55] = "Unknown Client",
+            [ 56] = "Unknown Client",
+            [ 57] = "csw_mpcorew (cpu)",
+            [ 58] = "Unknown Client",
+            [ 59] = "csw_ppcsahbdmaw (ppcs)",
+            [ 60] = "csw_ppcsahbslvw (ppcs)",
+            [ 61] = "csw_sataw (sata)",
+            [ 62] = "Unknown Client",
+            [ 63] = "Unknown Client",
+            [ 64] = "Unknown Client",
+            [ 65] = "Unknown Client",
+            [ 66] = "Unknown Client",
+            [ 67] = "Unknown Client",
+            [ 68] = "csr_ispra (isp2)",
+            [ 69] = "Unknown Client",
+            [ 70] = "csw_ispwa (isp2)",
+            [ 71] = "csw_ispwb (isp2)",
+            [ 72] = "Unknown Client",
+            [ 73] = "Unknown Client",
+            [ 74] = "csr_xusb_hostr (xusb_host)",
+            [ 75] = "csw_xusb_hostw (xusb_host)",
+            [ 76] = "csr_xusb_devr (xusb_dev)",
+            [ 77] = "csw_xusb_devw (xusb_dev)",
+            [ 78] = "csr_isprab (isp2b)",
+            [ 79] = "Unknown Client",
+            [ 80] = "csw_ispwab (isp2b)",
+            [ 81] = "csw_ispwbb (isp2b)",
+            [ 82] = "Unknown Client",
+            [ 83] = "Unknown Client",
+            [ 84] = "csr_tsecsrd (tsec)",
+            [ 85] = "csw_tsecswr (tsec)",
+            [ 86] = "csr_a9avpscr (a9avp)",
+            [ 87] = "csw_a9avpscw (a9avp)",
+            [ 88] = "csr_gpusrd (gpu)",
+            [ 89] = "csw_gpuswr (gpu)",
+            [ 90] = "csr_displayt (dc)",
+            [ 91] = "Unknown Client",
+            [ 92] = "Unknown Client",
+            [ 93] = "Unknown Client",
+            [ 94] = "Unknown Client",
+            [ 95] = "Unknown Client",
+            [ 96] = "csr_sdmmcra (sdmmc1a)",
+            [ 97] = "csr_sdmmcraa (sdmmc2a)",
+            [ 98] = "csr_sdmmcr (sdmmc3a)",
+            [ 99] = "csr_sdmmcrab (sdmmc4a)",
+            [100] = "csw_sdmmcwa (sdmmc1a)",
+            [101] = "csw_sdmmcwaa (sdmmc2a)",
+            [102] = "csw_sdmmcw (sdmmc3a)",
+            [103] = "csw_sdmmcwab (sdmmc4a)",
+            [104] = "Unknown Client",
+            [105] = "Unknown Client",
+            [106] = "Unknown Client",
+            [107] = "Unknown Client",
+            [108] = "csr_vicsrd (vic)",
+            [109] = "csw_vicswr (vic)",
+            [110] = "Unknown Client",
+            [111] = "Unknown Client",
+            [112] = "Unknown Client",
+            [113] = "Unknown Client",
+            [114] = "csw_viw (vi)",
+            [115] = "csr_displayd (dc)",
+            [116] = "Unknown Client",
+            [117] = "Unknown Client",
+            [118] = "Unknown Client",
+            [119] = "Unknown Client",
+            [120] = "csr_nvdecsrd (nvdec)",
+            [121] = "csw_nvdecswr (nvdec)",
+            [122] = "csr_aper (ape)",
+            [123] = "csw_apew (ape)",
+            [124] = "Unknown Client",
+            [125] = "Unknown Client",
+            [126] = "csr_nvjpgsrd (nvjpg)",
+            [127] = "csw_nvjpgswr (nvjpg)",
+            [128] = "csr_sesrd (se)",
+            [129] = "csw_seswr (se)",
+            [130] = "csr_axiapr (axiap)",
+            [131] = "csw_axiapw (axiap)",
+            [132] = "csr_etrr (etr)",
+            [133] = "csw_etrw (etr)",
+            [134] = "csr_tsecsrdb (tsecb)",
+            [135] = "csw_tsecswrb (tsecb)",
+            [136] = "csr_gpusrd2 (gpu)",
+            [137] = "csw_gpuswr2 (gpu)",
+        };
+
+        constexpr const char * GetMemoryControllerClientName(size_t i) {
+            if (i < util::size(MemoryControllerClientNames)) {
+                return MemoryControllerClientNames[i];
+            }
+            return "Unknown Client";
+        }
+
+        constexpr const char * const MemoryControllerErrorTypes[8] = {
+            "RSVD",
+            "Unknown",
+            "DECERR_EMEM",
+            "SECURITY_TRUSTZONE",
+            "SECURITY_CARVEOUT",
+            "Unknown",
+            "INVALID_SMMU_PAGE",
+            "Unknown",
+        };
+
+        class KMemoryControllerInterruptTask : public KInterruptTask {
+            public:
+                constexpr KMemoryControllerInterruptTask() : KInterruptTask() { /* ... */ }
+
+                virtual KInterruptTask *OnInterrupt(s32 interrupt_id) override {
+                    MESOSPHERE_UNUSED(interrupt_id);
+                    return this;
+                }
+
+                virtual void DoTask() override {
+                    #if defined(MESOSPHERE_ENABLE_MEMORY_CONTROLLER_INTERRUPT)
+                    {
+                        /* Clear the interrupt when we're done. */
+                        ON_SCOPE_EXIT { Kernel::GetInterruptManager().ClearInterrupt(KInterruptName_MemoryController); };
+
+                        /* Get and clear the interrupt status. */
+                        u32 int_status, err_status, err_adr;
+                        {
+                            int_status = ReadMcRegister(MC_INTSTATUS);
+                            err_status = ReadMcRegister(MC_ERR_STATUS);
+                            err_adr    = ReadMcRegister(MC_ERR_ADR);
+
+                            WriteMcRegister(MC_INTSTATUS, int_status);
+                        }
+
+                        /* Print the interrupt. */
+                        {
+                            constexpr auto GetBits = [] ALWAYS_INLINE_LAMBDA (u32 value, size_t ofs, size_t count) {
+                                return (value >> ofs) & ((1u << count) - 1);
+                            };
+
+                            constexpr auto GetBit = [GetBits] ALWAYS_INLINE_LAMBDA (u32 value, size_t ofs) {
+                                return (value >> ofs) & 1u;
+                            };
+
+                            MESOSPHERE_RELEASE_LOG("sMMU error interrupt\n");
+                            MESOSPHERE_RELEASE_LOG("    MC_INTSTATUS=%08x\n", int_status);
+                            MESOSPHERE_RELEASE_LOG("        DECERR_GENERALIZED_CARVEOUT=%d\n", GetBit(int_status, 17));
+                            MESOSPHERE_RELEASE_LOG("        DECERR_MTS=%d\n",                  GetBit(int_status, 16));
+                            MESOSPHERE_RELEASE_LOG("        SECERR_SEC=%d\n",                  GetBit(int_status, 13));
+                            MESOSPHERE_RELEASE_LOG("        DECERR_VPR=%d\n",                  GetBit(int_status, 12));
+                            MESOSPHERE_RELEASE_LOG("        INVALID_APB_ASID_UPDATE=%d\n",     GetBit(int_status, 11));
+                            MESOSPHERE_RELEASE_LOG("        INVALID_SMMU_PAGE=%d\n",           GetBit(int_status, 10));
+                            MESOSPHERE_RELEASE_LOG("        ARBITRATION_EMEM=%d\n",            GetBit(int_status,  9));
+                            MESOSPHERE_RELEASE_LOG("        SECURITY_VIOLATION=%d\n",          GetBit(int_status,  8));
+                            MESOSPHERE_RELEASE_LOG("        DECERR_EMEM=%d\n",                 GetBit(int_status,  6));
+                            MESOSPHERE_RELEASE_LOG("    MC_ERRSTATUS=%08x\n", err_status);
+                            MESOSPHERE_RELEASE_LOG("        ERR_TYPE=%d (%s)\n",                  GetBits(err_status, 28, 3), MemoryControllerErrorTypes[GetBits(err_status, 28, 3)]);
+                            MESOSPHERE_RELEASE_LOG("        ERR_INVALID_SMMU_PAGE_READABLE=%d\n", GetBit (err_status, 27));
+                            MESOSPHERE_RELEASE_LOG("        ERR_INVALID_SMMU_PAGE_WRITABLE=%d\n", GetBit (err_status, 26));
+                            MESOSPHERE_RELEASE_LOG("        ERR_INVALID_SMMU_NONSECURE=%d\n",     GetBit (err_status, 25));
+                            MESOSPHERE_RELEASE_LOG("        ERR_ADR_HI=%x\n",                     GetBits(err_status, 20, 2));
+                            MESOSPHERE_RELEASE_LOG("        ERR_SWAP=%d\n",                       GetBit (err_status, 18));
+                            MESOSPHERE_RELEASE_LOG("        ERR_SECURITY=%d %s\n",                GetBit (err_status, 17), GetBit(err_status, 17) ? "SECURE" : "NONSECURE");
+                            MESOSPHERE_RELEASE_LOG("        ERR_RW=%d %s\n",                      GetBit (err_status, 16), GetBit(err_status, 16) ? "WRITE" : "READ");
+                            MESOSPHERE_RELEASE_LOG("        ERR_ADR1=%x\n",                       GetBits(err_status, 12, 3));
+                            MESOSPHERE_RELEASE_LOG("        ERR_ID=%d %s\n",                      GetBits(err_status,  0, 8), GetMemoryControllerClientName(GetBits(err_status,  0, 8)));
+                            MESOSPHERE_RELEASE_LOG("    MC_ERRADR=%08x\n", err_adr);
+                            MESOSPHERE_RELEASE_LOG("        ERR_ADR=%lx\n", (static_cast<u64>(GetBits(err_status, 20, 2)) << 32) | static_cast<u64>(err_adr));
+                            MESOSPHERE_RELEASE_LOG("\n");
+                        }
+                    }
+                    #endif
+                }
+        };
+
+        /* Interrupt task global. */
+        constinit KMemoryControllerInterruptTask g_mc_interrupt_task;
+
+        /* Memory controller utilities. */
         void SmmuSynchronizationBarrier() {
             ReadMcRegister(MC_SMMU_CONFIG);
         }
@@ -405,7 +640,7 @@ namespace ams::kern::board::nintendo::nx {
 
     void KDevicePageTable::Initialize() {
         /* Set the memory controller register address. */
-        g_memory_controller_address = KMemoryLayout::GetMemoryControllerRegion().GetAddress();
+        g_memory_controller_address = KMemoryLayout::GetDevicePhysicalAddress(KMemoryRegionType_MemoryController);
 
         /* Allocate a page to use as a reserved/no device table. */
         const KVirtualAddress table_virt_addr = Kernel::GetPageTableManager().Allocate();
@@ -452,11 +687,23 @@ namespace ams::kern::board::nintendo::nx {
         /* Clear int status. */
         WriteMcRegister(MC_INTSTATUS, ReadMcRegister(MC_INTSTATUS));
 
+        /* If we're setting an interrupt handler, unmask all interrupts. */
+        #if defined(MESOSPHERE_ENABLE_MEMORY_CONTROLLER_INTERRUPT)
+        {
+            WriteMcRegister(MC_INTMASK, 0x33D40);
+        }
+        #endif
+
         /* Enable the SMMU */
         WriteMcRegister(MC_SMMU_CONFIG, 1);
         SmmuSynchronizationBarrier();
 
-        /* TODO: Install interrupt handler. */
+        /* Install interrupt handler. */
+        #if defined(MESOSPHERE_ENABLE_MEMORY_CONTROLLER_INTERRUPT)
+        {
+            Kernel::GetInterruptManager().BindHandler(std::addressof(g_mc_interrupt_task), KInterruptName_MemoryController, GetCurrentCoreId(), KInterruptController::PriorityLevel_High, true, true);
+        }
+        #endif
     }
 
     void KDevicePageTable::Lock() {