git subrepo push libraries

subrepo:
  subdir:   "libraries"
  merged:   "e05183a6f"
upstream:
  origin:   "https://github.com/Atmosphere-NX/Atmosphere-libs"
  branch:   "master"
  commit:   "e05183a6f"
git-subrepo:
  version:  "0.4.1"
  origin:   "???"
  commit:   "???"

.gitattributes

@@ -1 +1,14 @@
 config_templates/hbl_html/accessible-urls/accessible-urls.txt text eol=lf
+
+# Mark C++ "include" files as C++
+*.inc linguist-language=C++
+
+# Mark RapidJSON include as vendored
+libraries/include/stratosphere/rapidjson/** linguist-vendored
+
+# Mark emummc as vendored
+emummc/** linguist-vendored
+
+# Mark fatfs as vendored
+exosphere/mariko_fatal/source/fatfs/** linguist-vendored
+fusee/program/source/fatfs/** linguist-vendored

Makefile

@@ -97,12 +97,12 @@ dist-no-debug: all
 	cp config_templates/override_config.ini atmosphere-$(AMSVER)/atmosphere/config_templates/override_config.ini
 	cp config_templates/system_settings.ini atmosphere-$(AMSVER)/atmosphere/config_templates/system_settings.ini
 	cp config_templates/exosphere.ini atmosphere-$(AMSVER)/atmosphere/config_templates/exosphere.ini
-	mkdir config_templates/kip_patches
+	mkdir -p config_templates/kip_patches
 	cp -r config_templates/kip_patches atmosphere-$(AMSVER)/atmosphere/kip_patches
 	cp -r config_templates/hbl_html atmosphere-$(AMSVER)/atmosphere/hbl_html
 	mkdir -p atmosphere-$(AMSVER)/stratosphere_romfs/atmosphere/contents/0100000000000008
-	mkdir -p atmosphere-$(AMSVER)/stratosphere_romfs/atmosphere/contents/0100000000000008
 	mkdir -p atmosphere-$(AMSVER)/stratosphere_romfs/atmosphere/contents/010000000000000D
+	mkdir -p atmosphere-$(AMSVER)/stratosphere_romfs/atmosphere/contents/0100000000000017
 	mkdir -p atmosphere-$(AMSVER)/stratosphere_romfs/atmosphere/contents/010000000000002B
 	mkdir -p atmosphere-$(AMSVER)/stratosphere_romfs/atmosphere/contents/0100000000000032
 	mkdir -p atmosphere-$(AMSVER)/stratosphere_romfs/atmosphere/contents/0100000000000034
@@ -110,8 +110,12 @@ dist-no-debug: all
 	mkdir -p atmosphere-$(AMSVER)/stratosphere_romfs/atmosphere/contents/0100000000000037
 	mkdir -p atmosphere-$(AMSVER)/stratosphere_romfs/atmosphere/contents/010000000000003C
 	mkdir -p atmosphere-$(AMSVER)/stratosphere_romfs/atmosphere/contents/0100000000000042
+	mkdir -p atmosphere-$(AMSVER)/stratosphere_romfs/atmosphere/contents/0100000000000420
+	mkdir -p atmosphere-$(AMSVER)/stratosphere_romfs/atmosphere/contents/010000000000B240
+	mkdir -p atmosphere-$(AMSVER)/stratosphere_romfs/atmosphere/contents/010000000000D623
 	cp stratosphere/boot2/boot2.nsp atmosphere-$(AMSVER)/stratosphere_romfs/atmosphere/contents/0100000000000008/exefs.nsp
 	cp stratosphere/dmnt/dmnt.nsp atmosphere-$(AMSVER)/stratosphere_romfs/atmosphere/contents/010000000000000D/exefs.nsp
+	cp stratosphere/cs/cs.nsp atmosphere-$(AMSVER)/stratosphere_romfs/atmosphere/contents/0100000000000017/exefs.nsp
 	cp stratosphere/erpt/erpt.nsp atmosphere-$(AMSVER)/stratosphere_romfs/atmosphere/contents/010000000000002B/exefs.nsp
 	cp stratosphere/eclct.stub/eclct.stub.nsp atmosphere-$(AMSVER)/stratosphere_romfs/atmosphere/contents/0100000000000032/exefs.nsp
 	cp stratosphere/fatal/fatal.nsp atmosphere-$(AMSVER)/stratosphere_romfs/atmosphere/contents/0100000000000034/exefs.nsp
@@ -119,6 +123,9 @@ dist-no-debug: all
 	cp stratosphere/ro/ro.nsp atmosphere-$(AMSVER)/stratosphere_romfs/atmosphere/contents/0100000000000037/exefs.nsp
 	cp stratosphere/jpegdec/jpegdec.nsp atmosphere-$(AMSVER)/stratosphere_romfs/atmosphere/contents/010000000000003C/exefs.nsp
 	cp stratosphere/pgl/pgl.nsp atmosphere-$(AMSVER)/stratosphere_romfs/atmosphere/contents/0100000000000042/exefs.nsp
+	cp stratosphere/LogManager/LogManager.nsp atmosphere-$(AMSVER)/stratosphere_romfs/atmosphere/contents/0100000000000420/exefs.nsp
+	cp stratosphere/htc/htc.nsp atmosphere-$(AMSVER)/stratosphere_romfs/atmosphere/contents/010000000000B240/exefs.nsp
+	cp stratosphere/TioServer/TioServer.nsp atmosphere-$(AMSVER)/stratosphere_romfs/atmosphere/contents/010000000000D623/exefs.nsp
 	@build_romfs atmosphere-$(AMSVER)/stratosphere_romfs atmosphere-$(AMSVER)/atmosphere/stratosphere.romfs
 	rm -r atmosphere-$(AMSVER)/stratosphere_romfs
 	cp troposphere/reboot_to_payload/reboot_to_payload.nro atmosphere-$(AMSVER)/switch/reboot_to_payload.nro

config_templates/system_settings.ini

@@ -9,6 +9,13 @@
 ; Control whether RO should ease its validation of NROs.
 ; (note: this is normally not necessary, and ips patches can be used.)
 ; ease_nro_restriction = u8!0x1
+[lm]
+; Control whether lm should log to the SD card.
+; Note that this setting does nothing when log manager is not enabled.
+; enable_sd_card_logging = u8!0x1
+; Control the output directory for SD card logs.
+; Note that this setting does nothing when log manager is not enabled/sd card logging is not enabled.
+; sd_card_log_output_directory = str!atmosphere/binlogs
 ; Atmosphere custom settings
 [atmosphere]
 ; Reboot from fatal automatically after some number of milliseconds.
@@ -53,6 +60,10 @@
 ; Controls whether htc is enabled
 ; 0 = Disabled, 1 = Enabled
 ; enable_htc = u8!0x0
+; Controls whether atmosphere's log manager is enabled
+; Note that this setting is ignored (and treated as 1) when htc is enabled.
+; 0 = Disabled, 1 = Enabled
+; enable_log_manager = u8!0x0
 [hbloader]
 ; Controls the size of the homebrew heap when running as applet.
 ; If set to zero, all available applet memory is used as heap.

docs/changelog.md

@@ -1,4 +1,23 @@
 # Changelog
+## 1.1.1
++ A bug was fixed which caused some memory to leak when launching a game with mods enabled, eventually causing a crash after enough game launches without rebooting.
++ General system stability improvements to enhance the user's experience.
+## 1.1.0
++ Support was implemented for 13.0.0.
+  + `mesosphère` was updated to reflect the latest official kernel behavior.
+  + `ncm` was updated to reflect the latest official behaviors.
+  + `erpt` was  updated to reflect the latest official behaviors.
+    + Two new services ("sprofile") were added to `erpt`, and have been fully reimplemented.
+    + **Please Note**: These services provide a way for settings to be pushed to consoles over the internet without system update.
+      + Because there appear to be no settings pushed out yet, this implementation fundamentally cannot be fully tested right now, but hopefully there are no issues once settings begin being distributed.
++ The `LogManager` system module was reimplemented.
+  + This system module provides services that some games use for logging.
+  + Atmosphere's reimplementation supports logging to the SD card (if `lm!enable_sd_card_logging` is true) and to ams.TMA.
+    + To control the directory where logs are saved, modify the `lm!sd_card_log_output_directory` setting.
+  + Atmosphere's reimplementation is disabled by default (in order to save memory), but can be enabled by setting `lm!enable_log_manager` to true.
+    + This will allow reading over logs from games which use the services (or potentially logging from homebrew in the future), which can be useful to developers.
+    + Please note that when TMA is fully implemented in the future, enabling TMA will forcibly enable `LogManager`.
++ General system stability improvements to enhance the user's experience.
 ## 1.0.0
 + `fusee` was completely re-written in C++ to use the same atmosphere-libs APIs as the rest of atmosphere's code.
   + The rewrite was performed with a big emphasis on ensuring a good boot speed, and generally boot should be much faster than it was previously.

emummc/.gitrepo

@@ -6,7 +6,7 @@
 [subrepo]
 	remote = https://github.com/m4xw/emuMMC
 	branch = develop
-	commit = cbc294c390ed73bb281bc1028a8899c053427112
+	commit = f66087313546161a000ee196a788f0626caf80fa
 	parent = 38f9a76ba028995ed3274da3a45b0254f09d1f59
 	method = rebase
 	cmdver = 0.4.1

emummc/README.md

@@ -1,21 +1,21 @@
 # emuMMC
-*A SDMMC driver replacement for Nintendo's Filesystem Services, by **m4xw***  
+*A SDMMC driver replacement for Nintendo's Filesystem Services, by **m4xw***
 
 ### Supported Horizon Versions
-**1.0.0 - 11.0.0**
+**1.0.0 - 13.0.0**
 
 ## Features
-* Arbitrary SDMMC backend selection  
+* Arbitrary SDMMC backend selection
   **This allows loading eMMC from SD or even SD from eMMC**
-* On the fly hooking / patching, fully self-infesting  
+* On the fly hooking / patching, fully self-infesting
   **Only one payload required for all versions!**
-* File-based SDMMC backend support (from SD)  
+* File-based SDMMC backend support (from SD)
   **This allows loading eMMC images from hekate-backups (split or not)**
-* SDMMC device based sector offset (*currently eMMC only*)  
+* SDMMC device based sector offset (*currently eMMC only*)
   **Raw partition support for eMMC from SD with less performance overhead**
-* Full support for `/Nintendo` folder redirection to a arbitrary path  
+* Full support for `/Nintendo` folder redirection to a arbitrary path
   **No 8 char length restriction!**
-* exosphere based context configuration  
+* exosphere based context configuration
   **This includes full support for multiple emuMMC images**
 
 ## Compiling

emummc/source/FS/FS.h

@@ -37,4 +37,15 @@
 #define BOOT_PARTITION_SIZE 0x2000
 #define FS_READ_WRITE_ERROR 1048
 
+#define NAND_PATROL_SECTOR 0xC20
+#define NAND_PATROL_OFFSET 0x184000
+
+typedef struct _fs_nand_patrol_t
+{
+	uint8_t hmac[0x20];
+	unsigned int offset;
+	unsigned int count;
+	uint8_t rsvd[0x1D8];
+} fs_nand_patrol_t;
+
 #endif /* __FS_H__ */

emummc/source/FS/FS_offsets.c

@@ -55,6 +55,8 @@
 #include "offsets/1200_exfat.h"
 #include "offsets/1203.h"
 #include "offsets/1203_exfat.h"
+#include "offsets/1300.h"
+#include "offsets/1300_exfat.h"
 #include "../utils/fatal.h"
 
 #define GET_OFFSET_STRUCT_NAME(vers) g_offsets##vers
@@ -121,6 +123,8 @@ DEFINE_OFFSET_STRUCT(_1200);
 DEFINE_OFFSET_STRUCT(_1200_EXFAT);
 DEFINE_OFFSET_STRUCT(_1203);
 DEFINE_OFFSET_STRUCT(_1203_EXFAT);
+DEFINE_OFFSET_STRUCT(_1300);
+DEFINE_OFFSET_STRUCT(_1300_EXFAT);
 
 const fs_offsets_t *get_fs_offsets(enum FS_VER version) {
     switch (version) {
@@ -202,6 +206,10 @@ const fs_offsets_t *get_fs_offsets(enum FS_VER version) {
             return &(GET_OFFSET_STRUCT_NAME(_1203));
         case FS_VER_12_0_3_EXFAT:
             return &(GET_OFFSET_STRUCT_NAME(_1203_EXFAT));
+        case FS_VER_13_0_0:
+            return &(GET_OFFSET_STRUCT_NAME(_1300));
+        case FS_VER_13_0_0_EXFAT:
+            return &(GET_OFFSET_STRUCT_NAME(_1300_EXFAT));
         default:
             fatal_abort(Fatal_UnknownVersion);
     }

emummc/source/FS/FS_versions.h

@@ -80,6 +80,9 @@ enum FS_VER
     FS_VER_12_0_3,
     FS_VER_12_0_3_EXFAT,
 
+    FS_VER_13_0_0,
+    FS_VER_13_0_0_EXFAT,
+
     FS_VER_MAX,
 };
 

emummc/source/FS/offsets/1300.h

@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2019 m4xw <m4x@m4xw.net>
+ * Copyright (c) 2019 Atmosphere-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef __FS_1300_H__
+#define __FS_1300_H__
+
+// Accessor vtable getters
+#define FS_OFFSET_1300_SDMMC_ACCESSOR_GC   0x158C80
+#define FS_OFFSET_1300_SDMMC_ACCESSOR_SD   0x15AA90
+#define FS_OFFSET_1300_SDMMC_ACCESSOR_NAND 0x1591B0
+
+// Hooks
+#define FS_OFFSET_1300_SDMMC_WRAPPER_READ  0x154620
+#define FS_OFFSET_1300_SDMMC_WRAPPER_WRITE 0x1546E0
+#define FS_OFFSET_1300_RTLD                0x688
+#define FS_OFFSET_1300_RTLD_DESTINATION    ((uintptr_t)(INT64_C(-0x3C)))
+
+#define FS_OFFSET_1300_CLKRST_SET_MIN_V_CLK_RATE 0x153820
+
+// Misc funcs
+#define FS_OFFSET_1300_LOCK_MUTEX          0x29690
+#define FS_OFFSET_1300_UNLOCK_MUTEX        0x296E0
+
+#define FS_OFFSET_1300_SDMMC_WRAPPER_CONTROLLER_OPEN  0x154500
+#define FS_OFFSET_1300_SDMMC_WRAPPER_CONTROLLER_CLOSE 0x154590
+
+// Misc Data
+#define FS_OFFSET_1300_SD_MUTEX            0xE133E8
+#define FS_OFFSET_1300_NAND_MUTEX          0xE0E768
+#define FS_OFFSET_1300_ACTIVE_PARTITION    0xE0E7A8
+#define FS_OFFSET_1300_SDMMC_DAS_HANDLE    0xDF6E18
+
+// NOPs
+#define FS_OFFSET_1300_SD_DAS_INIT         0x27744
+
+// Nintendo Paths
+#define FS_OFFSET_1300_NINTENDO_PATHS \
+{ \
+	{.opcode_reg = 3, .adrp_offset = 0x0006EBE0, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 3, .adrp_offset = 0x0007BEEC, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 4, .adrp_offset = 0x00082294, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 4, .adrp_offset = 0x0009422C, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 0, .adrp_offset = 0, .add_rel_offset = 0}, \
+}
+
+#endif // __FS_1300_H__

emummc/source/FS/offsets/1300_exfat.h

@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2019 m4xw <m4x@m4xw.net>
+ * Copyright (c) 2019 Atmosphere-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef __FS_1300_EXFAT_H__
+#define __FS_1300_EXFAT_H__
+
+// Accessor vtable getters
+#define FS_OFFSET_1300_EXFAT_SDMMC_ACCESSOR_GC   0x158C80
+#define FS_OFFSET_1300_EXFAT_SDMMC_ACCESSOR_SD   0x15AA90
+#define FS_OFFSET_1300_EXFAT_SDMMC_ACCESSOR_NAND 0x1591B0
+
+// Hooks
+#define FS_OFFSET_1300_EXFAT_SDMMC_WRAPPER_READ  0x154620
+#define FS_OFFSET_1300_EXFAT_SDMMC_WRAPPER_WRITE 0x1546E0
+#define FS_OFFSET_1300_EXFAT_RTLD                0x688
+#define FS_OFFSET_1300_EXFAT_RTLD_DESTINATION    ((uintptr_t)(INT64_C(-0x3C)))
+
+#define FS_OFFSET_1300_EXFAT_CLKRST_SET_MIN_V_CLK_RATE 0x153820
+
+// Misc funcs
+#define FS_OFFSET_1300_EXFAT_LOCK_MUTEX          0x29690
+#define FS_OFFSET_1300_EXFAT_UNLOCK_MUTEX        0x296E0
+
+#define FS_OFFSET_1300_EXFAT_SDMMC_WRAPPER_CONTROLLER_OPEN  0x154500
+#define FS_OFFSET_1300_EXFAT_SDMMC_WRAPPER_CONTROLLER_CLOSE 0x154590
+
+// Misc Data
+#define FS_OFFSET_1300_EXFAT_SD_MUTEX            0xE203E8
+#define FS_OFFSET_1300_EXFAT_NAND_MUTEX          0xE1B768
+#define FS_OFFSET_1300_EXFAT_ACTIVE_PARTITION    0xE1B7A8
+#define FS_OFFSET_1300_EXFAT_SDMMC_DAS_HANDLE    0xE03E18
+
+// NOPs
+#define FS_OFFSET_1300_EXFAT_SD_DAS_INIT         0x27744
+
+// Nintendo Paths
+#define FS_OFFSET_1300_EXFAT_NINTENDO_PATHS \
+{ \
+	{.opcode_reg = 3, .adrp_offset = 0x0006EBE0, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 3, .adrp_offset = 0x0007BEEC, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 4, .adrp_offset = 0x00082294, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 4, .adrp_offset = 0x0009422C, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 0, .adrp_offset = 0, .add_rel_offset = 0}, \
+}
+
+#endif // __FS_1300_EXFAT_H__

emummc/source/emuMMC/emummc.c

@@ -89,7 +89,7 @@ static void _sdmmc_ensure_initialized(void)
     }
 }
 
-static void _file_based_update_filename(char *outFilename, u32 sd_path_len, u32 part_idx)
+static void _file_based_update_filename(char *outFilename, unsigned int sd_path_len, unsigned int part_idx)
 {
     snprintf(outFilename + sd_path_len, 3, "%02d", part_idx);
 }
@@ -103,9 +103,7 @@ static void _file_based_emmc_finalize(void)
         f_close(&f_emu.fp_boot1);
 
         for (int i = 0; i < f_emu.parts; i++)
-        {
             f_close(&f_emu.fp_gpp[i]);
-        }
 
         // Force unmount FAT volume.
         f_mount(NULL, "", 1);
@@ -114,12 +112,59 @@ static void _file_based_emmc_finalize(void)
     }
 }
 
+static void _nand_patrol_ensure_integrity(void)
+{
+    fs_nand_patrol_t nand_patrol;
+    static bool nand_patrol_checked = false;
+
+    if (!nand_patrol_checked)
+    {
+        if (emuMMC_ctx.EMMC_Type == emuMMC_SD_Raw)
+        {
+            unsigned int nand_patrol_sector = emuMMC_ctx.EMMC_StoragePartitionOffset + NAND_PATROL_SECTOR;
+            if (!sdmmc_storage_read(&sd_storage, nand_patrol_sector, 1, &nand_patrol))
+                goto out;
+
+            // Clear nand patrol if last offset exceeds storage.
+            if (nand_patrol.offset > sd_storage.sec_cnt)
+            {
+                memset(&nand_patrol, 0, sizeof(fs_nand_patrol_t));
+                sdmmc_storage_write(&sd_storage, nand_patrol_sector, 1, &nand_patrol);
+            }
+        }
+        else if (emuMMC_ctx.EMMC_Type == emuMMC_SD_File && fat_mounted)
+        {
+            FIL *fp = &f_emu.fp_boot0;
+            if (f_lseek(fp, NAND_PATROL_OFFSET) != FR_OK)
+                goto out;
+
+            if (f_read_fast(fp, &nand_patrol, sizeof(fs_nand_patrol_t)) != FR_OK)
+                goto out;
+
+            // Clear nand patrol if last offset exceeds total file based size.
+            if (nand_patrol.offset > f_emu.total_sect)
+            {
+                memset(&nand_patrol, 0, sizeof(fs_nand_patrol_t));
+
+                if (f_lseek(fp, NAND_PATROL_OFFSET) != FR_OK)
+                    goto out;
+
+                if (f_write_fast(fp, &nand_patrol, sizeof(fs_nand_patrol_t)) != FR_OK)
+                    goto out;
+
+                f_sync(fp);
+            }
+        }
+
+out:
+        nand_patrol_checked = true;
+    }
+}
+
 void sdmmc_finalize(void)
 {
     if (!sdmmc_storage_end(&sd_storage))
-    {
         fatal_abort(Fatal_InitSD);
-    }
 
     storageSDinitialized = false;
 }
@@ -137,14 +182,14 @@ static void _file_based_emmc_initialize(void)
     memcpy(path + path_len, "BOOT0", 6);
     if (f_open(&f_emu.fp_boot0, path, FA_READ | FA_WRITE) != FR_OK)
         fatal_abort(Fatal_FatfsFileOpen);
-    if (!f_expand_cltbl(&f_emu.fp_boot0, 0x400, f_emu.clmt_boot0, f_size(&f_emu.fp_boot0)))
+    if (!f_expand_cltbl(&f_emu.fp_boot0, EMUMMC_FP_CLMT_COUNT, f_emu.clmt_boot0, f_size(&f_emu.fp_boot0)))
         fatal_abort(Fatal_FatfsMemExhaustion);
 
     // Open BOOT1 physical partition.
     memcpy(path + path_len, "BOOT1", 6);
     if (f_open(&f_emu.fp_boot1, path, FA_READ | FA_WRITE) != FR_OK)
         fatal_abort(Fatal_FatfsFileOpen);
-    if (!f_expand_cltbl(&f_emu.fp_boot1, 0x400, f_emu.clmt_boot1, f_size(&f_emu.fp_boot1)))
+    if (!f_expand_cltbl(&f_emu.fp_boot1, EMUMMC_FP_CLMT_COUNT, f_emu.clmt_boot1, f_size(&f_emu.fp_boot1)))
         fatal_abort(Fatal_FatfsMemExhaustion);
 
     // Open handles for GPP physical partition files.
@@ -152,15 +197,14 @@ static void _file_based_emmc_initialize(void)
 
     if (f_open(&f_emu.fp_gpp[0], path, FA_READ | FA_WRITE) != FR_OK)
         fatal_abort(Fatal_FatfsFileOpen);
-    if (!f_expand_cltbl(&f_emu.fp_gpp[0], 0x400, &f_emu.clmt_gpp[0], f_size(&f_emu.fp_gpp[0])))
+    if (!f_expand_cltbl(&f_emu.fp_gpp[0], EMUMMC_FP_CLMT_COUNT, &f_emu.clmt_gpp[0], f_size(&f_emu.fp_gpp[0])))
         fatal_abort(Fatal_FatfsMemExhaustion);
 
-    f_emu.part_size = f_size(&f_emu.fp_gpp[0]) >> 9;
+    f_emu.part_size = (uint64_t)f_size(&f_emu.fp_gpp[0]) >> 9;
+    f_emu.total_sect = f_emu.part_size;
 
     // Iterate folder for split parts and stop if next doesn't exist.
-    // Supports up to 32 parts of any size.
-    // TODO: decide on max parts and define them. (hekate produces up to 30 parts on 1GB mode.)
-    for (f_emu.parts = 1; f_emu.parts < 32; f_emu.parts++)
+    for (f_emu.parts = 1; f_emu.parts < EMUMMC_FILE_MAX_PARTS; f_emu.parts++)
     {
         _file_based_update_filename(path, path_len, f_emu.parts);
 
@@ -173,8 +217,13 @@ static void _file_based_emmc_initialize(void)
             return;
         }
 
-        if (!f_expand_cltbl(&f_emu.fp_gpp[f_emu.parts], 0x400, &f_emu.clmt_gpp[f_emu.parts * 0x400], f_size(&f_emu.fp_gpp[f_emu.parts])))
+        if (!f_expand_cltbl(&f_emu.fp_gpp[f_emu.parts], EMUMMC_FP_CLMT_COUNT,
+            &f_emu.clmt_gpp[f_emu.parts * EMUMMC_FP_CLMT_COUNT], f_size(&f_emu.fp_gpp[f_emu.parts])))
+        {
             fatal_abort(Fatal_FatfsMemExhaustion);
+        }
+
+        f_emu.total_sect += (uint64_t)f_size(&f_emu.fp_gpp[f_emu.parts]) >> 9;
     }
 }
 
@@ -189,7 +238,7 @@ bool sdmmc_initialize(void)
             {
                 storageSDinitialized = true;
 
-                // File based emummc.
+                // Init file based emummc.
                 if ((emuMMC_ctx.EMMC_Type == emuMMC_SD_File) && !fat_mounted)
                 {
                     if (f_mount(&f_emu.sd_fs, "", 1) != FR_OK)
@@ -200,6 +249,9 @@ bool sdmmc_initialize(void)
                     _file_based_emmc_initialize();
                 }
 
+                // Check if nand patrol offset is inside limits.
+                _nand_patrol_ensure_integrity();
+
                 break;
             }
 
@@ -207,9 +259,7 @@ bool sdmmc_initialize(void)
         }
 
         if (!storageSDinitialized)
-        {
             fatal_abort(Fatal_InitSD);
-        }
     }
 
     return storageSDinitialized;
@@ -239,19 +289,17 @@ sdmmc_accessor_t *sdmmc_accessor_get(int mmc_id)
 void mutex_lock_handler(int mmc_id)
 {
     if (custom_driver)
-    {
         lock_mutex(sd_mutex);
-    }
+
     lock_mutex(nand_mutex);
 }
 
 void mutex_unlock_handler(int mmc_id)
 {
     unlock_mutex(nand_mutex);
+
     if (custom_driver)
-    {
         unlock_mutex(sd_mutex);
-    }
 }
 
 int sdmmc_nand_get_active_partition_index()
@@ -271,12 +319,16 @@ int sdmmc_nand_get_active_partition_index()
 
 static uint64_t emummc_read_write_inner(void *buf, unsigned int sector, unsigned int num_sectors, bool is_write)
 {
-    if ((emuMMC_ctx.EMMC_Type == emuMMC_SD_Raw))
+    if (emuMMC_ctx.EMMC_Type == emuMMC_SD_Raw)
     {
         // raw partition sector offset: emuMMC_ctx.EMMC_StoragePartitionOffset.
         sector += emuMMC_ctx.EMMC_StoragePartitionOffset;
         // Set physical partition offset.
         sector += (sdmmc_nand_get_active_partition_index() * BOOT_PARTITION_SIZE);
+
+        if (__builtin_expect(sector + num_sectors > sd_storage.sec_cnt, 0))
+            return 0; // Out of bounds. Can only happen with Nand Patrol if resized.
+
         if (!is_write)
             return sdmmc_storage_read(&sd_storage, sector, num_sectors, buf);
         else
@@ -290,6 +342,9 @@ static uint64_t emummc_read_write_inner(void *buf, unsigned int sector, unsigned
     case FS_EMMC_PARTITION_GPP:
         if (f_emu.parts)
         {
+            if (__builtin_expect(sector + num_sectors > f_emu.total_sect, 0))
+                return 0; // Out of bounds. Can only happen with Nand Patrol if resized.
+
             fp = &f_emu.fp_gpp[sector / f_emu.part_size];
             sector = sector % f_emu.part_size;
 
@@ -300,21 +355,21 @@ static uint64_t emummc_read_write_inner(void *buf, unsigned int sector, unsigned
                 while (remaining > 0) {
                     const unsigned int cur_sectors = MIN(remaining, f_emu.part_size - sector);
 
-                    if (f_lseek(fp, (u64)sector << 9) != FR_OK)
+                    if (f_lseek(fp, (uint64_t)sector << 9) != FR_OK)
                         return 0; // Out of bounds.
 
-                    if (is_write)
+                    if (!is_write)
                     {
-                        if (f_write_fast(fp, buf, (u64)cur_sectors << 9) != FR_OK)
+                        if (f_read_fast(fp, buf, (uint64_t)cur_sectors << 9) != FR_OK)
                             return 0;
                     }
                     else
                     {
-                        if (f_read_fast(fp, buf, (u64)cur_sectors << 9) != FR_OK)
+                        if (f_write_fast(fp, buf, (uint64_t)cur_sectors << 9) != FR_OK)
                             return 0;
                     }
 
-                    buf = (char *)buf + ((u64)cur_sectors << 9);
+                    buf = (char *)buf + ((uint64_t)cur_sectors << 9);
                     remaining -= cur_sectors;
                     sector = 0;
                     ++fp;
@@ -324,28 +379,25 @@ static uint64_t emummc_read_write_inner(void *buf, unsigned int sector, unsigned
             }
         }
         else
-        {
             fp = &f_emu.fp_gpp[0];
-        }
         break;
+
     case FS_EMMC_PARTITION_BOOT1:
         fp = &f_emu.fp_boot1;
         break;
+
     case FS_EMMC_PARTITION_BOOT0:
         fp = &f_emu.fp_boot0;
         break;
     }
 
-    if (f_lseek(fp, (u64)sector << 9) != FR_OK)
-        return 0; // Out of bounds.
+    if (f_lseek(fp, (uint64_t)sector << 9) != FR_OK)
+        return 0; // Out of bounds. Can only happen with Nand Patrol if resized.
 
-    uint64_t res = 0;
     if (!is_write)
-        res = !f_read_fast(fp, buf, (u64)num_sectors << 9);
+        return !f_read_fast(fp, buf, (uint64_t)num_sectors << 9);
     else
-        res = !f_write_fast(fp, buf, (u64)num_sectors << 9);
-
-    return res;
+        return !f_write_fast(fp, buf, (uint64_t)num_sectors << 9);
 }
 
 // Controller open wrapper
@@ -382,9 +434,7 @@ uint64_t sdmmc_wrapper_controller_close(int mmc_id)
     if (_this != NULL)
     {
         if (mmc_id == FS_SDMMC_SD)
-        {
             return 0;
-        }
 
         if (mmc_id == FS_SDMMC_EMMC)
         {
@@ -504,8 +554,6 @@ uint64_t sdmmc_wrapper_write(int mmc_id, unsigned int sector, unsigned int num_s
             mutex_lock_handler(mmc_id);
             _current_accessor = _this;
 
-            sector += 0;
-
             // Call hekates driver.
             if (sdmmc_storage_write(&sd_storage, sector, num_sectors, buf))
             {

emummc/source/emuMMC/emummc.h

@@ -36,6 +36,9 @@ extern "C" {
 #include "../FS/FS.h"
 #include "../libs/fatfs/ff.h"
 
+#define EMUMMC_FILE_MAX_PARTS 32
+#define EMUMMC_FP_CLMT_COUNT  1024
+
 // FS typedefs
 typedef sdmmc_accessor_t *(*_sdmmc_accessor_gc)();
 typedef sdmmc_accessor_t *(*_sdmmc_accessor_sd)();
@@ -63,11 +66,12 @@ typedef struct _file_based_ctxt
 	uint64_t parts;
 	uint64_t part_size;
 	FIL fp_boot0;
-	DWORD clmt_boot0[0x400];
+	DWORD clmt_boot0[EMUMMC_FP_CLMT_COUNT];
 	FIL fp_boot1;
-	DWORD clmt_boot1[0x400];
-	FIL fp_gpp[32];
-	DWORD clmt_gpp[0x8000];
+	DWORD clmt_boot1[EMUMMC_FP_CLMT_COUNT];
+	FIL fp_gpp[EMUMMC_FILE_MAX_PARTS];
+	DWORD clmt_gpp[EMUMMC_FILE_MAX_PARTS * EMUMMC_FP_CLMT_COUNT];
+	uint64_t total_sect;
 } file_based_ctxt;
 
 #ifdef __cplusplus

exosphere/program/source/boot/secmon_boot_key_data.s

@@ -85,10 +85,10 @@ _ZN3ams6secmon4boot15VolatileKeyDataE:
 /* We can get away with only including latest because exosphere supports newer-than-expected master key in engine. */
 /* TODO: Update on next change of keys. */
 /* Mariko Development Master Kek Source. */
-.byte 0x75, 0x2D, 0x2E, 0xF3, 0x2F, 0x3F, 0xFE, 0x65, 0xF4, 0xA9, 0x83, 0xB4, 0xED, 0x42, 0x63, 0xBA
+.byte 0x4D, 0x5A, 0xB2, 0xC9, 0xE9, 0xE4, 0x4E, 0xA4, 0xD3, 0xBF, 0x94, 0x12, 0x36, 0x30, 0xD0, 0x7F
 
 /* Mariko Production Master Kek Source. */
-.byte 0xE5, 0x41, 0xAC, 0xEC, 0xD1, 0xA7, 0xD1, 0xAB, 0xED, 0x03, 0x77, 0xF1, 0x27, 0xCA, 0xF8, 0xF1
+.byte 0x52, 0x71, 0x9B, 0xDF, 0xA7, 0x8B, 0x61, 0xD8, 0xD5, 0x85, 0x11, 0xE4, 0x8E, 0x4F, 0x74, 0xC6
 
 /* Development Master Key Vectors. */
 .byte 0x46, 0x22, 0xB4, 0x51, 0x9A, 0x7E, 0xA7, 0x7F, 0x62, 0xA1, 0x1F, 0x8F, 0xC5, 0x3A, 0xDB, 0xFE /* Zeroes encrypted with Master Key 00. */
@@ -103,6 +103,7 @@ _ZN3ams6secmon4boot15VolatileKeyDataE:
 .byte 0x43, 0x3D, 0xC5, 0x3B, 0xEF, 0x91, 0x02, 0x21, 0x61, 0x54, 0x63, 0x8A, 0x35, 0xE7, 0xCA, 0xEE /* Master key 08 encrypted with Master key 09. */
 .byte 0x6C, 0x2E, 0xCD, 0xB3, 0x34, 0x61, 0x77, 0xF5, 0xF9, 0xB1, 0xDD, 0x61, 0x98, 0x19, 0x3E, 0xD4 /* Master key 09 encrypted with Master key 0A. */
 .byte 0x21, 0x88, 0x6B, 0x10, 0x9E, 0x83, 0xD6, 0x52, 0xAB, 0x08, 0xDB, 0x6D, 0x39, 0xFF, 0x1C, 0x9C /* Master key 0A encrypted with Master key 0B. */
+.byte 0x8A, 0xCE, 0xC4, 0x7F, 0xBE, 0x08, 0x61, 0x88, 0xD3, 0x73, 0x64, 0x51, 0xE2, 0xB6, 0x53, 0x15 /* Master key 0B encrypted with Master key 0C. */
 
 /* Production Master Key Vectors. */
 .byte 0x0C, 0xF0, 0x59, 0xAC, 0x85, 0xF6, 0x26, 0x65, 0xE1, 0xE9, 0x19, 0x55, 0xE6, 0xF2, 0x67, 0x3D /* Zeroes encrypted with Master Key 00. */
@@ -117,6 +118,7 @@ _ZN3ams6secmon4boot15VolatileKeyDataE:
 .byte 0x4D, 0xD9, 0x98, 0x42, 0x45, 0x0D, 0xB1, 0x3C, 0x52, 0x0C, 0x9A, 0x44, 0xBB, 0xAD, 0xAF, 0x80 /* Master key 08 encrypted with Master key 09. */
 .byte 0xB8, 0x96, 0x9E, 0x4A, 0x00, 0x0D, 0xD6, 0x28, 0xB3, 0xD1, 0xDB, 0x68, 0x5F, 0xFB, 0xE1, 0x2A /* Master key 09 encrypted with Master key 0A. */
 .byte 0xC1, 0x8D, 0x16, 0xBB, 0x2A, 0xE4, 0x1D, 0xD4, 0xC2, 0xC1, 0xB6, 0x40, 0x94, 0x35, 0x63, 0x98 /* Master key 0A encrypted with Master key 0B. */
+.byte 0xA3, 0x24, 0x65, 0x75, 0xEA, 0xCC, 0x6E, 0x8D, 0xFB, 0x5A, 0x16, 0x50, 0x74, 0xD2, 0x15, 0x06 /* Master key 0B encrypted with Master key 0C. */
 
 /* Device Master Key Source Sources. */
 .byte 0x8B, 0x4E, 0x1C, 0x22, 0x42, 0x07, 0xC8, 0x73, 0x56, 0x94, 0x08, 0x8B, 0xCC, 0x47, 0x0F, 0x5D /* 4.0.0  Device Master Key Source Source. */
@@ -128,6 +130,7 @@ _ZN3ams6secmon4boot15VolatileKeyDataE:
 .byte 0x4A, 0xC3, 0x4E, 0x14, 0x8B, 0x96, 0x4A, 0xD5, 0xD4, 0x99, 0x73, 0xC4, 0x45, 0xAB, 0x8B, 0x49 /* 9.0.0  Device Master Key Source Source. */
 .byte 0x14, 0xB8, 0x74, 0x12, 0xCB, 0xBD, 0x0B, 0x8F, 0x20, 0xFB, 0x30, 0xDA, 0x27, 0xE4, 0x58, 0x94 /* 9.1.0  Device Master Key Source Source. */
 .byte 0xAA, 0xFD, 0xBC, 0xBB, 0x25, 0xC3, 0xA4, 0xEF, 0xE3, 0xEE, 0x58, 0x53, 0xB7, 0xF8, 0xDD, 0xD6 /* 12.1.0 Device Master Key Source Source. */
+.byte 0xE4, 0xF3, 0x45, 0x6F, 0x18, 0xA1, 0x89, 0xF8, 0xDA, 0x4C, 0x64, 0x75, 0x68, 0xE6, 0xBD, 0x4F /* 13.0.0 Device Master Key Source Source. */
 
 /* Development Device Master Kek Sources. */
 .byte 0xD6, 0xBD, 0x9F, 0xC6, 0x18, 0x09, 0xE1, 0x96, 0x20, 0x39, 0x60, 0xD2, 0x89, 0x83, 0x31, 0x34 /* 4.0.0  Device Master Kek Source. */
@@ -139,6 +142,7 @@ _ZN3ams6secmon4boot15VolatileKeyDataE:
 .byte 0x61, 0x6A, 0x88, 0x21, 0xA3, 0x52, 0xB0, 0x19, 0x16, 0x25, 0xA4, 0xE3, 0x4C, 0x54, 0x02, 0x0F /* 9.0.0  Device Master Kek Source. */
 .byte 0x9D, 0xB1, 0xAE, 0xCB, 0xF6, 0xF6, 0xE3, 0xFE, 0xAB, 0x6F, 0xCB, 0xAF, 0x38, 0x03, 0xFC, 0x7B /* 9.1.0  Device Master Kek Source. */
 .byte 0xC4, 0xBB, 0xF3, 0x9F, 0xA3, 0xAA, 0x00, 0x99, 0x7C, 0x97, 0xAD, 0x91, 0x8F, 0xE8, 0x45, 0xCB /* 12.1.0 Device Master Kek Source. */
+.byte 0x20, 0x20, 0xAA, 0xFB, 0x89, 0xC2, 0xF0, 0x70, 0xB5, 0xE0, 0xA3, 0x11, 0x8A, 0x29, 0x8D, 0x0F /* 13.0.0 Device Master Kek Source. */
 
 /* Production Device Master Kek Sources. */
 .byte 0x88, 0x62, 0x34, 0x6E, 0xFA, 0xF7, 0xD8, 0x3F, 0xE1, 0x30, 0x39, 0x50, 0xF0, 0xB7, 0x5D, 0x5D /* 4.0.0  Device Master Kek Source. */
@@ -150,3 +154,4 @@ _ZN3ams6secmon4boot15VolatileKeyDataE:
 .byte 0x03, 0xE7, 0xEB, 0x43, 0x1B, 0xCF, 0x5F, 0xB5, 0xED, 0xDC, 0x97, 0xAE, 0x21, 0x8D, 0x19, 0xED /* 9.0.0  Device Master Kek Source. */
 .byte 0xCE, 0xFE, 0x41, 0x0F, 0x46, 0x9A, 0x30, 0xD6, 0xF2, 0xE9, 0x0C, 0x6B, 0xB7, 0x15, 0x91, 0x36 /* 9.1.0  Device Master Kek Source. */
 .byte 0xC2, 0x65, 0x34, 0x6E, 0xC7, 0xC6, 0x5D, 0x97, 0x3E, 0x34, 0x5C, 0x6B, 0xB3, 0x7E, 0xC6, 0xE3 /* 12.1.0 Device Master Kek Source. */
+.byte 0x77, 0x52, 0x92, 0xF0, 0xAA, 0xE3, 0xFB, 0xE0, 0x60, 0x16, 0xB3, 0x78, 0x68, 0x53, 0xF7, 0xA8 /* 13.0.0 Device Master Kek Source. */

exosphere/program/source/boot/secmon_package2.cpp

@@ -94,7 +94,7 @@ namespace ams::secmon::boot {
         }
 
         /* Check that the key generation is one that we can use. */
-        static_assert(pkg1::KeyGeneration_Count == 12);
+        static_assert(pkg1::KeyGeneration_Count == 13);
         if (key_generation >= pkg1::KeyGeneration_Count) {
             return false;
         }

exosphere/program/source/smc/secmon_smc_info.cpp

@@ -33,14 +33,15 @@ namespace ams::secmon::smc {
             using PhysicalMemorySize = util::BitPack32::Field<16, 2>;
 
             /* Kernel view, from libmesosphere. */
-            using DebugFillMemory             = util::BitPack32::Field<0,                                 1, bool>;
-            using EnableUserExceptionHandlers = util::BitPack32::Field<DebugFillMemory::Next,             1, bool>;
-            using EnableUserPmuAccess         = util::BitPack32::Field<EnableUserExceptionHandlers::Next, 1, bool>;
-            using IncreaseThreadResourceLimit = util::BitPack32::Field<EnableUserPmuAccess::Next,         1, bool>;
-            using Reserved4                   = util::BitPack32::Field<IncreaseThreadResourceLimit::Next, 4, u32>;
-            using UseSecureMonitorPanicCall   = util::BitPack32::Field<Reserved4::Next,                   1, bool>;
-            using Reserved9                   = util::BitPack32::Field<UseSecureMonitorPanicCall::Next,   7, u32>;
-            using MemorySize                  = util::BitPack32::Field<Reserved9::Next,                   2, u32>; /* smc::MemorySize = pkg1::MemorySize */
+            using DebugFillMemory              = util::BitPack32::Field<0,                                  1, bool>;
+            using EnableUserExceptionHandlers  = util::BitPack32::Field<DebugFillMemory::Next,              1, bool>;
+            using EnableUserPmuAccess          = util::BitPack32::Field<EnableUserExceptionHandlers::Next,  1, bool>;
+            using IncreaseThreadResourceLimit  = util::BitPack32::Field<EnableUserPmuAccess::Next,          1, bool>;
+            using DisableDynamicResourceLimits = util::BitPack32::Field<IncreaseThreadResourceLimit::Next,  1, bool>;
+            using Reserved5                    = util::BitPack32::Field<DisableDynamicResourceLimits::Next, 3, u32>;
+            using UseSecureMonitorPanicCall    = util::BitPack32::Field<Reserved5::Next,                    1, bool>;
+            using Reserved9                    = util::BitPack32::Field<UseSecureMonitorPanicCall::Next,    7, u32>;
+            using MemorySize                   = util::BitPack32::Field<Reserved9::Next,                    2, u32>; /* smc::MemorySize = pkg1::MemorySize */
         };
 
         constexpr const pkg1::MemorySize DramIdToMemorySize[fuse::DramId_Count] = {
@@ -50,7 +51,7 @@ namespace ams::secmon::smc {
             [fuse::DramId_IowaHynix1y4GB]     = pkg1::MemorySize_4GB,
             [fuse::DramId_IcosaSamsung6GB]    = pkg1::MemorySize_6GB,
             [fuse::DramId_HoagHynix1y4GB]     = pkg1::MemorySize_4GB,
-            [fuse::DramId_CopperMicron4GB]    = pkg1::MemorySize_4GB,
+            [fuse::DramId_AulaHynix1y4GB]     = pkg1::MemorySize_4GB,
             [fuse::DramId_IowaX1X2Samsung4GB] = pkg1::MemorySize_4GB,
             [fuse::DramId_IowaSansung4GB]     = pkg1::MemorySize_4GB,
             [fuse::DramId_IowaSamsung8GB]     = pkg1::MemorySize_8GB,

fusee/build_package3.py

@@ -15,12 +15,12 @@ def read_file(fn):
 
 def pad(data, size):
     assert len(data) <= size
-    return (data + '\x00' * size)[:size]
+    return (data + b'\x00' * size)[:size]
 
 def get_overlay(program, i):
     return program[0x2B000 + 0x14000 * i:0x2B000 + 0x14000 * (i+1)]
 
-KIP_NAMES = ['Loader', 'NCM', 'ProcessManager', 'sm', 'boot', 'spl', 'ams_mitm']
+KIP_NAMES = [b'Loader', b'NCM', b'ProcessManager', b'sm', b'boot', b'spl', b'ams_mitm']
 
 def get_kips(ams_dir):
     emummc   = read_file(os.path.join(ams_dir, 'emummc/emummc_unpacked.kip'))
@@ -32,13 +32,13 @@ def get_kips(ams_dir):
     spl      = read_file(os.path.join(ams_dir, 'stratosphere/spl/spl.kip'))
     ams_mitm = read_file(os.path.join(ams_dir, 'stratosphere/ams_mitm/ams_mitm.kip'))
     return (emummc, {
-        'Loader' : loader,
-        'NCM' : ncm,
-        'ProcessManager' : pm,
-        'sm' : sm,
-        'boot' : boot,
-        'spl' : spl,
-        'ams_mitm' : ams_mitm,
+        b'Loader' : loader,
+        b'NCM' : ncm,
+        b'ProcessManager' : pm,
+        b'sm' : sm,
+        b'boot' : boot,
+        b'spl' : spl,
+        b'ams_mitm' : ams_mitm,
     })
 
 def write_kip_meta(f, kip, ofs):
@@ -81,14 +81,14 @@ def write_header(f, all_kips, wb_size, tk_size, xf_size, ex_size, ms_size, fs_si
     # Write git_revision;
     f.write(pk('<I', git_revision))
     # Write content metas
-    f.write(pk('<IIBBBBI16s', 0x000800, wb_size,  2, 0, 0, 0, 0xCCCCCCCC, 'warmboot'))
-    f.write(pk('<IIBBBBI16s', 0x002000, tk_size, 12, 0, 0, 0, 0xCCCCCCCC, 'tsec_keygen'))
-    f.write(pk('<IIBBBBI16s', 0x004000, xf_size, 11, 0, 0, 0, 0xCCCCCCCC, 'exosphere_fatal'))
-    f.write(pk('<IIBBBBI16s', 0x048000, ex_size,  1, 0, 0, 0, 0xCCCCCCCC, 'exosphere'))
-    f.write(pk('<IIBBBBI16s', 0x056000, ms_size, 10, 0, 0, 0, 0xCCCCCCCC, 'mesosphere'))
-    f.write(pk('<IIBBBBI16s', 0x7C0000, fs_size,  0, 0, 0, 0, 0xCCCCCCCC, 'fusee'))
-    f.write(pk('<IIBBBBI16s', 0x7E0000, rb_size,  3, 0, 0, 0, 0xCCCCCCCC, 'rebootstub'))
-    f.write(pk('<IIBBBBI16s', 0x100000, len(emummc),  8, 0, 0, 0, 0xCCCCCCCC, 'emummc'))
+    f.write(pk('<IIBBBBI16s', 0x000800, wb_size,  2, 0, 0, 0, 0xCCCCCCCC, b'warmboot'))
+    f.write(pk('<IIBBBBI16s', 0x002000, tk_size, 12, 0, 0, 0, 0xCCCCCCCC, b'tsec_keygen'))
+    f.write(pk('<IIBBBBI16s', 0x004000, xf_size, 11, 0, 0, 0, 0xCCCCCCCC, b'exosphere_fatal'))
+    f.write(pk('<IIBBBBI16s', 0x048000, ex_size,  1, 0, 0, 0, 0xCCCCCCCC, b'exosphere'))
+    f.write(pk('<IIBBBBI16s', 0x056000, ms_size, 10, 0, 0, 0, 0xCCCCCCCC, b'mesosphere'))
+    f.write(pk('<IIBBBBI16s', 0x7C0000, fs_size,  0, 0, 0, 0, 0xCCCCCCCC, b'fusee'))
+    f.write(pk('<IIBBBBI16s', 0x7E0000, rb_size,  3, 0, 0, 0, 0xCCCCCCCC, b'rebootstub'))
+    f.write(pk('<IIBBBBI16s', 0x100000, len(emummc),  8, 0, 0, 0, 0xCCCCCCCC, b'emummc'))
     ofs = (0x100000 + len(emummc) + 0xF) & ~0xF
     for kip_name in KIP_NAMES:
         kip_data = kips[kip_name]
@@ -119,7 +119,7 @@ def write_kips(f, all_kips):
     # Write kips
     tot = len(emummc)
     if (tot & 0xF):
-        f.write('\xCC' * (0x10 - (tot & 0xF)))
+        f.write(b'\xCC' * (0x10 - (tot & 0xF)))
         tot += 0xF
         tot &= ~0xF
     for kip_name in KIP_NAMES:
@@ -127,7 +127,7 @@ def write_kips(f, all_kips):
         f.write(kip_data)
         tot += len(kip_data)
         if (tot & 0xF):
-            f.write('\xCC' * (0x10 - (tot & 0xF)))
+            f.write(b'\xCC' * (0x10 - (tot & 0xF)))
             tot += 0xF
             tot &= ~0xF
     # Pad to 3 MB
@@ -140,7 +140,7 @@ def main(argc, argv):
     # Parse arguments
     ams_dir   = argv[1]
     target    = '' if argv[2] == 'release' else ('_%s' % argv[2])
-    revision  = int(argv[3], 16)
+    revision  = int(argv[3][:8], 16)
     major     = int(argv[4])
     minor     = int(argv[5])
     micro     = int(argv[6])

fusee/program/source/fusee_key_derivation.cpp

@@ -23,17 +23,17 @@ namespace ams::nxboot {
 
         alignas(se::AesBlockSize) constexpr inline const u8 MarikoMasterKekSource[se::AesBlockSize] = {
             /* TODO: Update on next change of keys. */
-            0xE5, 0x41, 0xAC, 0xEC, 0xD1, 0xA7, 0xD1, 0xAB, 0xED, 0x03, 0x77, 0xF1, 0x27, 0xCA, 0xF8, 0xF1
+            0x52, 0x71, 0x9B, 0xDF, 0xA7, 0x8B, 0x61, 0xD8, 0xD5, 0x85, 0x11, 0xE4, 0x8E, 0x4F, 0x74, 0xC6
         };
 
         alignas(se::AesBlockSize) constexpr inline const u8 MarikoMasterKekSourceDev[se::AesBlockSize] = {
             /* TODO: Update on next change of keys. */
-            0x75, 0x2D, 0x2E, 0xF3, 0x2F, 0x3F, 0xFE, 0x65, 0xF4, 0xA9, 0x83, 0xB4, 0xED, 0x42, 0x63, 0xBA
+            0x4D, 0x5A, 0xB2, 0xC9, 0xE9, 0xE4, 0x4E, 0xA4, 0xD3, 0xBF, 0x94, 0x12, 0x36, 0x30, 0xD0, 0x7F
         };
 
         alignas(se::AesBlockSize) constexpr inline const u8 EristaMasterKekSource[se::AesBlockSize] = {
             /* TODO: Update on next change of keys. */
-            0x84, 0x67, 0xB6, 0x7F, 0x13, 0x11, 0xAE, 0xE6, 0x58, 0x9B, 0x19, 0xAF, 0x13, 0x6C, 0x80, 0x7A
+            0x68, 0x3B, 0xCA, 0x54, 0xB8, 0x6F, 0x92, 0x48, 0xC3, 0x05, 0x76, 0x87, 0x88, 0x70, 0x79, 0x23
         };
 
         alignas(se::AesBlockSize) constexpr inline const u8 KeyblobKeySource[se::AesBlockSize] = {
@@ -66,6 +66,7 @@ namespace ams::nxboot {
             { 0x4A, 0xC3, 0x4E, 0x14, 0x8B, 0x96, 0x4A, 0xD5, 0xD4, 0x99, 0x73, 0xC4, 0x45, 0xAB, 0x8B, 0x49 }, /* 9.0.0  Device Master Key Source Source. */
             { 0x14, 0xB8, 0x74, 0x12, 0xCB, 0xBD, 0x0B, 0x8F, 0x20, 0xFB, 0x30, 0xDA, 0x27, 0xE4, 0x58, 0x94 }, /* 9.1.0  Device Master Key Source Source. */
             { 0xAA, 0xFD, 0xBC, 0xBB, 0x25, 0xC3, 0xA4, 0xEF, 0xE3, 0xEE, 0x58, 0x53, 0xB7, 0xF8, 0xDD, 0xD6 }, /* 12.1.0 Device Master Key Source Source. */
+            { 0xE4, 0xF3, 0x45, 0x6F, 0x18, 0xA1, 0x89, 0xF8, 0xDA, 0x4C, 0x64, 0x75, 0x68, 0xE6, 0xBD, 0x4F }, /* 13.0.0 Device Master Key Source Source. */
         };
 
         alignas(se::AesBlockSize) constexpr inline const u8 DeviceMasterKekSources[pkg1::OldDeviceMasterKeyCount][se::AesBlockSize] = {
@@ -78,6 +79,7 @@ namespace ams::nxboot {
             { 0x03, 0xE7, 0xEB, 0x43, 0x1B, 0xCF, 0x5F, 0xB5, 0xED, 0xDC, 0x97, 0xAE, 0x21, 0x8D, 0x19, 0xED }, /* 9.0.0  Device Master Kek Source. */
             { 0xCE, 0xFE, 0x41, 0x0F, 0x46, 0x9A, 0x30, 0xD6, 0xF2, 0xE9, 0x0C, 0x6B, 0xB7, 0x15, 0x91, 0x36 }, /* 9.1.0  Device Master Kek Source. */
             { 0xC2, 0x65, 0x34, 0x6E, 0xC7, 0xC6, 0x5D, 0x97, 0x3E, 0x34, 0x5C, 0x6B, 0xB3, 0x7E, 0xC6, 0xE3 }, /* 12.1.0 Device Master Kek Source. */
+            { 0x77, 0x52, 0x92, 0xF0, 0xAA, 0xE3, 0xFB, 0xE0, 0x60, 0x16, 0xB3, 0x78, 0x68, 0x53, 0xF7, 0xA8 }, /* 13.0.0 Device Master Kek Source. */
         };
 
         alignas(se::AesBlockSize) constexpr inline const u8 DeviceMasterKekSourcesDev[pkg1::OldDeviceMasterKeyCount][se::AesBlockSize] = {
@@ -90,6 +92,7 @@ namespace ams::nxboot {
             { 0x61, 0x6A, 0x88, 0x21, 0xA3, 0x52, 0xB0, 0x19, 0x16, 0x25, 0xA4, 0xE3, 0x4C, 0x54, 0x02, 0x0F }, /* 9.0.0  Device Master Kek Source. */
             { 0x9D, 0xB1, 0xAE, 0xCB, 0xF6, 0xF6, 0xE3, 0xFE, 0xAB, 0x6F, 0xCB, 0xAF, 0x38, 0x03, 0xFC, 0x7B }, /* 9.1.0  Device Master Kek Source. */
             { 0xC4, 0xBB, 0xF3, 0x9F, 0xA3, 0xAA, 0x00, 0x99, 0x7C, 0x97, 0xAD, 0x91, 0x8F, 0xE8, 0x45, 0xCB }, /* 12.1.0 Device Master Kek Source. */
+            { 0x20, 0x20, 0xAA, 0xFB, 0x89, 0xC2, 0xF0, 0x70, 0xB5, 0xE0, 0xA3, 0x11, 0x8A, 0x29, 0x8D, 0x0F }, /* 13.0.0 Device Master Kek Source. */
         };
 
         alignas(se::AesBlockSize) constexpr inline const u8 MasterKeySources[pkg1::KeyGeneration_Count][se::AesBlockSize] = {
@@ -105,6 +108,7 @@ namespace ams::nxboot {
             { 0x4D, 0xD9, 0x98, 0x42, 0x45, 0x0D, 0xB1, 0x3C, 0x52, 0x0C, 0x9A, 0x44, 0xBB, 0xAD, 0xAF, 0x80 }, /* Master key 08 encrypted with Master key 09. */
             { 0xB8, 0x96, 0x9E, 0x4A, 0x00, 0x0D, 0xD6, 0x28, 0xB3, 0xD1, 0xDB, 0x68, 0x5F, 0xFB, 0xE1, 0x2A }, /* Master key 09 encrypted with Master key 0A. */
             { 0xC1, 0x8D, 0x16, 0xBB, 0x2A, 0xE4, 0x1D, 0xD4, 0xC2, 0xC1, 0xB6, 0x40, 0x94, 0x35, 0x63, 0x98 }, /* Master key 0A encrypted with Master key 0B. */
+            { 0xA3, 0x24, 0x65, 0x75, 0xEA, 0xCC, 0x6E, 0x8D, 0xFB, 0x5A, 0x16, 0x50, 0x74, 0xD2, 0x15, 0x06 }, /* Master key 0B encrypted with Master key 0C. */
         };
 
         alignas(se::AesBlockSize) constexpr inline const u8 MasterKeySourcesDev[pkg1::KeyGeneration_Count][se::AesBlockSize] = {
@@ -120,6 +124,7 @@ namespace ams::nxboot {
             { 0x43, 0x3D, 0xC5, 0x3B, 0xEF, 0x91, 0x02, 0x21, 0x61, 0x54, 0x63, 0x8A, 0x35, 0xE7, 0xCA, 0xEE }, /* Master key 08 encrypted with Master key 09. */
             { 0x6C, 0x2E, 0xCD, 0xB3, 0x34, 0x61, 0x77, 0xF5, 0xF9, 0xB1, 0xDD, 0x61, 0x98, 0x19, 0x3E, 0xD4 }, /* Master key 09 encrypted with Master key 0A. */
             { 0x21, 0x88, 0x6B, 0x10, 0x9E, 0x83, 0xD6, 0x52, 0xAB, 0x08, 0xDB, 0x6D, 0x39, 0xFF, 0x1C, 0x9C }, /* Master key 0A encrypted with Master key 0B. */
+            { 0x8A, 0xCE, 0xC4, 0x7F, 0xBE, 0x08, 0x61, 0x88, 0xD3, 0x73, 0x64, 0x51, 0xE2, 0xB6, 0x53, 0x15 }, /* Master key 0B encrypted with Master key 0C. */
         };
 
         alignas(se::AesBlockSize) constinit u8 MasterKeys[pkg1::OldMasterKeyCount][se::AesBlockSize] = {};

fusee/program/source/fusee_package2.cpp

@@ -80,7 +80,7 @@ namespace ams::nxboot {
             }
 
             /* Check that the key generation is one that we can use. */
-            static_assert(pkg1::KeyGeneration_Count == 12);
+            static_assert(pkg1::KeyGeneration_Count == 13);
             if (key_generation >= pkg1::KeyGeneration_Count) {
                 return false;
             }

fusee/program/source/fusee_setup_horizon.cpp

@@ -270,6 +270,8 @@ namespace ams::nxboot {
                         target_firmware = ams::TargetFirmware_12_0_2;
                     } else if (std::memcmp(package1 + 0x10, "20210607", 8) == 0) {
                         target_firmware = ams::TargetFirmware_12_1_0;
+                    } else if (std::memcmp(package1 + 0x10, "20210805", 8) == 0) {
+                        target_firmware = ams::TargetFirmware_13_0_0;
                     } else {
                         ShowFatalError("Unable to identify package1!\n");
                     }
@@ -281,7 +283,9 @@ namespace ams::nxboot {
 
             #define CHECK_NCA(NCA_ID, VERSION) do { if (IsNcaExist(NCA_ID)) { return ams::TargetFirmware_##VERSION; } } while(0)
 
-            if (target_firmware >= ams::TargetFirmware_12_1_0) {
+            if (target_firmware >= ams::TargetFirmware_13_0_0) {
+                CHECK_NCA("bf2337ee88bd9f963a33b3ecbbc3732a", 13_0_0);
+            } else if (target_firmware >= ams::TargetFirmware_12_1_0) {
                 CHECK_NCA("9d9d83d68d9517f245f3e8cd7f93c416", 12_1_0);
             } else if (target_firmware >= ams::TargetFirmware_12_0_2) {
                 CHECK_NCA("a1863a5c0e1cedd442f5e60b0422dc15", 12_0_3);
@@ -622,16 +626,20 @@ namespace ams::nxboot {
                                     storage_ctx.flags[0] &= ~secmon::SecureMonitorConfigurationFlag_EnableUserModePerformanceCounterAccess;
                                 }
                             } else if (std::strcmp(entry.key, "blank_prodinfo_sysmmc") == 0) {
-                                if (entry.value[0] == '1' && !emummc_enabled) {
-                                    storage_ctx.flags[0] |= secmon::SecureMonitorConfigurationFlag_ShouldUseBlankCalibrationBinary;
-                                } else {
-                                    storage_ctx.flags[0] &= ~secmon::SecureMonitorConfigurationFlag_ShouldUseBlankCalibrationBinary;
+                                if (!emummc_enabled) {
+                                    if (entry.value[0] == '1') {
+                                        storage_ctx.flags[0] |= secmon::SecureMonitorConfigurationFlag_ShouldUseBlankCalibrationBinary;
+                                    } else {
+                                        storage_ctx.flags[0] &= ~secmon::SecureMonitorConfigurationFlag_ShouldUseBlankCalibrationBinary;
+                                    }
                                 }
                             } else if (std::strcmp(entry.key, "blank_prodinfo_emummc") == 0) {
-                                if (entry.value[0] == '1' && emummc_enabled) {
-                                    storage_ctx.flags[0] |= secmon::SecureMonitorConfigurationFlag_ShouldUseBlankCalibrationBinary;
-                                } else {
-                                    storage_ctx.flags[0] &= ~secmon::SecureMonitorConfigurationFlag_ShouldUseBlankCalibrationBinary;
+                                if (emummc_enabled) {
+                                    if (entry.value[0] == '1') {
+                                        storage_ctx.flags[0] |= secmon::SecureMonitorConfigurationFlag_ShouldUseBlankCalibrationBinary;
+                                    } else {
+                                        storage_ctx.flags[0] &= ~secmon::SecureMonitorConfigurationFlag_ShouldUseBlankCalibrationBinary;
+                                    }
                                 }
                             } else if (std::strcmp(entry.key, "allow_writing_to_cal_sysmmc") == 0) {
                                 if (entry.value[0] == '1') {
@@ -647,7 +655,7 @@ namespace ams::nxboot {
                             } else if (std::strcmp(entry.key, "log_baud_rate") == 0) {
                                 storage_ctx.log_baud_rate = ParseDecimalInteger(entry.value);
                             } else if (std::strcmp(entry.key, "log_inverted") == 0) {
-                                if (entry.value[0] == 1) {
+                                if (entry.value[0] == '1') {
                                     storage_ctx.log_flags |= uart::Flag_Inverted;
                                 }
                             }

fusee/program/source/fusee_stratosphere.cpp

@@ -147,6 +147,9 @@ namespace ams::nxboot {
             FsVersion_12_0_3,
             FsVersion_12_0_3_Exfat,
 
+            FsVersion_13_0_0,
+            FsVersion_13_0_0_Exfat,
+
             FsVersion_Count,
         };
 
@@ -209,6 +212,9 @@ namespace ams::nxboot {
 
             { 0xC8, 0x67, 0x62, 0xBE, 0x19, 0xA5, 0x1F, 0xA0 }, /* FsVersion_12_0_3 */
             { 0xE1, 0xE8, 0xD3, 0xD6, 0xA2, 0xFE, 0x0B, 0x10 }, /* FsVersion_12_0_3_Exfat */
+
+            { 0x7D, 0x20, 0x05, 0x47, 0x17, 0x8A, 0x83, 0x6A }, /* FsVersion_13_0_0 */
+            { 0x51, 0xEB, 0xFA, 0x9C, 0xCF, 0x66, 0xC0, 0x9E }, /* FsVersion_13_0_0_Exfat */
         };
 
         const InitialProcessBinaryHeader *FindInitialProcessBinary(const pkg2::Package2Header *header, const u8 *data, ams::TargetFirmware target_firmware) {
@@ -592,6 +598,10 @@ namespace ams::nxboot {
                 case FsVersion_12_0_3_Exfat:
                     AddPatch(fs_meta, 0x155579, NogcPatch0, sizeof(NogcPatch0));
                     AddPatch(fs_meta, 0x13EC34, NogcPatch1, sizeof(NogcPatch1));
+                case FsVersion_13_0_0:
+                case FsVersion_13_0_0_Exfat:
+                    AddPatch(fs_meta, 0x159119, NogcPatch0, sizeof(NogcPatch0));
+                    AddPatch(fs_meta, 0x1426D0, NogcPatch1, sizeof(NogcPatch1));
                     break;
                 default:
                     break;

fusee/program/source/mtc/fusee_mtc.cpp

@@ -37,7 +37,7 @@ namespace ams::nxboot {
             /* DramId_MarikoIowaHynix1y4gb     */ 0x10,
             /* DramId_EristaIcosaSamsung6gb    */ 0x01,
             /* DramId_MarikoHoagHynix1y4gb     */ 0x10,
-            /* DramId_EristaCopperMicron4gb    */ MemoryTrainingTableIndex_Invalid,
+            /* DramId_MarikoAulaHynix1y4gb     */ 0x10,
             /* DramId_MarikoIowax1x2Samsung4gb */ 0x00,
             /* DramId_MarikoIowaSamsung4gb     */ 0x05,
             /* DramId_MarikoIowaSamsung8gb     */ 0x06,

fusee/program/source/sdram/fusee_sdram.cpp

@@ -83,6 +83,8 @@ namespace ams::nxboot {
                         break;
                 switch (dram_id) {
                     HANDLE_DRAM_CASE( 3, 12)
+                    HANDLE_DRAM_CASE( 5, 12)
+                    HANDLE_DRAM_CASE( 6, 12)
                     HANDLE_DRAM_CASE( 7,  0)
                     HANDLE_DRAM_CASE( 8,  1)
                     HANDLE_DRAM_CASE( 9,  2)

libraries/.gitrepo

@@ -6,7 +6,7 @@
 [subrepo]
 	remote = https://github.com/Atmosphere-NX/Atmosphere-libs
 	branch = master
-	commit = 0c0bb8153ab5d56b2a428ba662dd1bff620b7404
-	parent = a1af1af74d465440b01744ccb054edaaaca0c192
+	commit = e05183a6f4849d2a55e5d75f03e8af51010648b7
+	parent = a3d9efb18cfcbdce7b286d1ead67313e0cb6af1f
 	method = merge
 	cmdver = 0.4.1

libraries/libexosphere/include/exosphere/fuse.hpp

@@ -54,7 +54,7 @@ namespace ams::fuse {
         DramId_IowaHynix1y4GB     =  3,
         DramId_IcosaSamsung6GB    =  4,
         DramId_HoagHynix1y4GB     =  5,
-        DramId_CopperMicron4GB    =  6,
+        DramId_AulaHynix1y4GB     =  6,
         DramId_IowaX1X2Samsung4GB =  7,
         DramId_IowaSansung4GB     =  8,
         DramId_IowaSamsung8GB     =  9,

libraries/libexosphere/include/exosphere/pkg1/pkg1_key_generation.hpp

@@ -32,6 +32,7 @@ namespace ams::pkg1 {
         KeyGeneration_9_0_0  = 0x09,
         KeyGeneration_9_1_0  = 0x0A,
         KeyGeneration_12_1_0 = 0x0B,
+        KeyGeneration_13_0_0 = 0x0C,
 
         KeyGeneration_Count,
 

libraries/libmesosphere/Makefile

@@ -142,7 +142,7 @@ $(OFILES_SRC)	: $(HFILES_BIN)
 
 kern_libc_generic.o: CFLAGS += -fno-builtin
 
-kern_k_auto_object.o: CXXFLAGS += -fno-lto
+kern_k_auto_object.o kern_k_debug_base_process_holder.o: CXXFLAGS += -fno-lto
 
 #---------------------------------------------------------------------------------
 %_bin.h %.bin.o	:	%.bin

libraries/libmesosphere/include/mesosphere.hpp

@@ -78,8 +78,7 @@
 #include <mesosphere/kern_select_debug.hpp>
 #include <mesosphere/kern_k_process.hpp>
 #include <mesosphere/kern_k_resource_limit.hpp>
-#include <mesosphere/kern_k_alpha.hpp>
-#include <mesosphere/kern_k_beta.hpp>
+#include <mesosphere/kern_k_io_pool.hpp>
 
 /* More Miscellaneous objects. */
 #include <mesosphere/kern_k_object_name.hpp>

libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp

@@ -293,7 +293,7 @@ namespace ams::kern::arch::arm64::init {
                 }
 
                 /* Swap the mappings. */
-                const u64 attr_preserve_mask  = (negative_block_size_for_mask | 0xFFFF000000000000ul) ^ ((1ul << 48) - 1);
+                const u64 attr_preserve_mask  = (block_size - 1) | 0xFFFF000000000000ul;
                 const size_t shift_for_contig = contig ? 4 : 0;
                 size_t advanced_size = 0;
                 const u64 src_attr_val = src_saved.GetRawAttributesUnsafeForSwap() & attr_preserve_mask;
@@ -726,8 +726,8 @@ namespace ams::kern::arch::arm64::init {
                 m_state.end_address  = address;
             }
 
-            ALWAYS_INLINE void InitializeFromState(uintptr_t state_val) {
-                m_state = *reinterpret_cast<State *>(state_val);
+            ALWAYS_INLINE void InitializeFromState(const State *state) {
+                m_state = *state;
             }
 
             ALWAYS_INLINE void GetFinalState(State *out) {

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_assembly_offsets.h

@@ -14,6 +14,7 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 #pragma once
+#include <mesosphere/kern_build_config.hpp>
 
 /* TODO: Different header for this? */
 #define AMS_KERN_NUM_SUPERVISOR_CALLS 0xC0
@@ -30,6 +31,10 @@
 #define THREAD_STACK_PARAMETERS_IS_IN_EXCEPTION_HANDLER 0x2D
 #define THREAD_STACK_PARAMETERS_IS_PINNED               0x2E
 
+#if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP)
+#define THREAD_STACK_PARAMETERS_IS_SINGLE_STEP          0x2F
+#endif
+
 /* ams::kern::arch::arm64::KThreadContext, https://github.com/Atmosphere-NX/Atmosphere/blob/master/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_thread_context.hpp */
 #define THREAD_CONTEXT_SIZE          0x290
 #define THREAD_CONTEXT_CPU_REGISTERS 0x000
@@ -151,7 +156,9 @@
 
 /* ams::kern::KScheduler (::SchedulingState), https://github.com/Atmosphere-NX/Atmosphere/blob/master/libraries/libmesosphere/include/mesosphere/kern_k_scheduler.hpp */
 /* NOTE: Due to constraints on ldarb relative offsets, KSCHEDULER_NEEDS_SCHEDULING cannot trivially be changed, and will require assembly edits. */
-#define KSCHEDULER_NEEDS_SCHEDULING               0x00
-#define KSCHEDULER_INTERRUPT_TASK_THREAD_RUNNABLE 0x01
-#define KSCHEDULER_HIGHEST_PRIORITY_THREAD        0x10
-#define KSCHEDULER_IDLE_THREAD_STACK              0x18
+#define KSCHEDULER_NEEDS_SCHEDULING        0x00
+#define KSCHEDULER_INTERRUPT_TASK_RUNNABLE 0x01
+#define KSCHEDULER_HIGHEST_PRIORITY_THREAD 0x10
+#define KSCHEDULER_IDLE_THREAD_STACK       0x18
+#define KSCHEDULER_PREVIOUS_THREAD         0x20
+#define KSCHEDULER_INTERRUPT_TASK_MANAGER  0x28

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu_system_registers.hpp

@@ -265,6 +265,10 @@ namespace ams::kern::arch::arm64::cpu {
                 return this->GetBits(12, 1) != 0;
             }
 
+            constexpr ALWAYS_INLINE bool GetSoftwareStep() const {
+                return this->GetBits(0, 1) != 0;
+            }
+
             constexpr ALWAYS_INLINE decltype(auto) SetMde(bool set) {
                 this->SetBit(15, set);
                 return *this;
@@ -274,6 +278,11 @@ namespace ams::kern::arch::arm64::cpu {
                 this->SetBit(12, set);
                 return *this;
             }
+
+            constexpr ALWAYS_INLINE decltype(auto) SetSoftwareStep(bool set) {
+                this->SetBit(0, set);
+                return *this;
+            }
     };
 
     MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(MultiprocessorAffinity) {

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_debug.hpp

@@ -31,7 +31,6 @@ namespace ams::kern::arch::arm64 {
         MESOSPHERE_AUTOOBJECT_TRAITS(KDebug, KSynchronizationObject);
         public:
             explicit KDebug() { /* ... */ }
-            virtual ~KDebug() { /* ... */ }
 
             static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
         public:

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table.hpp

@@ -22,9 +22,11 @@
 
 namespace ams::kern::arch::arm64 {
 
-    class KPageTable : public KPageTableBase {
+    class KPageTable final : public KPageTableBase {
         NON_COPYABLE(KPageTable);
         NON_MOVEABLE(KPageTable);
+        private:
+            friend class KPageTableBase;
         public:
             using TraversalEntry   = KPageTableImpl::TraversalEntry;
             using TraversalContext = KPageTableImpl::TraversalContext;
@@ -96,9 +98,9 @@ namespace ams::kern::arch::arm64 {
             u64 m_ttbr;
             u8 m_asid;
         protected:
-            virtual Result Operate(PageLinkedList *page_list, KProcessAddress virt_addr, size_t num_pages, KPhysicalAddress phys_addr, bool is_pa_valid, const KPageProperties properties, OperationType operation, bool reuse_ll) override;
-            virtual Result Operate(PageLinkedList *page_list, KProcessAddress virt_addr, size_t num_pages, const KPageGroup &page_group, const KPageProperties properties, OperationType operation, bool reuse_ll) override;
-            virtual void   FinalizeUpdate(PageLinkedList *page_list) override;
+            Result OperateImpl(PageLinkedList *page_list, KProcessAddress virt_addr, size_t num_pages, KPhysicalAddress phys_addr, bool is_pa_valid, const KPageProperties properties, OperationType operation, bool reuse_ll);
+            Result OperateImpl(PageLinkedList *page_list, KProcessAddress virt_addr, size_t num_pages, const KPageGroup &page_group, const KPageProperties properties, OperationType operation, bool reuse_ll);
+            void FinalizeUpdateImpl(PageLinkedList *page_list);
 
             KPageTableManager &GetPageTableManager() const { return *m_manager; }
         private:

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_process_page_table.hpp

@@ -96,6 +96,14 @@ namespace ams::kern::arch::arm64 {
                 return m_page_table.MapIo(phys_addr, size, perm);
             }
 
+            Result MapIoRegion(KProcessAddress dst_address, KPhysicalAddress phys_addr, size_t size, ams::svc::MemoryMapping mapping, ams::svc::MemoryPermission perm) {
+                return m_page_table.MapIoRegion(dst_address, phys_addr, size, mapping, perm);
+            }
+
+            Result UnmapIoRegion(KProcessAddress dst_address, KPhysicalAddress phys_addr, size_t size) {
+                return m_page_table.UnmapIoRegion(dst_address, phys_addr, size);
+            }
+
             Result MapStatic(KPhysicalAddress phys_addr, size_t size, KMemoryPermission perm) {
                 return m_page_table.MapStatic(phys_addr, size, perm);
             }
@@ -164,8 +172,8 @@ namespace ams::kern::arch::arm64 {
                 return m_page_table.UnlockForDeviceAddressSpace(address, size);
             }
 
-            Result UnlockForDeviceAddressSpacePartialMap(KProcessAddress address, size_t size, size_t mapped_size) {
-                return m_page_table.UnlockForDeviceAddressSpacePartialMap(address, size, mapped_size);
+            Result UnlockForDeviceAddressSpacePartialMap(KProcessAddress address, size_t size) {
+                return m_page_table.UnlockForDeviceAddressSpacePartialMap(address, size);
             }
 
             Result OpenMemoryRangeForMapDeviceAddressSpace(KPageTableBase::MemoryRange *out, KProcessAddress address, size_t size, KMemoryPermission perm, bool is_aligned) {

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_slab_heap_impl.hpp

@@ -16,6 +16,7 @@
 #pragma once
 #include <vapours.hpp>
 #include <mesosphere/kern_select_cpu.hpp>
+#include <mesosphere/kern_select_interrupt_manager.hpp>
 
 namespace ams::kern::arch::arm64 {
 
@@ -24,6 +25,32 @@ namespace ams::kern::arch::arm64 {
         { t.next } -> std::convertible_to<T *>;
     };
 
+    ALWAYS_INLINE bool IsSlabAtomicValid() {
+        /* Without careful consideration, slab heaps atomics are vulnerable to */
+        /* the ABA problem, when doing compare and swap of node pointers. */
+        /* We resolve this by using the ARM exclusive monitor; we bundle the */
+        /* load and store of the relevant values into a single exclusive monitor */
+        /* hold, preventing the ABA problem. */
+        /* However, our assembly must do both a load and a store under a single */
+        /* hold, at different memory addresses. Considering the case where the */
+        /* addresses are distinct but resolve to the same cache set (by chance), */
+        /* we can note that under a 1-way associative (direct-mapped) cache */
+        /* we would have as a guarantee that the second access would evict the */
+        /* cache line from the first access, invalidating our exclusive monitor */
+        /* hold. Thus, we require that the cache is not 1-way associative, for */
+        /* our implementation to be correct. */
+        {
+            /* Disable interrupts. */
+            KScopedInterruptDisable di;
+
+            /* Select L1 cache. */
+            cpu::SetCsselrEl1(0);
+
+            /* Check that the L1 cache is not direct-mapped. */
+            return cpu::CacheSizeIdRegisterAccessor().GetAssociativity() != 0;
+        }
+    }
+
     template<typename T> requires SlabHeapNode<T>
     ALWAYS_INLINE T *AllocateFromSlabAtomic(T **head) {
         u32 tmp;
@@ -36,10 +63,7 @@ namespace ams::kern::arch::arm64 {
             "    ldr    %[next], [%[node]]\n"
             "    stlxr  %w[tmp], %[next], [%[head]]\n"
             "    cbnz   %w[tmp], 1b\n"
-            "    b      3f\n"
             "2:\n"
-            "    clrex\n"
-            "3:\n"
             : [tmp]"=&r"(tmp), [node]"=&r"(node), [next]"=&r"(next), [head]"+&r"(head)
             :
             : "cc", "memory"
@@ -59,7 +83,6 @@ namespace ams::kern::arch::arm64 {
             "    str    %[next], [%[node]]\n"
             "    stlxr  %w[tmp], %[node], [%[head]]\n"
             "    cbnz   %w[tmp], 1b\n"
-            "2:\n"
             : [tmp]"=&r"(tmp), [node]"+&r"(node), [next]"=&r"(next), [head]"+&r"(head)
             :
             : "cc", "memory"

libraries/libmesosphere/include/mesosphere/board/nintendo/nx/kern_k_device_page_table.hpp

@@ -69,16 +69,16 @@ namespace ams::kern::board::nintendo::nx {
             Result Attach(ams::svc::DeviceName device_name, u64 space_address, u64 space_size);
             Result Detach(ams::svc::DeviceName device_name);
 
-            Result Map(size_t *out_mapped_size, KProcessPageTable *page_table, KProcessAddress process_address, size_t size, KDeviceVirtualAddress device_address, ams::svc::MemoryPermission device_perm, bool is_aligned, bool refresh_mappings);
+            Result Map(KProcessPageTable *page_table, KProcessAddress process_address, size_t size, KDeviceVirtualAddress device_address, ams::svc::MemoryPermission device_perm, bool is_aligned);
             Result Unmap(KProcessPageTable *page_table, KProcessAddress process_address, size_t size, KDeviceVirtualAddress device_address);
 
             void Unmap(KDeviceVirtualAddress device_address, size_t size) {
                 return this->UnmapImpl(device_address, size, false);
             }
         private:
-            Result MapDevicePage(size_t *out_mapped_size, s32 &num_pt, s32 max_pt, KPhysicalAddress phys_addr, u64 size, KDeviceVirtualAddress address, ams::svc::MemoryPermission device_perm);
+            Result MapDevicePage(KPhysicalAddress phys_addr, u64 size, KDeviceVirtualAddress address, ams::svc::MemoryPermission device_perm);
 
-            Result MapImpl(size_t *out_mapped_size, s32 &num_pt, s32 max_pt, KProcessPageTable *page_table, KProcessAddress process_address, size_t size, KDeviceVirtualAddress device_address, ams::svc::MemoryPermission device_perm, bool is_aligned);
+            Result MapImpl(KProcessPageTable *page_table, KProcessAddress process_address, size_t size, KDeviceVirtualAddress device_address, ams::svc::MemoryPermission device_perm, bool is_aligned);
             void UnmapImpl(KDeviceVirtualAddress address, u64 size, bool force);
 
             bool IsFree(KDeviceVirtualAddress address, u64 size) const;

libraries/libmesosphere/include/mesosphere/board/nintendo/nx/kern_k_system_control.hpp

@@ -16,6 +16,12 @@
 #pragma once
 #include <mesosphere/kern_common.hpp>
 
+namespace ams::kern {
+
+    struct InitialProcessBinaryLayout;
+
+}
+
 namespace ams::kern::board::nintendo::nx {
 
     class KSystemControl {
@@ -25,7 +31,7 @@ namespace ams::kern::board::nintendo::nx {
                     /* Initialization. */
                     static size_t GetIntendedMemorySize();
                     static KPhysicalAddress GetKernelPhysicalBaseAddress(uintptr_t base_address);
-                    static KPhysicalAddress GetInitialProcessBinaryPhysicalAddress();
+                    static void GetInitialProcessBinaryLayout(InitialProcessBinaryLayout *out);
                     static bool ShouldIncreaseThreadResourceLimit();
                     static void CpuOn(u64 core_id, uintptr_t entrypoint, uintptr_t arg);
                     static size_t GetApplicationPoolSize();

libraries/libmesosphere/include/mesosphere/init/kern_init_slab_setup.hpp

@@ -34,8 +34,8 @@ namespace ams::kern::init {
         size_t num_KObjectName;
         size_t num_KResourceLimit;
         size_t num_KDebug;
-        size_t num_KAlpha;
-        size_t num_KBeta;
+        size_t num_KIoPool;
+        size_t num_KIoRegion;
     };
 
     NOINLINE void InitializeSlabResourceCounts();

libraries/libmesosphere/include/mesosphere/kern_build_config.hpp

@@ -31,3 +31,4 @@
 
 //#define MESOSPHERE_BUILD_FOR_TRACING
 #define MESOSPHERE_ENABLE_PANIC_REGISTER_DUMP
+#define MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP

libraries/libmesosphere/include/mesosphere/kern_debug_log.hpp

@@ -28,6 +28,8 @@ namespace ams::kern {
             static NOINLINE void Printf(const char *format, ...) __attribute__((format(printf, 1, 2)));
             static NOINLINE void VPrintf(const char *format, ::std::va_list vl);
 
+            static NOINLINE void LogException(const char *str);
+
             static NOINLINE Result PrintUserString(ams::kern::svc::KUserPointer<const char *> user_str, size_t len);
 
             /* Functionality for preserving across sleep. */
@@ -49,6 +51,8 @@ namespace ams::kern {
 
 #endif
 
+#define MESOSPHERE_EXCEPTION_LOG(str) ::ams::kern::KDebugLog::LogException(str)
+
 #define MESOSPHERE_RELEASE_LOG(fmt, ...) ::ams::kern::KDebugLog::Printf((fmt), ## __VA_ARGS__)
 #define MESOSPHERE_RELEASE_VLOG(fmt, vl) ::ams::kern::KDebugLog::VPrintf((fmt), (vl))
 

libraries/libmesosphere/include/mesosphere/kern_initial_process.hpp

@@ -29,14 +29,19 @@ namespace ams::kern {
         u32 reserved;
     };
 
-    NOINLINE size_t CopyInitialProcessBinaryToKernelMemory();
-    NOINLINE void CreateAndRunInitialProcesses();
+    struct InitialProcessBinaryLayout {
+        uintptr_t address;
+        uintptr_t _08;
+    };
+
+    KPhysicalAddress GetInitialProcessBinaryPhysicalAddress();
+    void SetInitialProcessBinaryPhysicalAddress(KPhysicalAddress phys_addr);
 
     u64 GetInitialProcessIdMin();
     u64 GetInitialProcessIdMax();
-    KVirtualAddress GetInitialProcessBinaryAddress();
     size_t GetInitialProcessesSecureMemorySize();
 
-    void LoadInitialProcessBinaryHeaderDeprecated(KPhysicalAddress pool_end);
+    NOINLINE size_t CopyInitialProcessBinaryToKernelMemory();
+    NOINLINE void CreateAndRunInitialProcesses();
 
 }

libraries/libmesosphere/include/mesosphere/kern_k_alpha.hpp

@@ -1,41 +0,0 @@
-/*
- * Copyright (c) 2018-2020 Atmosphère-NX
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-#pragma once
-#include <mesosphere/kern_common.hpp>
-#include <mesosphere/kern_k_auto_object.hpp>
-#include <mesosphere/kern_slab_helpers.hpp>
-
-namespace ams::kern {
-
-    class KAlpha final : public KAutoObjectWithSlabHeapAndContainer<KAlpha, KAutoObjectWithList> {
-        MESOSPHERE_AUTOOBJECT_TRAITS(KAlpha, KAutoObject);
-        private:
-            /* NOTE: Official KAlpha has size 0x50, corresponding to 0x20 bytes of fields. */
-            /* TODO: Add these fields, if KAlpha is ever instantiable in the NX kernel. */
-        public:
-            explicit KAlpha() {
-                /* ... */
-            }
-
-            virtual ~KAlpha() { /* ... */ }
-
-            /* virtual void Finalize() override; */
-
-            virtual bool IsInitialized() const override { return false /* TODO */; }
-            static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
-    };
-
-}

libraries/libmesosphere/include/mesosphere/kern_k_auto_object.hpp

@@ -75,7 +75,6 @@ namespace ams::kern {
             static KAutoObject *Create(KAutoObject *ptr);
         public:
             constexpr ALWAYS_INLINE explicit KAutoObject() : m_next_closed_object(nullptr), m_ref_count(0) { MESOSPHERE_ASSERT_THIS(); }
-            virtual ~KAutoObject() { MESOSPHERE_ASSERT_THIS(); }
 
             /* Destroy is responsible for destroying the auto object's resources when ref_count hits zero. */
             virtual void Destroy() { MESOSPHERE_ASSERT_THIS(); }
@@ -139,8 +138,10 @@ namespace ams::kern {
         private:
             friend class KAutoObjectWithListContainer;
         private:
-            util::IntrusiveRedBlackTreeNode list_node;
+            util::IntrusiveRedBlackTreeNode m_list_node;
         public:
+            constexpr ALWAYS_INLINE KAutoObjectWithList() : m_list_node() { /* ... */ }
+
             static ALWAYS_INLINE int Compare(const KAutoObjectWithList &lhs, const KAutoObjectWithList &rhs) {
                 const u64 lid = lhs.GetId();
                 const u64 rid = rhs.GetId();

libraries/libmesosphere/include/mesosphere/kern_k_auto_object_container.hpp

@@ -24,7 +24,7 @@ namespace ams::kern {
         NON_COPYABLE(KAutoObjectWithListContainer);
         NON_MOVEABLE(KAutoObjectWithListContainer);
         public:
-            using ListType = util::IntrusiveRedBlackTreeMemberTraits<&KAutoObjectWithList::list_node>::TreeType<KAutoObjectWithList>;
+            using ListType = util::IntrusiveRedBlackTreeMemberTraits<&KAutoObjectWithList::m_list_node>::TreeType<KAutoObjectWithList>;
         public:
             class ListAccessor : public KScopedLightLock {
                 private:

libraries/libmesosphere/include/mesosphere/kern_k_class_token.hpp

@@ -104,17 +104,14 @@ namespace ams::kern {
                 KSession,
                 KSharedMemory,
                 KEvent,
-                KWritableEvent,
                 KLightClientSession,
                 KLightServerSession,
                 KTransferMemory,
                 KDeviceAddressSpace,
                 KSessionRequest,
                 KCodeMemory,
-
-                /* NOTE: True order for these has not been determined yet. */
-                KAlpha,
-                KBeta,
+                KIoPool,
+                KIoRegion,
 
                 FinalClassesEnd = FinalClassesStart + NumFinalClasses,
             };

libraries/libmesosphere/include/mesosphere/kern_k_client_port.hpp

@@ -34,7 +34,6 @@ namespace ams::kern {
             KPort *m_parent;
         public:
             constexpr KClientPort() : m_num_sessions(), m_peak_sessions(), m_max_sessions(), m_parent() { /* ... */ }
-            virtual ~KClientPort() { /* ... */ }
 
             void Initialize(KPort *parent, s32 max_sessions);
             void OnSessionFinalized();

libraries/libmesosphere/include/mesosphere/kern_k_client_session.hpp

@@ -20,14 +20,14 @@
 namespace ams::kern {
 
     class KSession;
+    class KEvent;
 
-    class KClientSession final : public KAutoObjectWithSlabHeapAndContainer<KClientSession, KAutoObjectWithList> {
+    class KClientSession final : public KAutoObject {
         MESOSPHERE_AUTOOBJECT_TRAITS(KClientSession, KAutoObject);
         private:
             KSession *m_parent;
         public:
             constexpr KClientSession() : m_parent() { /* ... */ }
-            virtual ~KClientSession() { /* ... */ }
 
             void Initialize(KSession *parent) {
                 /* Set member variables. */
@@ -40,7 +40,7 @@ namespace ams::kern {
             constexpr KSession *GetParent() const { return m_parent; }
 
             Result SendSyncRequest(uintptr_t address, size_t size);
-            Result SendAsyncRequest(KWritableEvent *event, uintptr_t address, size_t size);
+            Result SendAsyncRequest(KEvent *event, uintptr_t address, size_t size);
 
             void OnServerClosed();
     };

libraries/libmesosphere/include/mesosphere/kern_k_code_memory.hpp

@@ -35,8 +35,6 @@ namespace ams::kern {
                 /* ... */
             }
 
-            virtual ~KCodeMemory() { /* ... */ }
-
             Result Initialize(KProcessAddress address, size_t size);
             virtual void Finalize() override;
 

libraries/libmesosphere/include/mesosphere/kern_k_condition_variable.hpp

@@ -29,8 +29,8 @@ namespace ams::kern {
             constexpr KConditionVariable() : m_tree() { /* ... */ }
 
             /* Arbitration. */
-            Result SignalToAddress(KProcessAddress addr);
-            Result WaitForAddress(ams::svc::Handle handle, KProcessAddress addr, u32 value);
+            static Result SignalToAddress(KProcessAddress addr);
+            static Result WaitForAddress(ams::svc::Handle handle, KProcessAddress addr, u32 value);
 
             /* Condition variable. */
             void Signal(uintptr_t cv_key, s32 count);

libraries/libmesosphere/include/mesosphere/kern_k_debug_base.hpp

@@ -25,15 +25,31 @@ namespace ams::kern {
     class KDebugBase : public KSynchronizationObject {
         protected:
             using DebugEventList = util::IntrusiveListBaseTraits<KEventInfo>::ListType;
+        private:
+            class ProcessHolder {
+                private:
+                    friend class KDebugBase;
+                private:
+                    KProcess *m_process;
+                    std::atomic<u32> m_ref_count;
+                private:
+                    explicit ProcessHolder() : m_process(nullptr) { /* ... */ }
+
+                    void Attach(KProcess *process);
+                    void Detach();
+
+                    bool Open();
+                    void Close();
+            };
         private:
             DebugEventList m_event_info_list;
             u32 m_continue_flags;
-            KProcess *m_process;
+            ProcessHolder m_process_holder;
             KLightLock m_lock;
             KProcess::State m_old_process_state;
+            bool m_is_attached;
         public:
-            explicit KDebugBase() { /* ... */ }
-            virtual ~KDebugBase() { /* ... */ }
+            explicit KDebugBase() : m_event_info_list(), m_process_holder(), m_lock() { /* ... */ }
         protected:
             bool Is64Bit() const;
         public:
@@ -60,7 +76,21 @@ namespace ams::kern {
             Result GetDebugEventInfo(ams::svc::lp64::DebugEventInfo *out);
             Result GetDebugEventInfo(ams::svc::ilp32::DebugEventInfo *out);
 
-            KScopedAutoObject<KProcess> GetProcess();
+            ALWAYS_INLINE bool IsAttached() const {
+                return m_is_attached;
+            }
+
+            ALWAYS_INLINE bool OpenProcess() {
+                return m_process_holder.Open();
+            }
+
+            ALWAYS_INLINE void CloseProcess() {
+                return m_process_holder.Close();
+            }
+
+            ALWAYS_INLINE KProcess *GetProcessUnsafe() const {
+                return m_process_holder.m_process;
+            }
         private:
             void PushDebugEvent(ams::svc::DebugEvent event, uintptr_t param0 = 0, uintptr_t param1 = 0, uintptr_t param2 = 0, uintptr_t param3 = 0, uintptr_t param4 = 0);
             void EnqueueDebugEventInfo(KEventInfo *info);

libraries/libmesosphere/include/mesosphere/kern_k_device_address_space.hpp

@@ -31,7 +31,6 @@ namespace ams::kern {
             bool m_is_initialized;
         public:
             constexpr KDeviceAddressSpace() : m_lock(), m_table(), m_space_address(), m_space_size(), m_is_initialized() { /* ... */ }
-            virtual ~KDeviceAddressSpace() { /* ... */ }
 
             Result Initialize(u64 address, u64 size);
             virtual void Finalize() override;
@@ -42,18 +41,17 @@ namespace ams::kern {
             Result Attach(ams::svc::DeviceName device_name);
             Result Detach(ams::svc::DeviceName device_name);
 
-            Result Map(size_t *out_mapped_size, KProcessPageTable *page_table, KProcessAddress process_address, size_t size, u64 device_address, ams::svc::MemoryPermission device_perm, bool refresh_mappings) {
-                return this->Map(out_mapped_size, page_table, process_address, size, device_address, device_perm, false, refresh_mappings);
+            Result MapByForce(KProcessPageTable *page_table, KProcessAddress process_address, size_t size, u64 device_address, ams::svc::MemoryPermission device_perm) {
+                return this->Map(page_table, process_address, size, device_address, device_perm, false);
             }
 
             Result MapAligned(KProcessPageTable *page_table, KProcessAddress process_address, size_t size, u64 device_address, ams::svc::MemoryPermission device_perm) {
-                size_t dummy;
-                return this->Map(std::addressof(dummy), page_table, process_address, size, device_address, device_perm, true, false);
+                return this->Map(page_table, process_address, size, device_address, device_perm, true);
             }
 
             Result Unmap(KProcessPageTable *page_table, KProcessAddress process_address, size_t size, u64 device_address);
         private:
-            Result Map(size_t *out_mapped_size, KProcessPageTable *page_table, KProcessAddress process_address, size_t size, u64 device_address, ams::svc::MemoryPermission device_perm, bool is_aligned, bool refresh_mappings);
+            Result Map(KProcessPageTable *page_table, KProcessAddress process_address, size_t size, u64 device_address, ams::svc::MemoryPermission device_perm, bool is_aligned);
         public:
             static void Initialize();
     };

libraries/libmesosphere/include/mesosphere/kern_k_dynamic_resource_manager.hpp

@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_k_dynamic_slab_heap.hpp>
+
+namespace ams::kern {
+
+    template<typename T, bool ClearNode = false>
+    class KDynamicResourceManager {
+        NON_COPYABLE(KDynamicResourceManager);
+        NON_MOVEABLE(KDynamicResourceManager);
+        public:
+            using DynamicSlabType = KDynamicSlabHeap<T, ClearNode>;
+        private:
+            KDynamicPageManager *m_page_allocator{};
+            DynamicSlabType *m_slab_heap{};
+        public:
+            constexpr KDynamicResourceManager() = default;
+
+            constexpr ALWAYS_INLINE KVirtualAddress GetAddress() const { return m_slab_heap->GetAddress(); }
+            constexpr ALWAYS_INLINE size_t GetSize() const { return m_slab_heap->GetSize(); }
+            constexpr ALWAYS_INLINE size_t GetUsed() const { return m_slab_heap->GetUsed(); }
+            constexpr ALWAYS_INLINE size_t GetPeak() const { return m_slab_heap->GetPeak(); }
+            constexpr ALWAYS_INLINE size_t GetCount() const { return m_slab_heap->GetCount(); }
+
+            ALWAYS_INLINE void Initialize(KDynamicPageManager *page_allocator, DynamicSlabType *slab_heap) {
+                m_page_allocator = page_allocator;
+                m_slab_heap      = slab_heap;
+            }
+
+            T *Allocate() const {
+                return m_slab_heap->Allocate(m_page_allocator);
+            }
+
+            void Free(T *t) const {
+                m_slab_heap->Free(t);
+            }
+    };
+
+    class KBlockInfoManager       : public KDynamicResourceManager<KBlockInfo>{};
+    class KMemoryBlockSlabManager : public KDynamicResourceManager<KMemoryBlock>{};
+
+    using KBlockInfoSlabHeap   = typename KBlockInfoManager::DynamicSlabType;
+    using KMemoryBlockSlabHeap = typename KMemoryBlockSlabManager::DynamicSlabType;
+
+}

libraries/libmesosphere/include/mesosphere/kern_k_dynamic_slab_heap.hpp

@@ -23,95 +23,71 @@
 namespace ams::kern {
 
     template<typename T, bool ClearNode = false>
-    class KDynamicSlabHeap {
+    class KDynamicSlabHeap : protected impl::KSlabHeapImpl {
         NON_COPYABLE(KDynamicSlabHeap);
         NON_MOVEABLE(KDynamicSlabHeap);
         private:
-            using Impl       = impl::KSlabHeapImpl;
             using PageBuffer = KDynamicPageManager::PageBuffer;
         private:
-            Impl m_impl;
-            KDynamicPageManager *m_page_allocator;
-            std::atomic<size_t> m_used;
-            std::atomic<size_t> m_peak;
-            std::atomic<size_t> m_count;
-            KVirtualAddress m_address;
-            size_t m_size;
-        private:
-            ALWAYS_INLINE Impl *GetImpl() {
-                return std::addressof(m_impl);
-            }
-            ALWAYS_INLINE const Impl *GetImpl() const {
-                return std::addressof(m_impl);
-            }
+            std::atomic<size_t> m_used{};
+            std::atomic<size_t> m_peak{};
+            std::atomic<size_t> m_count{};
+            KVirtualAddress m_address{};
+            size_t m_size{};
         public:
-            constexpr KDynamicSlabHeap() : m_impl(), m_page_allocator(), m_used(), m_peak(), m_count(), m_address(), m_size() { /* ... */ }
+            constexpr KDynamicSlabHeap() = default;
 
-            constexpr KVirtualAddress GetAddress() const { return m_address; }
-            constexpr size_t GetSize() const { return m_size; }
-            constexpr size_t GetUsed() const { return m_used.load(); }
-            constexpr size_t GetPeak() const { return m_peak.load(); }
-            constexpr size_t GetCount() const { return m_count.load(); }
+            constexpr ALWAYS_INLINE KVirtualAddress GetAddress() const { return m_address; }
+            constexpr ALWAYS_INLINE size_t GetSize() const { return m_size; }
+            constexpr ALWAYS_INLINE size_t GetUsed() const { return m_used.load(); }
+            constexpr ALWAYS_INLINE size_t GetPeak() const { return m_peak.load(); }
+            constexpr ALWAYS_INLINE size_t GetCount() const { return m_count.load(); }
 
-            constexpr bool IsInRange(KVirtualAddress addr) const {
+            constexpr ALWAYS_INLINE bool IsInRange(KVirtualAddress addr) const {
                 return this->GetAddress() <= addr && addr <= this->GetAddress() + this->GetSize() - 1;
             }
 
-            void Initialize(KVirtualAddress memory, size_t sz) {
-                /* Set tracking fields. */
-                m_address = memory;
-                m_count   = sz / sizeof(T);
-                m_size    = m_count * sizeof(T);
-
-                /* Free blocks to memory. */
-                u8 *cur = GetPointer<u8>(m_address + m_size);
-                for (size_t i = 0; i < sz / sizeof(T); i++) {
-                    cur -= sizeof(T);
-                    this->GetImpl()->Free(cur);
-                }
-            }
-
-            void Initialize(KDynamicPageManager *page_allocator) {
-                m_page_allocator = page_allocator;
-                m_address        = m_page_allocator->GetAddress();
-                m_size           = m_page_allocator->GetSize();
-            }
-
-            void Initialize(KDynamicPageManager *page_allocator, size_t num_objects) {
+            ALWAYS_INLINE void Initialize(KDynamicPageManager *page_allocator, size_t num_objects) {
                 MESOSPHERE_ASSERT(page_allocator != nullptr);
 
                 /* Initialize members. */
-                this->Initialize(page_allocator);
+                m_address = page_allocator->GetAddress();
+                m_size    = page_allocator->GetSize();
+
+                /* Initialize the base allocator. */
+                KSlabHeapImpl::Initialize();
 
                 /* Allocate until we have the correct number of objects. */
                 while (m_count.load() < num_objects) {
-                    auto *allocated = reinterpret_cast<T *>(m_page_allocator->Allocate());
+                    auto *allocated = reinterpret_cast<T *>(page_allocator->Allocate());
                     MESOSPHERE_ABORT_UNLESS(allocated != nullptr);
+
                     for (size_t i = 0; i < sizeof(PageBuffer) / sizeof(T); i++) {
-                        this->GetImpl()->Free(allocated + i);
+                        KSlabHeapImpl::Free(allocated + i);
                     }
+
                     m_count.fetch_add(sizeof(PageBuffer) / sizeof(T));
                 }
             }
 
-            T *Allocate() {
-                T *allocated = reinterpret_cast<T *>(this->GetImpl()->Allocate());
+            ALWAYS_INLINE T *Allocate(KDynamicPageManager *page_allocator) {
+                T *allocated = static_cast<T *>(KSlabHeapImpl::Allocate());
 
                 /* If we successfully allocated and we should clear the node, do so. */
                 if constexpr (ClearNode) {
                     if (AMS_LIKELY(allocated != nullptr)) {
-                        reinterpret_cast<Impl::Node *>(allocated)->next = nullptr;
+                        reinterpret_cast<KSlabHeapImpl::Node *>(allocated)->next = nullptr;
                     }
                 }
 
                 /* If we fail to allocate, try to get a new page from our next allocator. */
-                if (AMS_UNLIKELY(allocated == nullptr)) {
-                    if (m_page_allocator != nullptr) {
-                        allocated = reinterpret_cast<T *>(m_page_allocator->Allocate());
+                if (AMS_UNLIKELY(allocated == nullptr) ) {
+                    if (page_allocator != nullptr) {
+                        allocated = reinterpret_cast<T *>(page_allocator->Allocate());
                         if (allocated != nullptr) {
                             /* If we succeeded in getting a page, free the rest to our slab. */
                             for (size_t i = 1; i < sizeof(PageBuffer) / sizeof(T); i++) {
-                                this->GetImpl()->Free(allocated + i);
+                                KSlabHeapImpl::Free(allocated + i);
                             }
                             m_count.fetch_add(sizeof(PageBuffer) / sizeof(T));
                         }
@@ -135,13 +111,10 @@ namespace ams::kern {
                 return allocated;
             }
 
-            void Free(T *t) {
-                this->GetImpl()->Free(t);
+            ALWAYS_INLINE void Free(T *t) {
+                KSlabHeapImpl::Free(t);
                 m_used.fetch_sub(1);
             }
     };
 
-    class KBlockInfoManager       : public KDynamicSlabHeap<KBlockInfo>{};
-    class KMemoryBlockSlabManager : public KDynamicSlabHeap<KMemoryBlock>{};
-
 }

libraries/libmesosphere/include/mesosphere/kern_k_event.hpp

@@ -18,26 +18,23 @@
 #include <mesosphere/kern_k_auto_object.hpp>
 #include <mesosphere/kern_slab_helpers.hpp>
 #include <mesosphere/kern_k_readable_event.hpp>
-#include <mesosphere/kern_k_writable_event.hpp>
 
 namespace ams::kern {
 
-    class KEvent final : public KAutoObjectWithSlabHeapAndContainer<KEvent, KAutoObjectWithList> {
+    class KEvent final : public KAutoObjectWithSlabHeapAndContainer<KEvent, KAutoObjectWithList, true> {
         MESOSPHERE_AUTOOBJECT_TRAITS(KEvent, KAutoObject);
         private:
             KReadableEvent m_readable_event;
-            KWritableEvent m_writable_event;
             KProcess *m_owner;
             bool m_initialized;
+            bool m_readable_event_destroyed;
         public:
             constexpr KEvent()
-                : m_readable_event(), m_writable_event(), m_owner(), m_initialized()
+                : m_readable_event(), m_owner(), m_initialized(), m_readable_event_destroyed()
             {
                 /* ... */
             }
 
-            virtual ~KEvent() { /* ... */ }
-
             void Initialize();
             virtual void Finalize() override;
 
@@ -49,7 +46,11 @@ namespace ams::kern {
             virtual KProcess *GetOwner() const override { return m_owner; }
 
             KReadableEvent &GetReadableEvent() { return m_readable_event; }
-            KWritableEvent &GetWritableEvent() { return m_writable_event; }
+
+            Result Signal();
+            Result Clear();
+
+            ALWAYS_INLINE void OnReadableEventDestroyed() { m_readable_event_destroyed = true; }
     };
 
 }

libraries/libmesosphere/include/mesosphere/kern_k_handle_table.hpp

@@ -54,14 +54,10 @@ namespace ams::kern {
             }
 
             union EntryInfo {
-                struct {
-                    u16 linear_id;
-                    u16 type;
-                } info;
-                s32 next_free_index;
+                u16 linear_id;
+                s16 next_free_index;
 
-                constexpr ALWAYS_INLINE u16 GetLinearId() const { return info.linear_id; }
-                constexpr ALWAYS_INLINE u16 GetType() const { return info.type; }
+                constexpr ALWAYS_INLINE u16 GetLinearId() const { return linear_id; }
                 constexpr ALWAYS_INLINE s32 GetNextFreeIndex() const { return next_free_index; }
             };
         private:
@@ -187,17 +183,8 @@ namespace ams::kern {
             NOINLINE Result Reserve(ams::svc::Handle *out_handle);
             NOINLINE void Unreserve(ams::svc::Handle handle);
 
-            template<typename T>
-            ALWAYS_INLINE Result Add(ams::svc::Handle *out_handle, T *obj) {
-                static_assert(std::is_base_of<KAutoObject, T>::value);
-                return this->Add(out_handle, obj, obj->GetTypeObj().GetClassToken());
-            }
-
-            template<typename T>
-            ALWAYS_INLINE void Register(ams::svc::Handle handle, T *obj) {
-                static_assert(std::is_base_of<KAutoObject, T>::value);
-                return this->Register(handle, obj, obj->GetTypeObj().GetClassToken());
-            }
+            NOINLINE Result Add(ams::svc::Handle *out_handle, KAutoObject *obj);
+            NOINLINE void Register(ams::svc::Handle handle, KAutoObject *obj);
 
             template<typename T>
             ALWAYS_INLINE bool GetMultipleObjects(T **out, const ams::svc::Handle *handles, size_t num_handles) const {
@@ -242,8 +229,6 @@ namespace ams::kern {
                 return false;
             }
         private:
-            NOINLINE Result Add(ams::svc::Handle *out_handle, KAutoObject *obj, u16 type);
-            NOINLINE void Register(ams::svc::Handle handle, KAutoObject *obj, u16 type);
 
             constexpr ALWAYS_INLINE s32 AllocateEntry() {
                 MESOSPHERE_ASSERT_THIS();

libraries/libmesosphere/include/mesosphere/kern_k_interrupt_event.hpp

@@ -32,7 +32,6 @@ namespace ams::kern {
             bool m_is_initialized;
         public:
             constexpr KInterruptEvent() : m_interrupt_id(-1), m_core_id(-1), m_is_initialized(false) { /* ... */ }
-            virtual ~KInterruptEvent() { /* ... */ }
 
             Result Initialize(int32_t interrupt_name, ams::svc::InterruptType type);
             virtual void Finalize() override;
@@ -49,13 +48,10 @@ namespace ams::kern {
     class KInterruptEventTask : public KSlabAllocated<KInterruptEventTask>, public KInterruptTask {
         private:
             KInterruptEvent *m_event;
-            KLightLock m_lock;
         public:
-            constexpr KInterruptEventTask() : m_event(nullptr), m_lock() { /* ... */ }
+            constexpr KInterruptEventTask() : m_event(nullptr) { /* ... */ }
             ~KInterruptEventTask() { /* ... */ }
 
-            KLightLock &GetLock() { return m_lock; }
-
             virtual KInterruptTask *OnInterrupt(s32 interrupt_id) override;
             virtual void DoTask() override;
 

libraries/libmesosphere/include/mesosphere/kern_k_interrupt_task_manager.hpp

@@ -27,28 +27,25 @@ namespace ams::kern {
                     KInterruptTask *m_head;
                     KInterruptTask *m_tail;
                 public:
-                    constexpr TaskQueue() : m_head(nullptr), m_tail(nullptr) { /* ... */ }
+                    constexpr ALWAYS_INLINE TaskQueue() : m_head(nullptr), m_tail(nullptr) { /* ... */ }
 
-                    constexpr KInterruptTask *GetHead() { return m_head; }
-                    constexpr bool IsEmpty() const { return m_head == nullptr; }
-                    constexpr void Clear() { m_head = nullptr; m_tail = nullptr; }
+                    constexpr ALWAYS_INLINE KInterruptTask *GetHead() { return m_head; }
+                    constexpr ALWAYS_INLINE bool IsEmpty() const { return m_head == nullptr; }
+                    constexpr ALWAYS_INLINE void Clear() { m_head = nullptr; m_tail = nullptr; }
 
                     void Enqueue(KInterruptTask *task);
                     void Dequeue();
             };
         private:
             TaskQueue m_task_queue;
-            KThread *m_thread;
-        private:
-            static void ThreadFunction(uintptr_t arg);
-            void ThreadFunctionImpl();
+            s64 m_cpu_time;
         public:
-            constexpr KInterruptTaskManager() : m_task_queue(), m_thread(nullptr) { /* ... */ }
+            constexpr KInterruptTaskManager() : m_task_queue(), m_cpu_time(0) { /* ... */ }
 
-            constexpr KThread *GetThread() const { return m_thread; }
+            constexpr ALWAYS_INLINE s64 GetCpuTime() const { return m_cpu_time; }
 
-            NOINLINE void Initialize();
             void EnqueueTask(KInterruptTask *task);
+            void DoTasks();
     };
 
 }

libraries/libmesosphere/include/mesosphere/kern_k_io_pool.hpp

@@ -17,32 +17,34 @@
 #include <mesosphere/kern_common.hpp>
 #include <mesosphere/kern_k_auto_object.hpp>
 #include <mesosphere/kern_slab_helpers.hpp>
+#include <mesosphere/kern_k_io_region.hpp>
 
 namespace ams::kern {
 
-    class KProcess;
-
-    class KBeta final : public KAutoObjectWithSlabHeapAndContainer<KBeta, KAutoObjectWithList> {
-        MESOSPHERE_AUTOOBJECT_TRAITS(KBeta, KAutoObject);
+    class KIoPool final : public KAutoObjectWithSlabHeapAndContainer<KIoPool, KAutoObjectWithList> {
+        MESOSPHERE_AUTOOBJECT_TRAITS(KIoPool, KAutoObject);
         private:
-            friend class KProcess;
+            using IoRegionList = util::IntrusiveListMemberTraits<&KIoRegion::m_pool_list_node>::ListType;
         private:
-            /* NOTE: Official KBeta has size 0x88, corresponding to 0x58 bytes of fields. */
-            /* TODO: Add these fields, if KBeta is ever instantiable in the NX kernel. */
-            util::IntrusiveListNode m_process_list_node;
+            KLightLock m_lock;
+            IoRegionList m_io_region_list;
+            ams::svc::IoPoolType m_pool_type;
+            bool m_is_initialized;
         public:
-            explicit KBeta()
-                : m_process_list_node()
-            {
+            static bool IsValidIoPoolType(ams::svc::IoPoolType pool_type);
+        public:
+            explicit KIoPool() : m_lock(), m_io_region_list(), m_is_initialized(false) {
                 /* ... */
             }
 
-            virtual ~KBeta() { /* ... */ }
+            Result Initialize(ams::svc::IoPoolType pool_type);
+            virtual void Finalize() override;
 
-            /* virtual void Finalize() override; */
-
-            virtual bool IsInitialized() const override { return false /* TODO */; }
+            virtual bool IsInitialized() const override { return m_is_initialized; }
             static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
+
+            Result AddIoRegion(KIoRegion *region);
+            void RemoveIoRegion(KIoRegion *region);
     };
 
 }

libraries/libmesosphere/include/mesosphere/kern_k_io_region.hpp

@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_k_auto_object.hpp>
+#include <mesosphere/kern_slab_helpers.hpp>
+
+namespace ams::kern {
+
+    class KProcess;
+    class KIoPool;
+
+    class KIoRegion final : public KAutoObjectWithSlabHeapAndContainer<KIoRegion, KAutoObjectWithList> {
+        MESOSPHERE_AUTOOBJECT_TRAITS(KIoRegion, KAutoObject);
+        private:
+            friend class KProcess;
+            friend class KIoPool;
+        private:
+            KLightLock m_lock;
+            KIoPool *m_pool;
+            KPhysicalAddress m_physical_address;
+            size_t m_size;
+            ams::svc::MemoryMapping m_mapping;
+            ams::svc::MemoryPermission m_perm;
+            bool m_is_initialized;
+            bool m_is_mapped;
+            util::IntrusiveListNode m_process_list_node;
+            util::IntrusiveListNode m_pool_list_node;
+        public:
+            explicit KIoRegion()
+                : m_lock(), m_pool(nullptr), m_is_initialized(false), m_process_list_node(), m_pool_list_node()
+            {
+                /* ... */
+            }
+
+            Result Initialize(KIoPool *pool, KPhysicalAddress phys_addr, size_t size, ams::svc::MemoryMapping mapping, ams::svc::MemoryPermission perm);
+            virtual void Finalize() override;
+
+            virtual bool IsInitialized() const override { return m_is_initialized; }
+            static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
+
+            Result Map(KProcessAddress address, size_t size, ams::svc::MemoryPermission map_perm);
+            Result Unmap(KProcessAddress address, size_t size);
+
+            bool Overlaps(KPhysicalAddress address, size_t size) const {
+                return m_physical_address <= (address + size - 1) && address <= (m_physical_address + m_size - 1);
+            }
+
+            ALWAYS_INLINE KPhysicalAddress GetAddress() const { return m_physical_address; }
+            ALWAYS_INLINE size_t GetSize() const { return m_size; }
+    };
+
+}

libraries/libmesosphere/include/mesosphere/kern_k_light_client_session.hpp

@@ -21,13 +21,12 @@ namespace ams::kern {
 
     class KLightSession;
 
-    class KLightClientSession final : public KAutoObjectWithSlabHeapAndContainer<KLightClientSession, KAutoObjectWithList> {
+    class KLightClientSession final : public KAutoObject {
         MESOSPHERE_AUTOOBJECT_TRAITS(KLightClientSession, KAutoObject);
         private:
             KLightSession *m_parent;
         public:
             constexpr KLightClientSession() : m_parent() { /* ... */ }
-            virtual ~KLightClientSession() { /* ... */ }
 
             void Initialize(KLightSession *parent) {
                 /* Set member variables. */

libraries/libmesosphere/include/mesosphere/kern_k_light_condition_variable.hpp

@@ -27,59 +27,9 @@ namespace ams::kern {
             KThread::WaiterList m_wait_list;
         public:
             constexpr ALWAYS_INLINE KLightConditionVariable() : m_wait_list() { /* ... */ }
-        private:
-            void WaitImpl(KLightLock *lock, s64 timeout, bool allow_terminating_thread) {
-                KThread *owner = GetCurrentThreadPointer();
-                KHardwareTimer *timer;
-
-                /* Sleep the thread. */
-                {
-                    KScopedSchedulerLockAndSleep lk(&timer, owner, timeout);
-
-                    if (!allow_terminating_thread && owner->IsTerminationRequested()) {
-                        lk.CancelSleep();
-                        return;
-                    }
-
-                    lock->Unlock();
-
-
-                    /* Set the thread as waiting. */
-                    GetCurrentThread().SetState(KThread::ThreadState_Waiting);
-
-                    /* Add the thread to the queue. */
-                    m_wait_list.push_back(GetCurrentThread());
-                }
-
-                /* Remove the thread from the wait list. */
-                {
-                    KScopedSchedulerLock sl;
-
-                    m_wait_list.erase(m_wait_list.iterator_to(GetCurrentThread()));
-                }
-
-                /* Cancel the task that the sleep setup. */
-                if (timer != nullptr) {
-                    timer->CancelTask(owner);
-                }
-
-                /* Re-acquire the lock. */
-                lock->Lock();
-            }
         public:
-            void Wait(KLightLock *lock, s64 timeout = -1ll, bool allow_terminating_thread = true) {
-                this->WaitImpl(lock, timeout, allow_terminating_thread);
-            }
-
-            void Broadcast() {
-                KScopedSchedulerLock lk;
-
-                /* Signal all threads. */
-                for (auto &thread : m_wait_list) {
-                    thread.SetState(KThread::ThreadState_Runnable);
-                }
-            }
-
+            void Wait(KLightLock *lock, s64 timeout = -1ll, bool allow_terminating_thread = true);
+            void Broadcast();
     };
 
 }

libraries/libmesosphere/include/mesosphere/kern_k_light_server_session.hpp

@@ -23,17 +23,16 @@ namespace ams::kern {
 
     class KLightSession;
 
-    class KLightServerSession final : public KAutoObjectWithSlabHeapAndContainer<KLightServerSession, KAutoObjectWithList>, public util::IntrusiveListBaseNode<KLightServerSession> {
+    class KLightServerSession final : public KAutoObject, public util::IntrusiveListBaseNode<KLightServerSession> {
         MESOSPHERE_AUTOOBJECT_TRAITS(KLightServerSession, KAutoObject);
         private:
             KLightSession *m_parent;
-            KThreadQueue m_request_queue;
-            KThreadQueue m_server_queue;
+            KThread::WaiterList m_request_list;
             KThread *m_current_request;
+            u64 m_server_thread_id;
             KThread *m_server_thread;
         public:
-            constexpr KLightServerSession() : m_parent(), m_request_queue(), m_server_queue(), m_current_request(), m_server_thread() { /* ... */ }
-            virtual ~KLightServerSession() { /* ... */ }
+            constexpr KLightServerSession() : m_parent(), m_request_list(), m_current_request(), m_server_thread_id(), m_server_thread() { /* ... */ }
 
             void Initialize(KLightSession *parent) {
                 /* Set member variables. */

libraries/libmesosphere/include/mesosphere/kern_k_light_session.hpp

@@ -25,7 +25,7 @@ namespace ams::kern {
     class KClientPort;
     class KProcess;
 
-    class KLightSession final : public KAutoObjectWithSlabHeapAndContainer<KLightSession, KAutoObjectWithList> {
+    class KLightSession final : public KAutoObjectWithSlabHeapAndContainer<KLightSession, KAutoObjectWithList, true> {
         MESOSPHERE_AUTOOBJECT_TRAITS(KLightSession, KAutoObject);
         private:
             enum class State : u8 {
@@ -52,8 +52,6 @@ namespace ams::kern {
                 /* ... */
             }
 
-            virtual ~KLightSession() { /* ... */ }
-
             void Initialize(KClientPort *client_port, uintptr_t name);
             virtual void Finalize() override;
 

libraries/libmesosphere/include/mesosphere/kern_k_memory_block.hpp

@@ -105,6 +105,8 @@ namespace ams::kern {
 
         KMemoryState_GeneratedCode      = ams::svc::MemoryState_GeneratedCode       | KMemoryState_FlagMapped | KMemoryState_FlagReferenceCounted    | KMemoryState_FlagCanDebug,
         KMemoryState_CodeOut            = ams::svc::MemoryState_CodeOut             | KMemoryState_FlagMapped | KMemoryState_FlagReferenceCounted,
+
+        KMemoryState_Coverage           = ams::svc::MemoryState_Coverage            | KMemoryState_FlagMapped,
     };
 
 #if 1
@@ -130,6 +132,7 @@ namespace ams::kern {
     static_assert(KMemoryState_Kernel           == 0x00002013);
     static_assert(KMemoryState_GeneratedCode    == 0x00402214);
     static_assert(KMemoryState_CodeOut          == 0x00402015);
+    static_assert(KMemoryState_Coverage         == 0x00002016);
 #endif
 
     enum KMemoryPermission : u8 {

libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp

@@ -144,10 +144,12 @@ namespace ams::kern {
             static NOINLINE const KMemoryRegion &GetPageTableHeapRegion()  { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_VirtualDramKernelPtHeap)); }
             static NOINLINE const KMemoryRegion &GetKernelStackRegion()    { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_KernelStack)); }
             static NOINLINE const KMemoryRegion &GetTempRegion()           { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_KernelTemp)); }
+            static NOINLINE const KMemoryRegion &GetSlabRegion()           { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_KernelSlab)); }
 
             static NOINLINE const KMemoryRegion &GetKernelTraceBufferRegion() { return Dereference(GetVirtualLinearMemoryRegionTree().FindByType(KMemoryRegionType_VirtualDramKernelTraceBuffer)); }
 
             static NOINLINE const KMemoryRegion &GetVirtualLinearRegion(KVirtualAddress address) { return Dereference(FindLinear(address)); }
+            static NOINLINE const KMemoryRegion &GetPhysicalLinearRegion(KPhysicalAddress address) { return Dereference(FindLinear(address)); }
 
             static NOINLINE const KMemoryRegion *GetPhysicalKernelTraceBufferRegion() { return GetPhysicalMemoryRegionTree().FindFirstDerived(KMemoryRegionType_KernelTraceBuffer); }
             static NOINLINE const KMemoryRegion *GetPhysicalOnMemoryBootImageRegion() { return GetPhysicalMemoryRegionTree().FindFirstDerived(KMemoryRegionType_OnMemoryBootImage); }

libraries/libmesosphere/include/mesosphere/kern_k_memory_manager.hpp

@@ -70,37 +70,37 @@ namespace ams::kern {
                 public:
                     Impl() : m_heap(), m_page_reference_counts(), m_management_region(), m_pool(), m_next(), m_prev() { /* ... */ }
 
-                    size_t Initialize(uintptr_t address, size_t size, KVirtualAddress management, KVirtualAddress management_end, Pool p);
+                    size_t Initialize(KPhysicalAddress address, size_t size, KVirtualAddress management, KVirtualAddress management_end, Pool p);
 
-                    KVirtualAddress AllocateBlock(s32 index, bool random) { return m_heap.AllocateBlock(index, random); }
-                    void Free(KVirtualAddress addr, size_t num_pages) { m_heap.Free(addr, num_pages); }
+                    KPhysicalAddress AllocateBlock(s32 index, bool random) { return m_heap.AllocateBlock(index, random); }
+                    void Free(KPhysicalAddress addr, size_t num_pages) { m_heap.Free(addr, num_pages); }
 
                     void SetInitialUsedHeapSize(size_t reserved_size) { m_heap.SetInitialUsedSize(reserved_size); }
 
                     void InitializeOptimizedMemory() { std::memset(GetVoidPointer(m_management_region), 0, CalculateOptimizedProcessOverheadSize(m_heap.GetSize())); }
 
-                    void TrackUnoptimizedAllocation(KVirtualAddress block, size_t num_pages);
-                    void TrackOptimizedAllocation(KVirtualAddress block, size_t num_pages);
+                    void TrackUnoptimizedAllocation(KPhysicalAddress block, size_t num_pages);
+                    void TrackOptimizedAllocation(KPhysicalAddress block, size_t num_pages);
 
-                    bool ProcessOptimizedAllocation(KVirtualAddress block, size_t num_pages, u8 fill_pattern);
+                    bool ProcessOptimizedAllocation(KPhysicalAddress block, size_t num_pages, u8 fill_pattern);
 
                     constexpr Pool GetPool() const { return m_pool; }
                     constexpr size_t GetSize() const { return m_heap.GetSize(); }
-                    constexpr KVirtualAddress GetEndAddress() const { return m_heap.GetEndAddress(); }
+                    constexpr KPhysicalAddress GetEndAddress() const { return m_heap.GetEndAddress(); }
 
                     size_t GetFreeSize() const { return m_heap.GetFreeSize(); }
 
                     void DumpFreeList() const { return m_heap.DumpFreeList(); }
 
-                    constexpr size_t GetPageOffset(KVirtualAddress address)      const { return m_heap.GetPageOffset(address); }
-                    constexpr size_t GetPageOffsetToEnd(KVirtualAddress address) const { return m_heap.GetPageOffsetToEnd(address); }
+                    constexpr size_t GetPageOffset(KPhysicalAddress address)      const { return m_heap.GetPageOffset(address); }
+                    constexpr size_t GetPageOffsetToEnd(KPhysicalAddress address) const { return m_heap.GetPageOffsetToEnd(address); }
 
                     constexpr void SetNext(Impl *n) { m_next = n; }
                     constexpr void SetPrev(Impl *n) { m_prev = n; }
                     constexpr Impl *GetNext() const { return m_next; }
                     constexpr Impl *GetPrev() const { return m_prev; }
 
-                    void OpenFirst(KVirtualAddress address, size_t num_pages) {
+                    void OpenFirst(KPhysicalAddress address, size_t num_pages) {
                         size_t index = this->GetPageOffset(address);
                         const size_t end = index + num_pages;
                         while (index < end) {
@@ -111,7 +111,7 @@ namespace ams::kern {
                         }
                     }
 
-                    void Open(KVirtualAddress address, size_t num_pages) {
+                    void Open(KPhysicalAddress address, size_t num_pages) {
                         size_t index = this->GetPageOffset(address);
                         const size_t end = index + num_pages;
                         while (index < end) {
@@ -122,7 +122,7 @@ namespace ams::kern {
                         }
                     }
 
-                    void Close(KVirtualAddress address, size_t num_pages) {
+                    void Close(KPhysicalAddress address, size_t num_pages) {
                         size_t index = this->GetPageOffset(address);
                         const size_t end = index + num_pages;
 
@@ -164,12 +164,12 @@ namespace ams::kern {
             u64 m_optimized_process_ids[Pool_Count];
             bool m_has_optimized_process[Pool_Count];
         private:
-            Impl &GetManager(KVirtualAddress address) {
-                return m_managers[KMemoryLayout::GetVirtualLinearRegion(address).GetAttributes()];
+            Impl &GetManager(KPhysicalAddress address) {
+                return m_managers[KMemoryLayout::GetPhysicalLinearRegion(address).GetAttributes()];
             }
 
-            const Impl &GetManager(KVirtualAddress address) const {
-                return m_managers[KMemoryLayout::GetVirtualLinearRegion(address).GetAttributes()];
+            const Impl &GetManager(KPhysicalAddress address) const {
+                return m_managers[KMemoryLayout::GetPhysicalLinearRegion(address).GetAttributes()];
             }
 
             constexpr Impl *GetFirstManager(Pool pool, Direction dir) {
@@ -197,15 +197,15 @@ namespace ams::kern {
             NOINLINE Result InitializeOptimizedMemory(u64 process_id, Pool pool);
             NOINLINE void FinalizeOptimizedMemory(u64 process_id, Pool pool);
 
-            NOINLINE KVirtualAddress AllocateAndOpenContinuous(size_t num_pages, size_t align_pages, u32 option);
+            NOINLINE KPhysicalAddress AllocateAndOpenContinuous(size_t num_pages, size_t align_pages, u32 option);
             NOINLINE Result AllocateAndOpen(KPageGroup *out, size_t num_pages, u32 option);
             NOINLINE Result AllocateAndOpenForProcess(KPageGroup *out, size_t num_pages, u32 option, u64 process_id, u8 fill_pattern);
 
-            Pool GetPool(KVirtualAddress address) const {
+            Pool GetPool(KPhysicalAddress address) const {
                 return this->GetManager(address).GetPool();
             }
 
-            void Open(KVirtualAddress address, size_t num_pages) {
+            void Open(KPhysicalAddress address, size_t num_pages) {
                 /* Repeatedly open references until we've done so for all pages. */
                 while (num_pages) {
                     auto &manager = this->GetManager(address);
@@ -221,7 +221,7 @@ namespace ams::kern {
                 }
             }
 
-            void Close(KVirtualAddress address, size_t num_pages) {
+            void Close(KPhysicalAddress address, size_t num_pages) {
                 /* Repeatedly close references until we've done so for all pages. */
                 while (num_pages) {
                     auto &manager = this->GetManager(address);

libraries/libmesosphere/include/mesosphere/kern_k_memory_region.hpp

@@ -237,12 +237,6 @@ namespace ams::kern {
         public:
             NOINLINE void InsertDirectly(uintptr_t address, uintptr_t last_address, u32 attr = 0, u32 type_id = 0);
             NOINLINE bool Insert(uintptr_t address, size_t size, u32 type_id, u32 new_attr = 0, u32 old_attr = 0);
-
-            NOINLINE KVirtualAddress GetRandomAlignedRegion(size_t size, size_t alignment, u32 type_id);
-
-            ALWAYS_INLINE KVirtualAddress GetRandomAlignedRegionWithGuard(size_t size, size_t alignment, u32 type_id, size_t guard_size) {
-                return this->GetRandomAlignedRegion(size + 2 * guard_size, alignment, type_id) + guard_size;
-            }
         public:
             /* Iterator accessors. */
             iterator begin() {

libraries/libmesosphere/include/mesosphere/kern_k_page_group.hpp

@@ -22,77 +22,121 @@ namespace ams::kern {
 
     class KBlockInfoManager;
 
-    class KBlockInfo : public util::IntrusiveListBaseNode<KBlockInfo> {
+    class KPageGroup;
+
+    class KBlockInfo {
         private:
-            KVirtualAddress m_address;
-            size_t m_num_pages;
+            friend class KPageGroup;
+        private:
+            KBlockInfo *m_next{};
+            u32 m_page_index{};
+            u32 m_num_pages{};
         public:
-            constexpr KBlockInfo() : util::IntrusiveListBaseNode<KBlockInfo>(), m_address(), m_num_pages() { /* ... */ }
+            constexpr KBlockInfo() = default;
 
-            constexpr void Initialize(KVirtualAddress addr, size_t np) {
-                m_address = addr;
-                m_num_pages = np;
+            constexpr ALWAYS_INLINE void Initialize(KPhysicalAddress addr, size_t np) {
+                MESOSPHERE_ASSERT(util::IsAligned(GetInteger(addr), PageSize));
+                MESOSPHERE_ASSERT(static_cast<u32>(np) == np);
+
+                m_page_index = GetInteger(addr) / PageSize;
+                m_num_pages  = np;
             }
 
-            constexpr KVirtualAddress GetAddress() const { return m_address; }
-            constexpr size_t GetNumPages() const { return m_num_pages; }
-            constexpr size_t GetSize() const { return this->GetNumPages() * PageSize; }
-            constexpr KVirtualAddress GetEndAddress() const { return this->GetAddress() + this->GetSize(); }
-            constexpr KVirtualAddress GetLastAddress() const { return this->GetEndAddress() - 1; }
+            constexpr ALWAYS_INLINE KPhysicalAddress GetAddress() const { return m_page_index * PageSize; }
+            constexpr ALWAYS_INLINE size_t GetNumPages() const { return m_num_pages; }
+            constexpr ALWAYS_INLINE size_t GetSize() const { return this->GetNumPages() * PageSize; }
+            constexpr ALWAYS_INLINE KPhysicalAddress GetEndAddress() const { return (m_page_index + m_num_pages) * PageSize; }
+            constexpr ALWAYS_INLINE KPhysicalAddress GetLastAddress() const { return this->GetEndAddress() - 1; }
 
-            constexpr bool IsEquivalentTo(const KBlockInfo &rhs) const {
-                return m_address == rhs.m_address && m_num_pages == rhs.m_num_pages;
+            constexpr ALWAYS_INLINE KBlockInfo *GetNext() const { return m_next; }
+
+            constexpr ALWAYS_INLINE bool IsEquivalentTo(const KBlockInfo &rhs) const {
+                return m_page_index == rhs.m_page_index && m_num_pages == rhs.m_num_pages;
             }
 
-            constexpr bool operator==(const KBlockInfo &rhs) const {
+            constexpr ALWAYS_INLINE bool operator==(const KBlockInfo &rhs) const {
                 return this->IsEquivalentTo(rhs);
             }
 
-            constexpr bool operator!=(const KBlockInfo &rhs) const {
+            constexpr ALWAYS_INLINE bool operator!=(const KBlockInfo &rhs) const {
                 return !(*this == rhs);
             }
 
-            constexpr bool IsStrictlyBefore(KVirtualAddress addr) const {
-                const KVirtualAddress end = this->GetEndAddress();
+            constexpr ALWAYS_INLINE bool IsStrictlyBefore(KPhysicalAddress addr) const {
+                const KPhysicalAddress end = this->GetEndAddress();
 
-                if (m_address != Null<KVirtualAddress> && end == Null<KVirtualAddress>) {
+                if (m_page_index != 0 && end == Null<KPhysicalAddress>) {
                     return false;
                 }
 
                 return end < addr;
             }
 
-            constexpr bool operator<(KVirtualAddress addr) const {
+            constexpr ALWAYS_INLINE bool operator<(KPhysicalAddress addr) const {
                 return this->IsStrictlyBefore(addr);
             }
 
-            constexpr bool TryConcatenate(KVirtualAddress addr, size_t np) {
-                if (addr != Null<KVirtualAddress> && addr == this->GetEndAddress()) {
+            constexpr ALWAYS_INLINE bool TryConcatenate(KPhysicalAddress addr, size_t np) {
+                if (addr != Null<KPhysicalAddress> && addr == this->GetEndAddress()) {
                     m_num_pages += np;
                     return true;
                 }
                 return false;
             }
+        private:
+            constexpr ALWAYS_INLINE void SetNext(KBlockInfo *next) {
+                m_next = next;
+            }
     };
+    static_assert(sizeof(KBlockInfo) <= 0x10);
 
     class KPageGroup {
         public:
-            using BlockInfoList = util::IntrusiveListBaseTraits<KBlockInfo>::ListType;
-            using iterator = BlockInfoList::const_iterator;
+            class Iterator {
+                public:
+                    using iterator_category = std::forward_iterator_tag;
+                    using value_type        = const KBlockInfo;
+                    using difference_type   = std::ptrdiff_t;
+                    using pointer           = value_type *;
+                    using reference         = value_type &;
+                private:
+                    pointer m_node;
+                public:
+                    constexpr explicit ALWAYS_INLINE Iterator(pointer n) : m_node(n) { /* ... */ }
+
+                    constexpr ALWAYS_INLINE bool operator==(const Iterator &rhs) const { return m_node == rhs.m_node; }
+                    constexpr ALWAYS_INLINE bool operator!=(const Iterator &rhs) const { return !(*this == rhs); }
+
+                    constexpr ALWAYS_INLINE pointer operator->() const { return m_node; }
+                    constexpr ALWAYS_INLINE reference operator*() const { return *m_node; }
+
+                    constexpr ALWAYS_INLINE Iterator &operator++() {
+                        m_node = m_node->GetNext();
+                        return *this;
+                    }
+
+                    constexpr ALWAYS_INLINE Iterator operator++(int) {
+                        const Iterator it{*this};
+                        ++(*this);
+                        return it;
+                    }
+            };
         private:
-            BlockInfoList m_block_list;
+            KBlockInfo *m_first_block;
+            KBlockInfo *m_last_block;
             KBlockInfoManager *m_manager;
         public:
-            explicit KPageGroup(KBlockInfoManager *m) : m_block_list(), m_manager(m) { /* ... */ }
+            explicit KPageGroup(KBlockInfoManager *m) : m_first_block(), m_last_block(), m_manager(m) { /* ... */ }
             ~KPageGroup() { this->Finalize(); }
 
+            void CloseAndReset();
             void Finalize();
 
-            iterator begin() const { return m_block_list.begin(); }
-            iterator end() const { return m_block_list.end(); }
-            bool empty() const { return m_block_list.empty(); }
+            ALWAYS_INLINE Iterator begin() const { return Iterator{m_first_block}; }
+            ALWAYS_INLINE Iterator end() const { return Iterator{nullptr}; }
+            ALWAYS_INLINE bool empty() const { return m_first_block == nullptr; }
 
-            Result AddBlock(KVirtualAddress addr, size_t num_pages);
+            Result AddBlock(KPhysicalAddress addr, size_t num_pages);
             void Open() const;
             void Close() const;
 
@@ -100,11 +144,11 @@ namespace ams::kern {
 
             bool IsEquivalentTo(const KPageGroup &rhs) const;
 
-            bool operator==(const KPageGroup &rhs) const {
+            ALWAYS_INLINE bool operator==(const KPageGroup &rhs) const {
                 return this->IsEquivalentTo(rhs);
             }
 
-            bool operator!=(const KPageGroup &rhs) const {
+            ALWAYS_INLINE bool operator!=(const KPageGroup &rhs) const {
                 return !(*this == rhs);
             }
     };

libraries/libmesosphere/include/mesosphere/kern_k_page_heap.hpp

@@ -54,7 +54,7 @@ namespace ams::kern {
             class Block {
                 private:
                     KPageBitmap m_bitmap;
-                    KVirtualAddress m_heap_address;
+                    KPhysicalAddress m_heap_address;
                     uintptr_t m_end_offset;
                     size_t m_block_shift;
                     size_t m_next_block_shift;
@@ -68,13 +68,13 @@ namespace ams::kern {
                     constexpr size_t GetNumFreeBlocks() const { return m_bitmap.GetNumBits(); }
                     constexpr size_t GetNumFreePages() const { return this->GetNumFreeBlocks() * this->GetNumPages(); }
 
-                    u64 *Initialize(KVirtualAddress addr, size_t size, size_t bs, size_t nbs, u64 *bit_storage) {
+                    u64 *Initialize(KPhysicalAddress addr, size_t size, size_t bs, size_t nbs, u64 *bit_storage) {
                         /* Set shifts. */
                         m_block_shift = bs;
                         m_next_block_shift = nbs;
 
                         /* Align up the address. */
-                        KVirtualAddress end = addr + size;
+                        KPhysicalAddress end = addr + size;
                         const size_t align = (m_next_block_shift != 0) ? (u64(1) << m_next_block_shift) : (u64(1) << m_block_shift);
                         addr = util::AlignDown(GetInteger(addr), align);
                         end  = util::AlignUp(GetInteger(end), align);
@@ -84,7 +84,7 @@ namespace ams::kern {
                         return m_bitmap.Initialize(bit_storage, m_end_offset);
                     }
 
-                    KVirtualAddress PushBlock(KVirtualAddress address) {
+                    KPhysicalAddress PushBlock(KPhysicalAddress address) {
                         /* Set the bit for the free block. */
                         size_t offset = (address - m_heap_address) >> this->GetShift();
                         m_bitmap.SetBit(offset);
@@ -99,14 +99,14 @@ namespace ams::kern {
                         }
 
                         /* We couldn't coalesce, or we're already as big as possible. */
-                        return Null<KVirtualAddress>;
+                        return Null<KPhysicalAddress>;
                     }
 
-                    KVirtualAddress PopBlock(bool random) {
+                    KPhysicalAddress PopBlock(bool random) {
                         /* Find a free block. */
                         ssize_t soffset = m_bitmap.FindFreeBlock(random);
                         if (soffset < 0) {
-                            return Null<KVirtualAddress>;
+                            return Null<KPhysicalAddress>;
                         }
                         const size_t offset = static_cast<size_t>(soffset);
 
@@ -123,27 +123,27 @@ namespace ams::kern {
                     }
             };
         private:
-            KVirtualAddress m_heap_address;
+            KPhysicalAddress m_heap_address;
             size_t m_heap_size;
             size_t m_initial_used_size;
             size_t m_num_blocks;
             Block m_blocks[NumMemoryBlockPageShifts];
         private:
-            void Initialize(KVirtualAddress heap_address, size_t heap_size, KVirtualAddress management_address, size_t management_size, const size_t *block_shifts, size_t num_block_shifts);
+            void Initialize(KPhysicalAddress heap_address, size_t heap_size, KVirtualAddress management_address, size_t management_size, const size_t *block_shifts, size_t num_block_shifts);
             size_t GetNumFreePages() const;
 
-            void FreeBlock(KVirtualAddress block, s32 index);
+            void FreeBlock(KPhysicalAddress block, s32 index);
         public:
             KPageHeap() : m_heap_address(), m_heap_size(), m_initial_used_size(), m_num_blocks(), m_blocks() { /* ... */ }
 
-            constexpr KVirtualAddress GetAddress() const { return m_heap_address; }
+            constexpr KPhysicalAddress GetAddress() const { return m_heap_address; }
             constexpr size_t GetSize() const { return m_heap_size; }
-            constexpr KVirtualAddress GetEndAddress() const { return this->GetAddress() + this->GetSize(); }
-            constexpr size_t GetPageOffset(KVirtualAddress block) const { return (block - this->GetAddress()) / PageSize; }
-            constexpr size_t GetPageOffsetToEnd(KVirtualAddress block) const { return (this->GetEndAddress() - block) / PageSize; }
+            constexpr KPhysicalAddress GetEndAddress() const { return this->GetAddress() + this->GetSize(); }
+            constexpr size_t GetPageOffset(KPhysicalAddress block) const { return (block - this->GetAddress()) / PageSize; }
+            constexpr size_t GetPageOffsetToEnd(KPhysicalAddress block) const { return (this->GetEndAddress() - block) / PageSize; }
 
-            void Initialize(KVirtualAddress heap_address, size_t heap_size, KVirtualAddress management_address, size_t management_size) {
-                return Initialize(heap_address, heap_size, management_address, management_size, MemoryBlockPageShifts, NumMemoryBlockPageShifts);
+            void Initialize(KPhysicalAddress heap_address, size_t heap_size, KVirtualAddress management_address, size_t management_size) {
+                return this->Initialize(heap_address, heap_size, management_address, management_size, MemoryBlockPageShifts, NumMemoryBlockPageShifts);
             }
 
             size_t GetFreeSize() const { return this->GetNumFreePages() * PageSize; }
@@ -158,8 +158,8 @@ namespace ams::kern {
                 m_initial_used_size = m_heap_size - free_size - reserved_size;
             }
 
-            KVirtualAddress AllocateBlock(s32 index, bool random);
-            void Free(KVirtualAddress addr, size_t num_pages);
+            KPhysicalAddress AllocateBlock(s32 index, bool random);
+            void Free(KPhysicalAddress addr, size_t num_pages);
         private:
             static size_t CalculateManagementOverheadSize(size_t region_size, const size_t *block_shifts, size_t num_block_shifts);
         public:

libraries/libmesosphere/include/mesosphere/kern_k_page_table_base.hpp

@@ -57,7 +57,7 @@ namespace ams::kern {
             using TraversalContext = KPageTableImpl::TraversalContext;
 
             struct MemoryRange {
-                KVirtualAddress address;
+                KPhysicalAddress address;
                 size_t size;
 
                 void Close();
@@ -178,7 +178,6 @@ namespace ams::kern {
             KResourceLimit *m_resource_limit{};
             const KMemoryRegion *m_cached_physical_linear_region{};
             const KMemoryRegion *m_cached_physical_heap_region{};
-            const KMemoryRegion *m_cached_virtual_heap_region{};
             MemoryFillValue m_heap_fill_value{};
             MemoryFillValue m_ipc_fill_value{};
             MemoryFillValue m_stack_fill_value{};
@@ -218,9 +217,13 @@ namespace ams::kern {
             size_t GetRegionSize(KMemoryState state) const;
             bool CanContain(KProcessAddress addr, size_t size, KMemoryState state) const;
         protected:
-            virtual Result Operate(PageLinkedList *page_list, KProcessAddress virt_addr, size_t num_pages, KPhysicalAddress phys_addr, bool is_pa_valid, const KPageProperties properties, OperationType operation, bool reuse_ll) = 0;
-            virtual Result Operate(PageLinkedList *page_list, KProcessAddress virt_addr, size_t num_pages, const KPageGroup &page_group, const KPageProperties properties, OperationType operation, bool reuse_ll) = 0;
-            virtual void   FinalizeUpdate(PageLinkedList *page_list) = 0;
+            /* NOTE: These three functions (Operate, Operate, FinalizeUpdate) are virtual functions */
+            /* in Nintendo's kernel. We devirtualize them, since KPageTable is the only derived */
+            /* class, and this avoids unnecessary virtual function calls. See "kern_select_page_table.hpp" */
+            /* for definition of these functions. */
+            Result Operate(PageLinkedList *page_list, KProcessAddress virt_addr, size_t num_pages, KPhysicalAddress phys_addr, bool is_pa_valid, const KPageProperties properties, OperationType operation, bool reuse_ll);
+            Result Operate(PageLinkedList *page_list, KProcessAddress virt_addr, size_t num_pages, const KPageGroup &page_group, const KPageProperties properties, OperationType operation, bool reuse_ll);
+            void FinalizeUpdate(PageLinkedList *page_list);
 
             ALWAYS_INLINE KPageTableImpl &GetImpl() { return m_impl; }
             ALWAYS_INLINE const KPageTableImpl &GetImpl() const { return m_impl; }
@@ -257,18 +260,6 @@ namespace ams::kern {
                 return KMemoryLayout::IsHeapPhysicalAddress(m_cached_physical_heap_region, phys_addr);
             }
 
-            ALWAYS_INLINE bool IsHeapVirtualAddress(KVirtualAddress virt_addr) {
-                MESOSPHERE_ASSERT(this->IsLockedByCurrentThread());
-
-                return KMemoryLayout::IsHeapVirtualAddress(m_cached_virtual_heap_region, virt_addr);
-            }
-
-            ALWAYS_INLINE bool IsHeapVirtualAddress(KVirtualAddress virt_addr, size_t size) {
-                MESOSPHERE_ASSERT(this->IsLockedByCurrentThread());
-
-                return KMemoryLayout::IsHeapVirtualAddress(m_cached_virtual_heap_region, virt_addr, size);
-            }
-
             ALWAYS_INLINE bool ContainsPages(KProcessAddress addr, size_t num_pages) const {
                 return (m_address_space_start <= addr) && (num_pages <= (m_address_space_end - m_address_space_start) / PageSize) && (addr + num_pages * PageSize - 1 <= m_address_space_end - 1);
             }
@@ -352,6 +343,8 @@ namespace ams::kern {
             Result MapCodeMemory(KProcessAddress dst_address, KProcessAddress src_address, size_t size);
             Result UnmapCodeMemory(KProcessAddress dst_address, KProcessAddress src_address, size_t size);
             Result MapIo(KPhysicalAddress phys_addr, size_t size, KMemoryPermission perm);
+            Result MapIoRegion(KProcessAddress dst_address, KPhysicalAddress phys_addr, size_t size, ams::svc::MemoryMapping mapping, ams::svc::MemoryPermission perm);
+            Result UnmapIoRegion(KProcessAddress dst_address, KPhysicalAddress phys_addr, size_t size);
             Result MapStatic(KPhysicalAddress phys_addr, size_t size, KMemoryPermission perm);
             Result MapRegion(KMemoryRegionType region_type, KMemoryPermission perm);
 
@@ -388,7 +381,7 @@ namespace ams::kern {
             Result LockForUnmapDeviceAddressSpace(KProcessAddress address, size_t size);
 
             Result UnlockForDeviceAddressSpace(KProcessAddress address, size_t size);
-            Result UnlockForDeviceAddressSpacePartialMap(KProcessAddress address, size_t size, size_t mapped_size);
+            Result UnlockForDeviceAddressSpacePartialMap(KProcessAddress address, size_t size);
 
             Result OpenMemoryRangeForMapDeviceAddressSpace(KPageTableBase::MemoryRange *out, KProcessAddress address, size_t size, KMemoryPermission perm, bool is_aligned);
             Result OpenMemoryRangeForUnmapDeviceAddressSpace(MemoryRange *out, KProcessAddress address, size_t size);

libraries/libmesosphere/include/mesosphere/kern_k_page_table_manager.hpp

@@ -15,58 +15,27 @@
  */
 #pragma once
 #include <mesosphere/kern_common.hpp>
-#include <mesosphere/kern_slab_helpers.hpp>
-#include <mesosphere/kern_k_dynamic_slab_heap.hpp>
+#include <mesosphere/kern_k_page_table_slab_heap.hpp>
+#include <mesosphere/kern_k_dynamic_resource_manager.hpp>
 
 namespace ams::kern {
 
-    namespace impl {
-
-        class PageTablePage {
-            private:
-                u8 m_buffer[PageSize];
-            public:
-                ALWAYS_INLINE PageTablePage() { /* Do not initialize anything. */ }
-        };
-        static_assert(sizeof(PageTablePage) == PageSize);
-
-    }
-
-    class KPageTableManager : public KDynamicSlabHeap<impl::PageTablePage, true> {
+    class KPageTableManager : public KDynamicResourceManager<impl::PageTablePage, true> {
         public:
-            using RefCount = u16;
-            static constexpr size_t PageTableSize = sizeof(impl::PageTablePage);
-            static_assert(PageTableSize == PageSize);
+            using RefCount = KPageTableSlabHeap::RefCount;
+            static constexpr size_t PageTableSize = KPageTableSlabHeap::PageTableSize;
         private:
-            using BaseHeap = KDynamicSlabHeap<impl::PageTablePage, true>;
+            using BaseHeap = KDynamicResourceManager<impl::PageTablePage, true>;
         private:
-            RefCount *m_ref_counts;
+            KPageTableSlabHeap *m_pt_heap{};
         public:
-            static constexpr size_t CalculateReferenceCountSize(size_t size) {
-                return (size / PageSize) * sizeof(RefCount);
-            }
-        public:
-            constexpr KPageTableManager() : BaseHeap(), m_ref_counts() { /* ... */ }
-        private:
-            void Initialize(RefCount *rc) {
-                m_ref_counts = rc;
-                for (size_t i = 0; i < this->GetSize() / PageSize; i++) {
-                    m_ref_counts[i] = 0;
-                }
-            }
+            constexpr KPageTableManager() = default;
 
-            constexpr RefCount *GetRefCountPointer(KVirtualAddress addr) const {
-                return std::addressof(m_ref_counts[(addr - this->GetAddress()) / PageSize]);
-            }
-        public:
-            void Initialize(KDynamicPageManager *page_allocator, RefCount *rc) {
-                BaseHeap::Initialize(page_allocator);
-                this->Initialize(rc);
-            }
+            ALWAYS_INLINE void Initialize(KDynamicPageManager *page_allocator, KPageTableSlabHeap *pt_heap) {
+                m_pt_heap = pt_heap;
 
-            void Initialize(KDynamicPageManager *page_allocator, size_t object_count, RefCount *rc) {
-                BaseHeap::Initialize(page_allocator, object_count);
-                this->Initialize(rc);
+                static_assert(std::derived_from<KPageTableSlabHeap, DynamicSlabType>);
+                BaseHeap::Initialize(page_allocator, pt_heap);
             }
 
             KVirtualAddress Allocate() {
@@ -74,33 +43,23 @@ namespace ams::kern {
             }
 
             void Free(KVirtualAddress addr) {
-                /* Free the page. */
-                BaseHeap::Free(GetPointer<impl::PageTablePage>(addr));
+                return BaseHeap::Free(GetPointer<impl::PageTablePage>(addr));
             }
 
-            RefCount GetRefCount(KVirtualAddress addr) const {
-                MESOSPHERE_ASSERT(this->IsInRange(addr));
-                return *this->GetRefCountPointer(addr);
+            ALWAYS_INLINE RefCount GetRefCount(KVirtualAddress addr) const {
+                return m_pt_heap->GetRefCount(addr);
             }
 
-            void Open(KVirtualAddress addr, int count) {
-                MESOSPHERE_ASSERT(this->IsInRange(addr));
-
-                *this->GetRefCountPointer(addr) += count;
-
-                MESOSPHERE_ABORT_UNLESS(this->GetRefCount(addr) > 0);
+            ALWAYS_INLINE void Open(KVirtualAddress addr, int count) {
+                return m_pt_heap->Open(addr, count);
             }
 
-            bool Close(KVirtualAddress addr, int count) {
-                MESOSPHERE_ASSERT(this->IsInRange(addr));
-                MESOSPHERE_ABORT_UNLESS(this->GetRefCount(addr) >= count);
-
-                *this->GetRefCountPointer(addr) -= count;
-                return this->GetRefCount(addr) == 0;
+            ALWAYS_INLINE bool Close(KVirtualAddress addr, int count) {
+                return m_pt_heap->Close(addr, count);
             }
 
-            constexpr bool IsInPageTableHeap(KVirtualAddress addr) const {
-                return this->IsInRange(addr);
+            constexpr ALWAYS_INLINE bool IsInPageTableHeap(KVirtualAddress addr) const {
+                return m_pt_heap->IsInRange(addr);
             }
     };
 

libraries/libmesosphere/include/mesosphere/kern_k_page_table_slab_heap.hpp

@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_slab_helpers.hpp>
+#include <mesosphere/kern_k_dynamic_slab_heap.hpp>
+
+namespace ams::kern {
+
+    namespace impl {
+
+        class PageTablePage {
+            private:
+                u8 m_buffer[PageSize];
+            public:
+                ALWAYS_INLINE PageTablePage() { /* Do not initialize anything. */ }
+        };
+        static_assert(sizeof(PageTablePage) == PageSize);
+
+    }
+
+    class KPageTableSlabHeap : public KDynamicSlabHeap<impl::PageTablePage, true> {
+        public:
+            using RefCount = u16;
+            static constexpr size_t PageTableSize = sizeof(impl::PageTablePage);
+            static_assert(PageTableSize == PageSize);
+        private:
+            using BaseHeap = KDynamicSlabHeap<impl::PageTablePage, true>;
+        private:
+            RefCount *m_ref_counts{};
+        public:
+            static constexpr ALWAYS_INLINE size_t CalculateReferenceCountSize(size_t size) {
+                return (size / PageSize) * sizeof(RefCount);
+            }
+        public:
+            constexpr KPageTableSlabHeap() = default;
+        private:
+            ALWAYS_INLINE void Initialize(RefCount *rc) {
+                m_ref_counts = rc;
+                for (size_t i = 0; i < this->GetSize() / PageSize; i++) {
+                    m_ref_counts[i] = 0;
+                }
+            }
+
+            constexpr ALWAYS_INLINE RefCount *GetRefCountPointer(KVirtualAddress addr) const {
+                return m_ref_counts + ((addr - this->GetAddress()) / PageSize);
+            }
+        public:
+            ALWAYS_INLINE void Initialize(KDynamicPageManager *page_allocator, size_t object_count, RefCount *rc) {
+                BaseHeap::Initialize(page_allocator, object_count);
+                this->Initialize(rc);
+            }
+
+            ALWAYS_INLINE RefCount GetRefCount(KVirtualAddress addr) const {
+                MESOSPHERE_ASSERT(this->IsInRange(addr));
+                return *this->GetRefCountPointer(addr);
+            }
+
+            ALWAYS_INLINE void Open(KVirtualAddress addr, int count) {
+                MESOSPHERE_ASSERT(this->IsInRange(addr));
+
+                *this->GetRefCountPointer(addr) += count;
+
+                MESOSPHERE_ABORT_UNLESS(this->GetRefCount(addr) > 0);
+            }
+
+            ALWAYS_INLINE bool Close(KVirtualAddress addr, int count) {
+                MESOSPHERE_ASSERT(this->IsInRange(addr));
+                MESOSPHERE_ABORT_UNLESS(this->GetRefCount(addr) >= count);
+
+                *this->GetRefCountPointer(addr) -= count;
+                return this->GetRefCount(addr) == 0;
+            }
+
+            constexpr ALWAYS_INLINE bool IsInPageTableHeap(KVirtualAddress addr) const {
+                return this->IsInRange(addr);
+            }
+    };
+
+}

libraries/libmesosphere/include/mesosphere/kern_k_port.hpp

@@ -42,7 +42,6 @@ namespace ams::kern {
             bool m_is_light;
         public:
             constexpr KPort() : m_server(), m_client(), m_name(), m_state(State::Invalid), m_is_light() { /* ... */ }
-            virtual ~KPort() { /* ... */ }
 
             static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
 

libraries/libmesosphere/include/mesosphere/kern_k_process.hpp

@@ -22,14 +22,14 @@
 #include <mesosphere/kern_k_thread.hpp>
 #include <mesosphere/kern_k_thread_local_page.hpp>
 #include <mesosphere/kern_k_shared_memory_info.hpp>
-#include <mesosphere/kern_k_beta.hpp>
+#include <mesosphere/kern_k_io_region.hpp>
 #include <mesosphere/kern_k_worker_task.hpp>
 #include <mesosphere/kern_select_page_table.hpp>
 #include <mesosphere/kern_k_condition_variable.hpp>
 #include <mesosphere/kern_k_address_arbiter.hpp>
 #include <mesosphere/kern_k_capabilities.hpp>
 #include <mesosphere/kern_k_wait_object.hpp>
-#include <mesosphere/kern_k_dynamic_slab_heap.hpp>
+#include <mesosphere/kern_k_dynamic_resource_manager.hpp>
 #include <mesosphere/kern_k_page_table_manager.hpp>
 
 namespace ams::kern {
@@ -53,7 +53,7 @@ namespace ams::kern {
             static constexpr size_t AslrAlignment = KernelAslrAlignment;
         private:
             using SharedMemoryInfoList = util::IntrusiveListBaseTraits<KSharedMemoryInfo>::ListType;
-            using BetaList = util::IntrusiveListMemberTraits<&KBeta::m_process_list_node>::ListType;
+            using IoRegionList = util::IntrusiveListMemberTraits<&KIoRegion::m_process_list_node>::ListType;
             using TLPTree = util::IntrusiveRedBlackTreeBaseTraits<KThreadLocalPage>::TreeType<KThreadLocalPage>;
             using TLPIterator = TLPTree::iterator;
         private:
@@ -96,7 +96,7 @@ namespace ams::kern {
             KThread                    *m_exception_thread{};
             ThreadList                  m_thread_list{};
             SharedMemoryInfoList        m_shared_memory_list{};
-            BetaList                    m_beta_list{};
+            IoRegionList                m_io_region_list{};
             bool                        m_is_suspended{};
             bool                        m_is_immortal{};
             bool                        m_is_jit_debug{};
@@ -121,6 +121,9 @@ namespace ams::kern {
             KMemoryBlockSlabManager     m_memory_block_slab_manager{};
             KBlockInfoManager           m_block_info_manager{};
             KPageTableManager           m_page_table_manager{};
+            KMemoryBlockSlabHeap        m_memory_block_heap{};
+            KBlockInfoSlabHeap          m_block_info_heap{};
+            KPageTableSlabHeap          m_page_table_heap{};
         private:
             Result Initialize(const ams::svc::CreateProcessParameter &params);
 
@@ -143,7 +146,6 @@ namespace ams::kern {
             }
         public:
             KProcess() { /* ... */ }
-            virtual ~KProcess() { /* ... */ }
 
             Result Initialize(const ams::svc::CreateProcessParameter &params, const KPageGroup &pg, const u32 *caps, s32 num_caps, KResourceLimit *res_limit, KMemoryManager::Pool pool, bool immortal);
             Result Initialize(const ams::svc::CreateProcessParameter &params, svc::KUserPointer<const u32 *> caps, s32 num_caps, KResourceLimit *res_limit, KMemoryManager::Pool pool);
@@ -273,6 +275,9 @@ namespace ams::kern {
             Result AddSharedMemory(KSharedMemory *shmem, KProcessAddress address, size_t size);
             void RemoveSharedMemory(KSharedMemory *shmem, KProcessAddress address, size_t size);
 
+            void AddIoRegion(KIoRegion *io_region);
+            void RemoveIoRegion(KIoRegion *io_region);
+
             Result CreateThreadLocalRegion(KProcessAddress *out);
             Result DeleteThreadLocalRegion(KProcessAddress addr);
             void *GetThreadLocalRegionPointer(KProcessAddress addr);
@@ -342,14 +347,6 @@ namespace ams::kern {
             void UnpinCurrentThread();
             void UnpinThread(KThread *thread);
 
-            Result SignalToAddress(KProcessAddress address) {
-                return m_cond_var.SignalToAddress(address);
-            }
-
-            Result WaitForAddress(ams::svc::Handle handle, KProcessAddress address, u32 tag) {
-                return m_cond_var.WaitForAddress(handle, address, tag);
-            }
-
             void SignalConditionVariable(uintptr_t cv_key, int32_t count) {
                 return m_cond_var.Signal(cv_key, count);
             }

libraries/libmesosphere/include/mesosphere/kern_k_readable_event.hpp

@@ -28,21 +28,16 @@ namespace ams::kern {
             KEvent *m_parent;
         public:
             constexpr explicit KReadableEvent() : KSynchronizationObject(), m_is_signaled(), m_parent() { MESOSPHERE_ASSERT_THIS(); }
-            virtual ~KReadableEvent() { MESOSPHERE_ASSERT_THIS(); }
 
-            constexpr void Initialize(KEvent *parent) {
-                MESOSPHERE_ASSERT_THIS();
-                m_is_signaled  = false;
-                m_parent       = parent;
-            }
+            void Initialize(KEvent *parent);
 
             constexpr KEvent *GetParent() const { return m_parent; }
 
+            Result Signal();
+            Result Clear();
+
             virtual bool IsSignaled() const override;
             virtual void Destroy() override;
-
-            virtual Result Signal();
-            virtual Result Clear();
             virtual Result Reset();
     };
 

libraries/libmesosphere/include/mesosphere/kern_k_resource_limit.hpp

@@ -34,7 +34,6 @@ namespace ams::kern {
             KLightConditionVariable m_cond_var;
         public:
             constexpr ALWAYS_INLINE KResourceLimit() : m_limit_values(), m_current_values(), m_current_hints(), m_peak_values(), m_lock(), m_waiter_count(), m_cond_var() { /* ... */ }
-            virtual ~KResourceLimit() { /* ... */ }
 
             static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
 
@@ -48,6 +47,8 @@ namespace ams::kern {
 
             Result SetLimitValue(ams::svc::LimitableResource which, s64 value);
 
+            void Add(ams::svc::LimitableResource which, s64 value);
+
             bool Reserve(ams::svc::LimitableResource which, s64 value);
             bool Reserve(ams::svc::LimitableResource which, s64 value, s64 timeout);
             void Release(ams::svc::LimitableResource which, s64 value);

libraries/libmesosphere/include/mesosphere/kern_k_scheduler.hpp

@@ -17,6 +17,7 @@
 #include <mesosphere/kern_select_cpu.hpp>
 #include <mesosphere/kern_k_thread.hpp>
 #include <mesosphere/kern_k_priority_queue.hpp>
+#include <mesosphere/kern_k_interrupt_task_manager.hpp>
 #include <mesosphere/kern_k_scheduler_lock.hpp>
 
 namespace ams::kern {
@@ -39,11 +40,13 @@ namespace ams::kern {
 
             struct SchedulingState {
                 std::atomic<u8> needs_scheduling;
-                bool interrupt_task_thread_runnable;
+                bool interrupt_task_runnable;
                 bool should_count_idle;
                 u64  idle_count;
                 KThread *highest_priority_thread;
                 void *idle_thread_stack;
+                KThread *prev_thread;
+                KInterruptTaskManager *interrupt_task_manager;
             };
         private:
             friend class KScopedSchedulerLock;
@@ -53,28 +56,29 @@ namespace ams::kern {
             SchedulingState m_state;
             bool m_is_active;
             s32 m_core_id;
-            KThread *m_prev_thread;
             s64 m_last_context_switch_time;
             KThread *m_idle_thread;
             std::atomic<KThread *> m_current_thread;
         public:
             constexpr KScheduler()
-                : m_state(), m_is_active(false), m_core_id(0), m_prev_thread(nullptr), m_last_context_switch_time(0), m_idle_thread(nullptr), m_current_thread(nullptr)
+                : m_state(), m_is_active(false), m_core_id(0), m_last_context_switch_time(0), m_idle_thread(nullptr), m_current_thread(nullptr)
             {
-                m_state.needs_scheduling = true;
-                m_state.interrupt_task_thread_runnable = false;
-                m_state.should_count_idle = false;
-                m_state.idle_count = 0;
-                m_state.idle_thread_stack = nullptr;
+                m_state.needs_scheduling        = true;
+                m_state.interrupt_task_runnable = false;
+                m_state.should_count_idle       = false;
+                m_state.idle_count              = 0;
+                m_state.idle_thread_stack       = nullptr;
                 m_state.highest_priority_thread = nullptr;
+                m_state.prev_thread             = nullptr;
+                m_state.interrupt_task_manager  = nullptr;
             }
 
             NOINLINE void Initialize(KThread *idle_thread);
             NOINLINE void Activate();
 
             ALWAYS_INLINE void SetInterruptTaskRunnable() {
-                m_state.interrupt_task_thread_runnable = true;
-                m_state.needs_scheduling               = true;
+                m_state.interrupt_task_runnable = true;
+                m_state.needs_scheduling        = true;
             }
 
             ALWAYS_INLINE void RequestScheduleOnInterrupt() {
@@ -94,7 +98,7 @@ namespace ams::kern {
             }
 
             ALWAYS_INLINE KThread *GetPreviousThread() const {
-                return m_prev_thread;
+                return m_state.prev_thread;
             }
 
             ALWAYS_INLINE KThread *GetSchedulerCurrentThread() const {
@@ -108,8 +112,6 @@ namespace ams::kern {
             /* Static private API. */
             static ALWAYS_INLINE KSchedulerPriorityQueue &GetPriorityQueue() { return s_priority_queue; }
             static NOINLINE u64 UpdateHighestPriorityThreadsImpl();
-
-            static NOINLINE void InterruptTaskThreadToRunnable();
         public:
             /* Static public API. */
             static ALWAYS_INLINE bool CanSchedule() { return GetCurrentThread().GetDisableDispatchCount() == 0; }
@@ -124,13 +126,14 @@ namespace ams::kern {
                 GetCurrentThread().DisableDispatch();
             }
 
-            static NOINLINE void EnableScheduling(u64 cores_needing_scheduling) {
+            static ALWAYS_INLINE void EnableScheduling(u64 cores_needing_scheduling) {
                 MESOSPHERE_ASSERT(GetCurrentThread().GetDisableDispatchCount() >= 1);
 
+                GetCurrentScheduler().RescheduleOtherCores(cores_needing_scheduling);
+
                 if (GetCurrentThread().GetDisableDispatchCount() > 1) {
                     GetCurrentThread().EnableDispatch();
                 } else {
-                    GetCurrentScheduler().RescheduleOtherCores(cores_needing_scheduling);
                     GetCurrentScheduler().RescheduleCurrentCore();
                 }
             }
@@ -176,14 +179,23 @@ namespace ams::kern {
 
             ALWAYS_INLINE void RescheduleCurrentCore() {
                 MESOSPHERE_ASSERT(GetCurrentThread().GetDisableDispatchCount() == 1);
-                {
-                    /* Disable interrupts, and then context switch. */
-                    KScopedInterruptDisable intr_disable;
-                    ON_SCOPE_EXIT { GetCurrentThread().EnableDispatch(); };
 
-                    if (m_state.needs_scheduling.load()) {
-                        Schedule();
-                    }
+                GetCurrentThread().EnableDispatch();
+
+                if (m_state.needs_scheduling.load()) {
+                    /* Disable interrupts, and then check again if rescheduling is needed. */
+                    KScopedInterruptDisable intr_disable;
+
+                    GetCurrentScheduler().RescheduleCurrentCoreImpl();
+                }
+            }
+
+            ALWAYS_INLINE void RescheduleCurrentCoreImpl() {
+                /* Check that scheduling is needed. */
+                if (AMS_LIKELY(m_state.needs_scheduling.load())) {
+                    GetCurrentThread().DisableDispatch();
+                    this->Schedule();
+                    GetCurrentThread().EnableDispatch();
                 }
             }
 
@@ -199,10 +211,12 @@ namespace ams::kern {
     };
 
     consteval bool KScheduler::ValidateAssemblyOffsets() {
-        static_assert(__builtin_offsetof(KScheduler, m_state.needs_scheduling)               == KSCHEDULER_NEEDS_SCHEDULING);
-        static_assert(__builtin_offsetof(KScheduler, m_state.interrupt_task_thread_runnable) == KSCHEDULER_INTERRUPT_TASK_THREAD_RUNNABLE);
-        static_assert(__builtin_offsetof(KScheduler, m_state.highest_priority_thread)        == KSCHEDULER_HIGHEST_PRIORITY_THREAD);
-        static_assert(__builtin_offsetof(KScheduler, m_state.idle_thread_stack)              == KSCHEDULER_IDLE_THREAD_STACK);
+        static_assert(__builtin_offsetof(KScheduler, m_state.needs_scheduling)        == KSCHEDULER_NEEDS_SCHEDULING);
+        static_assert(__builtin_offsetof(KScheduler, m_state.interrupt_task_runnable) == KSCHEDULER_INTERRUPT_TASK_RUNNABLE);
+        static_assert(__builtin_offsetof(KScheduler, m_state.highest_priority_thread) == KSCHEDULER_HIGHEST_PRIORITY_THREAD);
+        static_assert(__builtin_offsetof(KScheduler, m_state.idle_thread_stack)       == KSCHEDULER_IDLE_THREAD_STACK);
+        static_assert(__builtin_offsetof(KScheduler, m_state.prev_thread)             == KSCHEDULER_PREVIOUS_THREAD);
+        static_assert(__builtin_offsetof(KScheduler, m_state.interrupt_task_manager)  == KSCHEDULER_INTERRUPT_TASK_MANAGER);
 
         return true;
     }

libraries/libmesosphere/include/mesosphere/kern_k_scheduler_lock.hpp

@@ -45,7 +45,7 @@ namespace ams::kern {
                 return m_owner_thread == GetCurrentThreadPointer();
             }
 
-            void Lock() {
+            NOINLINE void Lock() {
                 MESOSPHERE_ASSERT_THIS();
 
                 if (this->IsLockedByCurrentThread()) {
@@ -67,7 +67,7 @@ namespace ams::kern {
                 }
             }
 
-            void Unlock() {
+            NOINLINE void Unlock() {
                 MESOSPHERE_ASSERT_THIS();
                 MESOSPHERE_ASSERT(this->IsLockedByCurrentThread());
                 MESOSPHERE_ASSERT(m_lock_count > 0);

libraries/libmesosphere/include/mesosphere/kern_k_server_port.hpp

@@ -35,7 +35,6 @@ namespace ams::kern {
             KPort *m_parent;
         public:
             constexpr KServerPort() : m_session_list(), m_light_session_list(), m_parent() { /* ... */ }
-            virtual ~KServerPort() { /* ... */ }
 
             void Initialize(KPort *parent);
             void EnqueueSession(KServerSession *session);

libraries/libmesosphere/include/mesosphere/kern_k_server_session.hpp

@@ -34,7 +34,6 @@ namespace ams::kern {
             KLightLock m_lock;
         public:
             constexpr KServerSession() : m_parent(), m_request_list(), m_current_request(), m_lock() { /* ... */ }
-            virtual ~KServerSession() { /* ... */ }
 
             virtual void Destroy() override;
 

libraries/libmesosphere/include/mesosphere/kern_k_session.hpp

@@ -25,7 +25,7 @@ namespace ams::kern {
     class KClientPort;
     class KProcess;
 
-    class KSession final : public KAutoObjectWithSlabHeapAndContainer<KSession, KAutoObjectWithList> {
+    class KSession final : public KAutoObjectWithSlabHeapAndContainer<KSession, KAutoObjectWithList, true> {
         MESOSPHERE_AUTOOBJECT_TRAITS(KSession, KAutoObject);
         private:
             enum class State : u8 {
@@ -57,8 +57,6 @@ namespace ams::kern {
                 /* ... */
             }
 
-            virtual ~KSession() { /* ... */ }
-
             void Initialize(KClientPort *client_port, uintptr_t name);
             virtual void Finalize() override;
 

libraries/libmesosphere/include/mesosphere/kern_k_session_request.hpp

@@ -17,14 +17,14 @@
 #include <mesosphere/kern_common.hpp>
 #include <mesosphere/kern_k_auto_object.hpp>
 #include <mesosphere/kern_slab_helpers.hpp>
-#include <mesosphere/kern_k_writable_event.hpp>
+#include <mesosphere/kern_k_event.hpp>
 #include <mesosphere/kern_k_thread.hpp>
 #include <mesosphere/kern_k_process.hpp>
 #include <mesosphere/kern_k_memory_block.hpp>
 
 namespace ams::kern {
 
-    class KSessionRequest final : public KSlabAllocated<KSessionRequest>, public KAutoObject, public util::IntrusiveListBaseNode<KSessionRequest> {
+    class KSessionRequest final : public KSlabAllocated<KSessionRequest, true>, public KAutoObject, public util::IntrusiveListBaseNode<KSessionRequest> {
         MESOSPHERE_AUTOOBJECT_TRAITS(KSessionRequest, KAutoObject);
         public:
             class SessionMappings {
@@ -126,12 +126,11 @@ namespace ams::kern {
             SessionMappings m_mappings;
             KThread *m_thread;
             KProcess *m_server;
-            KWritableEvent *m_event;
+            KEvent *m_event;
             uintptr_t m_address;
             size_t m_size;
         public:
             constexpr KSessionRequest() : m_mappings(), m_thread(), m_server(), m_event(), m_address(), m_size() { /* ... */ }
-            virtual ~KSessionRequest() { /* ... */ }
 
             static KSessionRequest *Create() {
                 KSessionRequest *req = KSessionRequest::Allocate();
@@ -141,12 +140,20 @@ namespace ams::kern {
                 return req;
             }
 
+            static KSessionRequest *CreateFromUnusedSlabMemory() {
+                KSessionRequest *req = KSessionRequest::AllocateFromUnusedSlabMemory();
+                if (req != nullptr) {
+                    KAutoObject::Create(req);
+                }
+                return req;
+            }
+
             virtual void Destroy() override {
                 this->Finalize();
                 KSessionRequest::Free(this);
             }
 
-            void Initialize(KWritableEvent *event, uintptr_t address, size_t size) {
+            void Initialize(KEvent *event, uintptr_t address, size_t size) {
                 m_mappings.Initialize();
 
                 m_thread  = std::addressof(GetCurrentThread());
@@ -177,7 +184,7 @@ namespace ams::kern {
             static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
 
             constexpr ALWAYS_INLINE KThread *GetThread() const { return m_thread; }
-            constexpr ALWAYS_INLINE KWritableEvent *GetEvent() const { return m_event; }
+            constexpr ALWAYS_INLINE KEvent *GetEvent() const { return m_event; }
             constexpr ALWAYS_INLINE uintptr_t GetAddress() const { return m_address; }
             constexpr ALWAYS_INLINE size_t GetSize() const { return m_size; }
             constexpr ALWAYS_INLINE KProcess *GetServerProcess() const { return m_server; }

libraries/libmesosphere/include/mesosphere/kern_k_shared_memory.hpp

@@ -35,14 +35,12 @@ namespace ams::kern {
             bool m_is_initialized;
         public:
             explicit KSharedMemory()
-                : m_page_group(std::addressof(Kernel::GetBlockInfoManager())), m_resource_limit(nullptr), m_owner_process_id(std::numeric_limits<u64>::max()),
+                : m_page_group(std::addressof(Kernel::GetSystemBlockInfoManager())), m_resource_limit(nullptr), m_owner_process_id(std::numeric_limits<u64>::max()),
                   m_owner_perm(ams::svc::MemoryPermission_None), m_remote_perm(ams::svc::MemoryPermission_None), m_is_initialized(false)
             {
                 /* ... */
             }
 
-            virtual ~KSharedMemory() { /* ... */ }
-
             Result Initialize(KProcess *owner, size_t size, ams::svc::MemoryPermission own_perm, ams::svc::MemoryPermission rem_perm);
             virtual void Finalize() override;
 

libraries/libmesosphere/include/mesosphere/kern_k_slab_heap.hpp

@@ -16,11 +16,13 @@
 #pragma once
 #include <mesosphere/kern_common.hpp>
 #include <mesosphere/kern_k_typed_address.hpp>
+#include <mesosphere/kern_k_memory_layout.hpp>
 
 #if defined(ATMOSPHERE_ARCH_ARM64)
 
     #include <mesosphere/arch/arm64/kern_k_slab_heap_impl.hpp>
     namespace ams::kern {
+        using ams::kern::arch::arm64::IsSlabAtomicValid;
         using ams::kern::arch::arm64::AllocateFromSlabAtomic;
         using ams::kern::arch::arm64::FreeToSlabAtomic;
     }
@@ -44,78 +46,73 @@ namespace ams::kern {
                     Node *next;
                 };
             private:
-                Node * m_head;
-                size_t m_obj_size;
+                Node *m_head{nullptr};
             public:
-                constexpr KSlabHeapImpl() : m_head(nullptr), m_obj_size(0) { MESOSPHERE_ASSERT_THIS(); }
+                constexpr KSlabHeapImpl() = default;
 
-                void Initialize(size_t size) {
-                    MESOSPHERE_INIT_ABORT_UNLESS(m_head == nullptr);
-                    m_obj_size = size;
+                void Initialize() {
+                    MESOSPHERE_ABORT_UNLESS(m_head == nullptr);
+                    MESOSPHERE_ABORT_UNLESS(IsSlabAtomicValid());
                 }
 
-                Node *GetHead() const {
+                ALWAYS_INLINE Node *GetHead() const {
                     return m_head;
                 }
 
-                size_t GetObjectSize() const {
-                    return m_obj_size;
-                }
-
-                void *Allocate() {
-                    MESOSPHERE_ASSERT_THIS();
-
+                ALWAYS_INLINE void *Allocate() {
                     return AllocateFromSlabAtomic(std::addressof(m_head));
                 }
 
-                void Free(void *obj) {
-                    MESOSPHERE_ASSERT_THIS();
-
-                    Node *node = reinterpret_cast<Node *>(obj);
-
-                    return FreeToSlabAtomic(std::addressof(m_head), node);
+                ALWAYS_INLINE void Free(void *obj) {
+                    return FreeToSlabAtomic(std::addressof(m_head), static_cast<Node *>(obj));
                 }
         };
 
     }
 
-    class KSlabHeapBase {
+    template<bool SupportDynamicExpansion>
+    class KSlabHeapBase : protected impl::KSlabHeapImpl {
         NON_COPYABLE(KSlabHeapBase);
         NON_MOVEABLE(KSlabHeapBase);
         private:
-            using Impl = impl::KSlabHeapImpl;
+            size_t m_obj_size{};
+            uintptr_t m_peak{};
+            uintptr_t m_start{};
+            uintptr_t m_end{};
         private:
-            Impl m_impl;
-            uintptr_t m_peak;
-            uintptr_t m_start;
-            uintptr_t m_end;
-        private:
-            ALWAYS_INLINE Impl *GetImpl() {
-                return std::addressof(m_impl);
-            }
-            ALWAYS_INLINE const Impl *GetImpl() const {
-                return std::addressof(m_impl);
+            ALWAYS_INLINE void UpdatePeakImpl(uintptr_t obj) {
+                static_assert(std::atomic_ref<uintptr_t>::is_always_lock_free);
+                std::atomic_ref<uintptr_t> peak_ref(m_peak);
+
+                const uintptr_t alloc_peak = obj + this->GetObjectSize();
+                uintptr_t cur_peak = m_peak;
+                do {
+                    if (alloc_peak <= cur_peak) {
+                        break;
+                    }
+                } while (!peak_ref.compare_exchange_strong(cur_peak, alloc_peak));
             }
         public:
-            constexpr KSlabHeapBase() : m_impl(), m_peak(0), m_start(0), m_end(0) { MESOSPHERE_ASSERT_THIS(); }
+            constexpr KSlabHeapBase() = default;
 
             ALWAYS_INLINE bool Contains(uintptr_t address) const {
                 return m_start <= address && address < m_end;
             }
 
-            void InitializeImpl(size_t obj_size, void *memory, size_t memory_size) {
-                MESOSPHERE_ASSERT_THIS();
-
+            void Initialize(size_t obj_size, void *memory, size_t memory_size) {
                 /* Ensure we don't initialize a slab using null memory. */
                 MESOSPHERE_ABORT_UNLESS(memory != nullptr);
 
+                /* Set our object size. */
+                m_obj_size = obj_size;
+
                 /* Initialize the base allocator. */
-                this->GetImpl()->Initialize(obj_size);
+                KSlabHeapImpl::Initialize();
 
                 /* Set our tracking variables. */
                 const size_t num_obj = (memory_size / obj_size);
                 m_start = reinterpret_cast<uintptr_t>(memory);
-                m_end = m_start + num_obj * obj_size;
+                m_end  = m_start + num_obj * obj_size;
                 m_peak = m_start;
 
                 /* Free the objects. */
@@ -123,75 +120,91 @@ namespace ams::kern {
 
                 for (size_t i = 0; i < num_obj; i++) {
                     cur -= obj_size;
-                    this->GetImpl()->Free(cur);
+                    KSlabHeapImpl::Free(cur);
                 }
             }
 
-            size_t GetSlabHeapSize() const {
+            ALWAYS_INLINE size_t GetSlabHeapSize() const {
                 return (m_end - m_start) / this->GetObjectSize();
             }
 
-            size_t GetObjectSize() const {
-                return this->GetImpl()->GetObjectSize();
+            ALWAYS_INLINE size_t GetObjectSize() const {
+                return m_obj_size;
             }
 
-            void *AllocateImpl() {
-                MESOSPHERE_ASSERT_THIS();
-
-                void *obj = this->GetImpl()->Allocate();
+            ALWAYS_INLINE void *Allocate() {
+                void *obj = KSlabHeapImpl::Allocate();
 
                 /* Track the allocated peak. */
                 #if defined(MESOSPHERE_BUILD_FOR_DEBUGGING)
                 if (AMS_LIKELY(obj != nullptr)) {
-                    static_assert(std::atomic_ref<uintptr_t>::is_always_lock_free);
-                    std::atomic_ref<uintptr_t> peak_ref(m_peak);
-
-                    const uintptr_t alloc_peak = reinterpret_cast<uintptr_t>(obj) + this->GetObjectSize();
-                    uintptr_t cur_peak = m_peak;
-                    do {
-                        if (alloc_peak <= cur_peak) {
-                            break;
+                    if constexpr (SupportDynamicExpansion) {
+                        if (this->Contains(reinterpret_cast<uintptr_t>(obj))) {
+                            this->UpdatePeakImpl(reinterpret_cast<uintptr_t>(obj));
+                        } else {
+                            this->UpdatePeakImpl(reinterpret_cast<uintptr_t>(m_end) - this->GetObjectSize());
                         }
-                    } while (!peak_ref.compare_exchange_strong(cur_peak, alloc_peak));
+                    } else {
+                        this->UpdatePeakImpl(reinterpret_cast<uintptr_t>(obj));
+                    }
                 }
                 #endif
 
                 return obj;
             }
 
-            void FreeImpl(void *obj) {
-                MESOSPHERE_ASSERT_THIS();
-
+            ALWAYS_INLINE void Free(void *obj) {
                 /* Don't allow freeing an object that wasn't allocated from this heap. */
-                MESOSPHERE_ABORT_UNLESS(this->Contains(reinterpret_cast<uintptr_t>(obj)));
+                const bool contained = this->Contains(reinterpret_cast<uintptr_t>(obj));
+                if constexpr (SupportDynamicExpansion) {
+                    const bool is_slab = KMemoryLayout::GetSlabRegion().Contains(reinterpret_cast<uintptr_t>(obj));
+                    MESOSPHERE_ABORT_UNLESS(contained || is_slab);
+                } else {
+                    MESOSPHERE_ABORT_UNLESS(contained);
+                }
 
-                this->GetImpl()->Free(obj);
+                KSlabHeapImpl::Free(obj);
             }
 
-            size_t GetObjectIndexImpl(const void *obj) const {
+            ALWAYS_INLINE size_t GetObjectIndex(const void *obj) const {
+                if constexpr (SupportDynamicExpansion) {
+                    if (!this->Contains(reinterpret_cast<uintptr_t>(obj))) {
+                        return std::numeric_limits<size_t>::max();
+                    }
+                }
+
                 return (reinterpret_cast<uintptr_t>(obj) - m_start) / this->GetObjectSize();
             }
 
-            size_t GetPeakIndex() const {
-                return this->GetObjectIndexImpl(reinterpret_cast<const void *>(m_peak));
+            ALWAYS_INLINE size_t GetPeakIndex() const {
+                return this->GetObjectIndex(reinterpret_cast<const void *>(m_peak));
             }
 
-            uintptr_t GetSlabHeapAddress() const {
+            ALWAYS_INLINE uintptr_t GetSlabHeapAddress() const {
                 return m_start;
             }
 
-            size_t GetNumRemaining() const {
+            ALWAYS_INLINE size_t GetNumRemaining() const {
                 size_t remaining = 0;
 
                 /* Only calculate the number of remaining objects under debug configuration. */
                 #if defined(MESOSPHERE_BUILD_FOR_DEBUGGING)
                 while (true) {
-                    auto *cur = this->GetImpl()->GetHead();
+                    auto *cur = this->GetHead();
                     remaining = 0;
 
-                    while (this->Contains(reinterpret_cast<uintptr_t>(cur))) {
-                        ++remaining;
-                        cur = cur->next;
+                    if constexpr (SupportDynamicExpansion) {
+                        const auto &slab_region = KMemoryLayout::GetSlabRegion();
+
+                        while (this->Contains(reinterpret_cast<uintptr_t>(cur)) || slab_region.Contains(reinterpret_cast<uintptr_t>(cur))) {
+                            ++remaining;
+                            cur = cur->next;
+                        }
+                    } else {
+                        while (this->Contains(reinterpret_cast<uintptr_t>(cur))) {
+                            ++remaining;
+                            cur = cur->next;
+                        }
                     }
 
                     if (cur == nullptr) {
@@ -204,29 +217,31 @@ namespace ams::kern {
             }
     };
 
-    template<typename T>
-    class KSlabHeap : public KSlabHeapBase {
+    template<typename T, bool SupportDynamicExpansion>
+    class KSlabHeap : public KSlabHeapBase<SupportDynamicExpansion> {
+        private:
+            using BaseHeap = KSlabHeapBase<SupportDynamicExpansion>;
         public:
-            constexpr KSlabHeap() : KSlabHeapBase() { /* ... */ }
+            constexpr KSlabHeap() = default;
 
             void Initialize(void *memory, size_t memory_size) {
-                this->InitializeImpl(sizeof(T), memory, memory_size);
+                BaseHeap::Initialize(sizeof(T), memory, memory_size);
             }
 
-            T *Allocate() {
-                T *obj = reinterpret_cast<T *>(this->AllocateImpl());
+            ALWAYS_INLINE T *Allocate() {
+                T *obj = static_cast<T *>(BaseHeap::Allocate());
                 if (AMS_LIKELY(obj != nullptr)) {
                     std::construct_at(obj);
                 }
                 return obj;
             }
 
-            void Free(T *obj) {
-                this->FreeImpl(obj);
+            ALWAYS_INLINE void Free(T *obj) {
+                BaseHeap::Free(obj);
             }
 
-            size_t GetObjectIndex(const T *obj) const {
-                return this->GetObjectIndexImpl(obj);
+            ALWAYS_INLINE size_t GetObjectIndex(const T *obj) const {
+                return BaseHeap::GetObjectIndex(obj);
             }
     };
 

libraries/libmesosphere/include/mesosphere/kern_k_synchronization_object.hpp

@@ -33,7 +33,6 @@ namespace ams::kern {
             ThreadListNode *m_thread_list_tail;
         protected:
             constexpr ALWAYS_INLINE explicit KSynchronizationObject() : KAutoObjectWithList(), m_thread_list_head(), m_thread_list_tail() { MESOSPHERE_ASSERT_THIS(); }
-            virtual ~KSynchronizationObject() { MESOSPHERE_ASSERT_THIS(); }
 
             virtual void OnFinalizeSynchronizationObject() { MESOSPHERE_ASSERT_THIS(); }
 
@@ -46,7 +45,39 @@ namespace ams::kern {
         public:
             virtual void Finalize() override;
             virtual bool IsSignaled() const { AMS_INFINITE_LOOP(); }
-            virtual void DumpWaiters();
+
+            void DumpWaiters();
+
+            ALWAYS_INLINE void LinkNode(ThreadListNode *node) {
+                /* Link the node to the list. */
+                if (m_thread_list_tail == nullptr) {
+                    m_thread_list_head = node;
+                } else {
+                    m_thread_list_tail->next = node;
+                }
+
+                m_thread_list_tail = node;
+            }
+
+            ALWAYS_INLINE void UnlinkNode(ThreadListNode *node) {
+                /* Unlink the node from the list. */
+                ThreadListNode *prev_ptr = reinterpret_cast<ThreadListNode *>(std::addressof(m_thread_list_head));
+                ThreadListNode *prev_val = nullptr;
+                ThreadListNode *prev, *tail_prev;
+
+                do {
+                    prev      = prev_ptr;
+                    prev_ptr  = prev_ptr->next;
+                    tail_prev = prev_val;
+                    prev_val  = prev_ptr;
+                } while (prev_ptr != node);
+
+                if (m_thread_list_tail == node) {
+                    m_thread_list_tail = tail_prev;
+                }
+
+                prev->next = node->next;
+            }
     };
 
 }

libraries/libmesosphere/include/mesosphere/kern_k_target_system.hpp

@@ -23,12 +23,13 @@ namespace ams::kern {
         private:
             friend class KSystemControl;
         private:
-            static inline bool s_is_debug_mode;
-            static inline bool s_enable_debug_logging;
-            static inline bool s_enable_user_exception_handlers;
-            static inline bool s_enable_debug_memory_fill;
-            static inline bool s_enable_user_pmu_access;
-            static inline bool s_enable_kernel_debugging;
+            static inline constinit bool s_is_debug_mode;
+            static inline constinit bool s_enable_debug_logging;
+            static inline constinit bool s_enable_user_exception_handlers;
+            static inline constinit bool s_enable_debug_memory_fill;
+            static inline constinit bool s_enable_user_pmu_access;
+            static inline constinit bool s_enable_kernel_debugging;
+            static inline constinit bool s_enable_dynamic_resource_limits;
         private:
             static ALWAYS_INLINE void SetIsDebugMode(bool en) { s_is_debug_mode = en; }
             static ALWAYS_INLINE void EnableDebugLogging(bool en) { s_enable_debug_logging = en; }
@@ -36,6 +37,7 @@ namespace ams::kern {
             static ALWAYS_INLINE void EnableDebugMemoryFill(bool en) { s_enable_debug_memory_fill = en; }
             static ALWAYS_INLINE void EnableUserPmuAccess(bool en) { s_enable_user_pmu_access = en; }
             static ALWAYS_INLINE void EnableKernelDebugging(bool en) { s_enable_kernel_debugging = en; }
+            static ALWAYS_INLINE void EnableDynamicResourceLimits(bool en) { s_enable_dynamic_resource_limits = en; }
         public:
             static ALWAYS_INLINE bool IsDebugMode() { return s_is_debug_mode; }
             static ALWAYS_INLINE bool IsDebugLoggingEnabled() { return s_enable_debug_logging; }
@@ -43,6 +45,7 @@ namespace ams::kern {
             static ALWAYS_INLINE bool IsDebugMemoryFillEnabled() { return s_enable_debug_memory_fill; }
             static ALWAYS_INLINE bool IsUserPmuAccessEnabled() { return s_enable_user_pmu_access; }
             static ALWAYS_INLINE bool IsKernelDebuggingEnabled() { return s_enable_kernel_debugging; }
+            static ALWAYS_INLINE bool IsDynamicResourceLimitsEnabled() { return s_enable_dynamic_resource_limits; }
     };
 
 }

libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp

@@ -37,6 +37,7 @@ namespace ams::kern {
             friend class KProcess;
             friend class KConditionVariable;
             friend class KAddressArbiter;
+            friend class KThreadQueue;
         public:
             static constexpr s32 MainThreadPriority = 1;
             static constexpr s32 IdleThreadPriority = 64;
@@ -92,6 +93,9 @@ namespace ams::kern {
                 bool is_calling_svc;
                 bool is_in_exception_handler;
                 bool is_pinned;
+                #if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP)
+                bool is_single_step;
+                #endif
             };
             static_assert(alignof(StackParameters) == 0x10);
             static_assert(sizeof(StackParameters) == THREAD_STACK_PARAMETERS_SIZE);
@@ -106,6 +110,10 @@ namespace ams::kern {
             static_assert(__builtin_offsetof(StackParameters, is_in_exception_handler) == THREAD_STACK_PARAMETERS_IS_IN_EXCEPTION_HANDLER);
             static_assert(__builtin_offsetof(StackParameters, is_pinned)               == THREAD_STACK_PARAMETERS_IS_PINNED);
 
+            #if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP)
+            static_assert(__builtin_offsetof(StackParameters, is_single_step)          == THREAD_STACK_PARAMETERS_IS_SINGLE_STEP);
+            #endif
+
             struct QueueEntry {
                 private:
                     KThread *m_prev;
@@ -184,7 +192,6 @@ namespace ams::kern {
             KAffinityMask                   m_physical_affinity_mask{};
             u64                             m_thread_id{};
             std::atomic<s64>                m_cpu_time{};
-            KSynchronizationObject         *m_synced_object{};
             KProcessAddress                 m_address_key{};
             KProcess                       *m_parent{};
             void                           *m_kernel_stack_top{};
@@ -197,9 +204,7 @@ namespace ams::kern {
             s64                             m_last_scheduled_tick{};
             QueueEntry                      m_per_core_priority_queue_entry[cpu::NumCores]{};
             KLightLock                     *m_waiting_lock{};
-
-            KThreadQueue                   *m_sleeping_queue{};
-
+            KThreadQueue                   *m_wait_queue{};
             WaiterList                      m_waiter_list{};
             WaiterList                      m_pinned_waiter_list{};
             KThread                        *m_lock_owner{};
@@ -208,6 +213,7 @@ namespace ams::kern {
             u32                             m_address_key_value{};
             u32                             m_suspend_request_flags{};
             u32                             m_suspend_allowed_flags{};
+            s32                             m_synced_index{};
             Result                          m_wait_result;
             Result                          m_debug_exception_result;
             s32                             m_base_priority{};
@@ -232,10 +238,7 @@ namespace ams::kern {
         public:
             constexpr KThread() : m_wait_result(svc::ResultNoSynchronizationObject()), m_debug_exception_result(ResultSuccess()) { /* ... */ }
 
-            virtual ~KThread() { /* ... */ }
-
             Result Initialize(KThreadFunction func, uintptr_t arg, void *kern_stack_top, KProcessAddress user_stack_top, s32 prio, s32 virt_core, KProcess *owner, ThreadType type);
-
         private:
             static Result InitializeThread(KThread *thread, KThreadFunction func, uintptr_t arg, KProcessAddress user_stack_top, s32 prio, s32 virt_core, KProcess *owner, ThreadType type);
         public:
@@ -325,6 +328,25 @@ namespace ams::kern {
                 return this->GetStackParameters().current_svc_id;
             }
 
+            #if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP)
+
+            ALWAYS_INLINE void SetSingleStep() {
+                MESOSPHERE_ASSERT_THIS();
+                this->GetStackParameters().is_single_step = true;
+            }
+
+            ALWAYS_INLINE void ClearSingleStep() {
+                MESOSPHERE_ASSERT_THIS();
+                this->GetStackParameters().is_single_step = false;
+            }
+
+            ALWAYS_INLINE bool IsSingleStep() const {
+                MESOSPHERE_ASSERT_THIS();
+                return this->GetStackParameters().is_single_step;
+            }
+
+            #endif
+
             ALWAYS_INLINE void RegisterDpc(DpcFlag flag) {
                 this->GetStackParameters().dpc_flags.fetch_or(flag);
             }
@@ -351,6 +373,8 @@ namespace ams::kern {
             void FinishTermination();
 
             void IncreaseBasePriority(s32 priority);
+
+            NOINLINE void SetState(ThreadState state);
         public:
             constexpr u64 GetThreadId() const { return m_thread_id; }
 
@@ -367,7 +391,6 @@ namespace ams::kern {
 
             constexpr ThreadState GetState() const { return static_cast<ThreadState>(m_thread_state & ThreadState_Mask); }
             constexpr ThreadState GetRawState() const { return m_thread_state; }
-            NOINLINE void SetState(ThreadState state);
 
             NOINLINE KThreadContext *GetContextForSchedulerLoop();
 
@@ -419,8 +442,6 @@ namespace ams::kern {
             constexpr QueueEntry &GetPriorityQueueEntry(s32 core) { return m_per_core_priority_queue_entry[core]; }
             constexpr const QueueEntry &GetPriorityQueueEntry(s32 core) const { return m_per_core_priority_queue_entry[core]; }
 
-            constexpr void SetSleepingQueue(KThreadQueue *q) { m_sleeping_queue = q; }
-
             constexpr ConditionVariableThreadTree *GetConditionVariableTree() const { return m_condvar_tree; }
 
             constexpr s32 GetNumKernelWaiters() const { return m_num_kernel_waiters; }
@@ -437,29 +458,22 @@ namespace ams::kern {
             constexpr void SetLockOwner(KThread *owner) { m_lock_owner = owner; }
             constexpr KThread *GetLockOwner() const { return m_lock_owner; }
 
-            constexpr void SetSyncedObject(KSynchronizationObject *obj, Result wait_res) {
-                MESOSPHERE_ASSERT_THIS();
+            constexpr void ClearWaitQueue() { m_wait_queue = nullptr; }
 
-                m_synced_object = obj;
-                m_wait_result = wait_res;
-            }
+            void BeginWait(KThreadQueue *queue);
+            void NotifyAvailable(KSynchronizationObject *signaled_object, Result wait_result);
+            void EndWait(Result wait_result);
+            void CancelWait(Result wait_result, bool cancel_timer_task);
 
-            constexpr Result GetWaitResult(KSynchronizationObject **out) const {
-                MESOSPHERE_ASSERT_THIS();
+            constexpr void SetSyncedIndex(s32 index) { m_synced_index = index; }
+            constexpr s32 GetSyncedIndex() const { return m_synced_index; }
 
-                *out = m_synced_object;
-                return m_wait_result;
-            }
+            constexpr void SetWaitResult(Result wait_res) { m_wait_result = wait_res; }
+            constexpr Result GetWaitResult() const { return m_wait_result; }
 
-            constexpr void SetDebugExceptionResult(Result result) {
-                MESOSPHERE_ASSERT_THIS();
-                m_debug_exception_result = result;
-            }
+            constexpr void SetDebugExceptionResult(Result result) { m_debug_exception_result = result; }
 
-            constexpr Result GetDebugExceptionResult() const {
-                MESOSPHERE_ASSERT_THIS();
-                return m_debug_exception_result;
-            }
+            constexpr Result GetDebugExceptionResult() const { return m_debug_exception_result; }
 
             void WaitCancel();
 
@@ -562,8 +576,6 @@ namespace ams::kern {
                 }
             }
 
-            void Wakeup();
-
             void SetBasePriority(s32 priority);
             Result SetPriorityToIdle();
 

libraries/libmesosphere/include/mesosphere/kern_k_thread_queue.hpp

@@ -16,69 +16,28 @@
 #pragma once
 #include <mesosphere/kern_common.hpp>
 #include <mesosphere/kern_k_thread.hpp>
+#include <mesosphere/kern_select_hardware_timer.hpp>
 
 namespace ams::kern {
 
     class KThreadQueue {
         private:
-            KThread::WaiterList m_wait_list;
+            KHardwareTimer *m_hardware_timer;
         public:
-            constexpr ALWAYS_INLINE KThreadQueue() : m_wait_list() { /* ... */ }
+            constexpr ALWAYS_INLINE KThreadQueue() : m_hardware_timer(nullptr) { /* ... */ }
 
-            bool IsEmpty() const { return m_wait_list.empty(); }
+            constexpr void SetHardwareTimer(KHardwareTimer *timer) { m_hardware_timer = timer; }
 
-            KThread::WaiterList::iterator begin() { return m_wait_list.begin(); }
-            KThread::WaiterList::iterator end() { return m_wait_list.end(); }
+            virtual void NotifyAvailable(KThread *waiting_thread, KSynchronizationObject *signaled_object, Result wait_result);
+            virtual void EndWait(KThread *waiting_thread, Result wait_result);
+            virtual void CancelWait(KThread *waiting_thread, Result wait_result, bool cancel_timer_task);
+    };
 
-            bool SleepThread(KThread *t) {
-                KScopedSchedulerLock sl;
+    class KThreadQueueWithoutEndWait : public KThreadQueue {
+        public:
+            constexpr ALWAYS_INLINE KThreadQueueWithoutEndWait() : KThreadQueue() { /* ... */ }
 
-                /* If the thread needs terminating, don't enqueue it. */
-                if (t->IsTerminationRequested()) {
-                    return false;
-                }
-
-                /* Set the thread's queue and mark it as waiting. */
-                t->SetSleepingQueue(this);
-                t->SetState(KThread::ThreadState_Waiting);
-
-                /* Add the thread to the queue. */
-                m_wait_list.push_back(*t);
-
-                return true;
-            }
-
-            void WakeupThread(KThread *t) {
-                KScopedSchedulerLock sl;
-
-                /* Remove the thread from the queue. */
-                m_wait_list.erase(m_wait_list.iterator_to(*t));
-
-                /* Mark the thread as no longer sleeping. */
-                t->SetState(KThread::ThreadState_Runnable);
-                t->SetSleepingQueue(nullptr);
-            }
-
-            KThread *WakeupFrontThread() {
-                KScopedSchedulerLock sl;
-
-                if (m_wait_list.empty()) {
-                    return nullptr;
-                } else {
-                    /* Remove the thread from the queue. */
-                    auto it = m_wait_list.begin();
-                    KThread *thread = std::addressof(*it);
-                    m_wait_list.erase(it);
-
-                    MESOSPHERE_ASSERT(thread->GetState() == KThread::ThreadState_Waiting);
-
-                    /* Mark the thread as no longer sleeping. */
-                    thread->SetState(KThread::ThreadState_Runnable);
-                    thread->SetSleepingQueue(nullptr);
-
-                    return thread;
-                }
-            }
+            virtual void EndWait(KThread *waiting_thread, Result wait_result) override final;
     };
 
 }

libraries/libmesosphere/include/mesosphere/kern_k_transfer_memory.hpp

@@ -35,8 +35,6 @@ namespace ams::kern {
                 /* ... */
             }
 
-            virtual ~KTransferMemory() { /* ... */ }
-
             Result Initialize(KProcessAddress addr, size_t size, ams::svc::MemoryPermission own_perm);
             virtual void Finalize() override;
 

libraries/libmesosphere/include/mesosphere/kern_k_unused_slab_memory.hpp

@@ -13,28 +13,15 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
-#include <mesosphere.hpp>
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_k_typed_address.hpp>
 
 namespace ams::kern {
 
-    void KWritableEvent::Initialize(KEvent *p) {
-        /* Set parent, open a reference to the readable event. */
-        m_parent = p;
-        m_parent->GetReadableEvent().Open();
-    }
-
-    Result KWritableEvent::Signal() {
-        return m_parent->GetReadableEvent().Signal();
-    }
-
-    Result KWritableEvent::Clear() {
-        return m_parent->GetReadableEvent().Clear();
-    }
-
-    void KWritableEvent::Destroy() {
-        /* Close our references. */
-        m_parent->GetReadableEvent().Close();
-        m_parent->Close();
-    }
+    /* Utilities to allocate/free memory from the "unused" gaps between slab heaps. */
+    /* See KTargetSystem::IsDynamicResourceLimitsEnabled() usage for more context. */
+    KVirtualAddress AllocateUnusedSlabMemory(size_t size, size_t alignment);
+    void FreeUnusedSlabMemory(KVirtualAddress address, size_t size);
 
 }

libraries/libmesosphere/include/mesosphere/kern_k_wait_object.hpp

@@ -20,14 +20,13 @@
 
 namespace ams::kern {
 
-    class KWaitObject : public KTimerTask {
+    class KWaitObject {
         private:
             KThread::WaiterList m_wait_list;
-            bool  m_timer_used;
+            KThread *m_next_thread;
         public:
-            constexpr KWaitObject() : m_wait_list(), m_timer_used() { /* ... */ }
+            constexpr KWaitObject() : m_wait_list(), m_next_thread() { /* ... */ }
 
-            virtual void OnTimer() override;
             Result Synchronize(s64 timeout);
     };
 

libraries/libmesosphere/include/mesosphere/kern_k_worker_task_manager.hpp

@@ -32,9 +32,7 @@ namespace ams::kern {
         private:
             KWorkerTask *m_head_task;
             KWorkerTask *m_tail_task;
-            KThread *m_thread;
-            WorkerType m_type;
-            bool m_active;
+            KThread *m_waiting_thread;
         private:
             static void ThreadFunction(uintptr_t arg);
             void ThreadFunctionImpl();
@@ -42,9 +40,9 @@ namespace ams::kern {
             KWorkerTask *GetTask();
             void AddTask(KWorkerTask *task);
         public:
-            constexpr KWorkerTaskManager() : m_head_task(), m_tail_task(), m_thread(), m_type(WorkerType_Count), m_active() { /* ... */ }
+            constexpr KWorkerTaskManager() : m_head_task(), m_tail_task(), m_waiting_thread() { /* ... */ }
 
-            NOINLINE void Initialize(WorkerType wt, s32 priority);
+            NOINLINE void Initialize(s32 priority);
             static void AddTask(WorkerType type, KWorkerTask *task);
     };
 

libraries/libmesosphere/include/mesosphere/kern_k_writable_event.hpp

@@ -1,44 +0,0 @@
-/*
- * Copyright (c) 2018-2020 Atmosphère-NX
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-#pragma once
-#include <mesosphere/kern_common.hpp>
-#include <mesosphere/kern_k_auto_object.hpp>
-#include <mesosphere/kern_slab_helpers.hpp>
-
-namespace ams::kern {
-
-    class KEvent;
-
-    class KWritableEvent final : public KAutoObjectWithSlabHeapAndContainer<KWritableEvent, KAutoObjectWithList> {
-        MESOSPHERE_AUTOOBJECT_TRAITS(KWritableEvent, KAutoObject);
-        private:
-            KEvent *m_parent;
-        public:
-            constexpr explicit KWritableEvent() : m_parent(nullptr) { /* ... */ }
-            virtual ~KWritableEvent() { /* ... */ }
-
-            virtual void Destroy() override;
-
-            static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
-
-            void Initialize(KEvent *p);
-            Result Signal();
-            Result Clear();
-
-            constexpr KEvent *GetParent() const { return m_parent; }
-    };
-
-}

libraries/libmesosphere/include/mesosphere/kern_kernel.hpp

@@ -63,14 +63,21 @@ namespace ams::kern {
             static constexpr size_t ApplicationMemoryBlockSlabHeapSize = 20000;
             static constexpr size_t SystemMemoryBlockSlabHeapSize      = 10000;
             static constexpr size_t BlockInfoSlabHeapSize              = 4000;
+            static constexpr size_t ReservedDynamicPageCount           = 70;
         private:
             static State s_state;
             static KResourceLimit s_system_resource_limit;
             static KMemoryManager s_memory_manager;
-            static KPageTableManager s_page_table_manager;
+            static KPageTableSlabHeap s_page_table_heap;
+            static KMemoryBlockSlabHeap s_app_memory_block_heap;
+            static KMemoryBlockSlabHeap s_sys_memory_block_heap;
+            static KBlockInfoSlabHeap s_block_info_heap;
+            static KPageTableManager s_app_page_table_manager;
+            static KPageTableManager s_sys_page_table_manager;
             static KMemoryBlockSlabManager s_app_memory_block_manager;
             static KMemoryBlockSlabManager s_sys_memory_block_manager;
-            static KBlockInfoManager s_block_info_manager;
+            static KBlockInfoManager s_app_block_info_manager;
+            static KBlockInfoManager s_sys_block_info_manager;
             static KSupervisorPageTable s_supervisor_page_table;
             static KUnsafeMemory s_unsafe_memory;
             static KWorkerTaskManager s_worker_task_managers[KWorkerTaskManager::WorkerType_Count];
@@ -130,12 +137,20 @@ namespace ams::kern {
                 return s_sys_memory_block_manager;
             }
 
-            static ALWAYS_INLINE KBlockInfoManager &GetBlockInfoManager() {
-                return s_block_info_manager;
+            static ALWAYS_INLINE KBlockInfoManager &GetApplicationBlockInfoManager() {
+                return s_app_block_info_manager;
             }
 
-            static ALWAYS_INLINE KPageTableManager &GetPageTableManager() {
-                return s_page_table_manager;
+            static ALWAYS_INLINE KBlockInfoManager &GetSystemBlockInfoManager() {
+                return s_sys_block_info_manager;
+            }
+
+            static ALWAYS_INLINE KPageTableManager &GetApplicationPageTableManager() {
+                return s_app_page_table_manager;
+            }
+
+            static ALWAYS_INLINE KPageTableManager &GetSystemPageTableManager() {
+                return s_sys_page_table_manager;
             }
 
             static ALWAYS_INLINE KSupervisorPageTable &GetKernelPageTable() {

libraries/libmesosphere/include/mesosphere/kern_select_page_table.hpp

@@ -32,3 +32,24 @@
     #error "Unknown architecture for KPageTable"
 
 #endif
+
+namespace ams::kern {
+
+    /* NOTE: These three functions (Operate, Operate, FinalizeUpdate) are virtual functions */
+    /* in Nintendo's kernel. We devirtualize them, since KPageTable is the only derived */
+    /* class, and this avoids unnecessary virtual function calls. */
+    static_assert(std::derived_from<KPageTable, KPageTableBase>);
+
+    ALWAYS_INLINE Result KPageTableBase::Operate(PageLinkedList *page_list, KProcessAddress virt_addr, size_t num_pages, KPhysicalAddress phys_addr, bool is_pa_valid, const KPageProperties properties, OperationType operation, bool reuse_ll) {
+        return static_cast<KPageTable *>(this)->OperateImpl(page_list, virt_addr, num_pages, phys_addr, is_pa_valid, properties, operation, reuse_ll);
+    }
+
+    ALWAYS_INLINE Result KPageTableBase::Operate(PageLinkedList *page_list, KProcessAddress virt_addr, size_t num_pages, const KPageGroup &page_group, const KPageProperties properties, OperationType operation, bool reuse_ll) {
+        return static_cast<KPageTable *>(this)->OperateImpl(page_list, virt_addr, num_pages, page_group, properties, operation, reuse_ll);
+    }
+
+    ALWAYS_INLINE void KPageTableBase::FinalizeUpdate(PageLinkedList *page_list) {
+        return static_cast<KPageTable *>(this)->FinalizeUpdateImpl(page_list);
+    }
+
+}

libraries/libmesosphere/include/mesosphere/kern_slab_helpers.hpp

@@ -18,15 +18,16 @@
 #include <mesosphere/kern_k_auto_object.hpp>
 #include <mesosphere/kern_k_slab_heap.hpp>
 #include <mesosphere/kern_k_auto_object_container.hpp>
+#include <mesosphere/kern_k_unused_slab_memory.hpp>
 
 namespace ams::kern {
 
-    template<class Derived>
+    template<class Derived, bool SupportDynamicExpansion = false>
     class KSlabAllocated {
         private:
-            static inline KSlabHeap<Derived> s_slab_heap;
+            static constinit inline KSlabHeap<Derived, SupportDynamicExpansion> s_slab_heap;
         public:
-            constexpr KSlabAllocated() { /* ... */ }
+            constexpr KSlabAllocated() = default;
 
             size_t GetSlabIndex() const {
                 return s_slab_heap.GetIndex(static_cast<const Derived *>(this));
@@ -36,14 +37,25 @@ namespace ams::kern {
                 s_slab_heap.Initialize(memory, memory_size);
             }
 
-            static ALWAYS_INLINE Derived *Allocate() {
+            static Derived *Allocate() {
                 return s_slab_heap.Allocate();
             }
 
-            static ALWAYS_INLINE void Free(Derived *obj) {
+            static void Free(Derived *obj) {
                 s_slab_heap.Free(obj);
             }
 
+            template<bool Enable = SupportDynamicExpansion, typename = typename std::enable_if<Enable>::type>
+            static Derived *AllocateFromUnusedSlabMemory() {
+                static_assert(Enable == SupportDynamicExpansion);
+
+                Derived * const obj = GetPointer<Derived>(AllocateUnusedSlabMemory(sizeof(Derived), alignof(Derived)));
+                if (AMS_LIKELY(obj != nullptr)) {
+                    std::construct_at(obj);
+                }
+                return obj;
+            }
+
             static size_t GetObjectSize() { return s_slab_heap.GetObjectSize(); }
             static size_t GetSlabHeapSize() { return s_slab_heap.GetSlabHeapSize(); }
             static size_t GetPeakIndex() { return s_slab_heap.GetPeakIndex(); }
@@ -52,12 +64,12 @@ namespace ams::kern {
             static size_t GetNumRemaining() { return s_slab_heap.GetNumRemaining(); }
     };
 
-    template<typename Derived, typename Base>
+    template<typename Derived, typename Base, bool SupportDynamicExpansion = false>
     class KAutoObjectWithSlabHeapAndContainer : public Base {
         static_assert(std::is_base_of<KAutoObjectWithList, Base>::value);
         private:
-            static inline KSlabHeap<Derived> s_slab_heap;
-            static inline KAutoObjectWithListContainer s_container;
+            static constinit inline KSlabHeap<Derived, SupportDynamicExpansion> s_slab_heap;
+            static constinit inline KAutoObjectWithListContainer s_container;
         private:
             static ALWAYS_INLINE Derived *Allocate() {
                 return s_slab_heap.Allocate();
@@ -73,8 +85,7 @@ namespace ams::kern {
                     ALWAYS_INLINE ~ListAccessor() { /* ... */ }
             };
         public:
-            constexpr KAutoObjectWithSlabHeapAndContainer() : Base() { /* ... */ }
-            virtual ~KAutoObjectWithSlabHeapAndContainer() { /* ... */ }
+            constexpr KAutoObjectWithSlabHeapAndContainer() = default;
 
             virtual void Destroy() override {
                 const bool is_initialized = this->IsInitialized();
@@ -110,6 +121,18 @@ namespace ams::kern {
                 return obj;
             }
 
+            template<bool Enable = SupportDynamicExpansion, typename = typename std::enable_if<Enable>::type>
+            static Derived *CreateFromUnusedSlabMemory() {
+                static_assert(Enable == SupportDynamicExpansion);
+
+                Derived * const obj = GetPointer<Derived>(AllocateUnusedSlabMemory(sizeof(Derived), alignof(Derived)));
+                if (AMS_LIKELY(obj != nullptr)) {
+                    std::construct_at(obj);
+                    KAutoObject::Create(obj);
+                }
+                return obj;
+            }
+
             static void Register(Derived *obj) {
                 return s_container.Register(obj);
             }

libraries/libmesosphere/source/arch/arm64/kern_exception_handlers.cpp

@@ -109,6 +109,11 @@ namespace ams::kern::arch::arm64 {
                     break;
             }
 
+            /* In the event that we return from this exception, we want SPSR.SS set so that we advance an instruction if single-stepping. */
+            #if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP)
+            context->psr |= (1ul << 21);
+            #endif
+
             /* If we should process the user exception (and it's not a breakpoint), try to enter. */
             const bool is_software_break = (ec == EsrEc_Unknown || ec == EsrEc_IllegalExecution || ec == EsrEc_BkptInstruction || ec == EsrEc_BrkInstruction);
             const bool is_breakpoint     = (ec == EsrEc_BreakPointEl0 || ec == EsrEc_SoftwareStepEl0 || ec == EsrEc_WatchPointEl0);
@@ -215,6 +220,15 @@ namespace ams::kern::arch::arm64 {
                 }
             }
 
+            /* If we should, clear the thread's state as single-step. */
+            #if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP)
+            if (AMS_UNLIKELY(GetCurrentThread().IsSingleStep())) {
+                GetCurrentThread().ClearSingleStep();
+                cpu::MonitorDebugSystemControlRegisterAccessor().SetSoftwareStep(false).Store();
+                cpu::EnsureInstructionConsistency();
+            }
+            #endif
+
             {
                 /* Collect additional information based on the ec. */
                 ams::svc::DebugException exception;
@@ -290,16 +304,40 @@ namespace ams::kern::arch::arm64 {
                     return;
                 }
 
-                /* Print that an exception occurred. */
-                MESOSPHERE_RELEASE_LOG("Exception occurred. %016lx\n", GetCurrentProcess().GetProgramId());
-
+                #if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP)
                 {
-                    /* Print the current thread's registers. */
-                    KDebug::PrintRegister();
+                    if (ec != EsrEc_SoftwareStepEl0) {
+                        /* If the exception wasn't single-step, print details. */
+                        MESOSPHERE_EXCEPTION_LOG("Exception occurred. ");
 
-                    /* Print a backtrace. */
-                    KDebug::PrintBacktrace();
+                        {
+                            /* Print the current thread's registers. */
+                            KDebug::PrintRegister();
+
+                            /* Print a backtrace. */
+                            KDebug::PrintBacktrace();
+                        }
+                    } else {
+                        /* If the exception was single-step and we have no debug object, we should just return. */
+                        if (AMS_UNLIKELY(!cur_process.IsAttachedToDebugger())) {
+                            return;
+                        }
+                    }
                 }
+                #else
+                {
+                    /* Print that an exception occurred. */
+                    MESOSPHERE_EXCEPTION_LOG("Exception occurred. ");
+
+                    {
+                        /* Print the current thread's registers. */
+                        KDebug::PrintRegister();
+
+                        /* Print a backtrace. */
+                        KDebug::PrintBacktrace();
+                    }
+                }
+                #endif
 
                 /* If the SVC is handled, handle it. */
                 if (!svc::ResultNotHandled::Includes(result)) {
@@ -458,7 +496,7 @@ namespace ams::kern::arch::arm64 {
         }
 
         /* Print that an exception occurred. */
-        MESOSPHERE_RELEASE_LOG("Exception occurred. %016lx\n", GetCurrentProcess().GetProgramId());
+        MESOSPHERE_EXCEPTION_LOG("Exception occurred. ");
 
         /* Exit the current process. */
         GetCurrentProcess().Exit();
@@ -559,6 +597,7 @@ namespace ams::kern::arch::arm64 {
                 KDpcManager::HandleDpc();
             }
         }
+
         /* Note that we're no longer in an exception handler. */
         GetCurrentThread().ClearInExceptionHandler();
     }

libraries/libmesosphere/source/arch/arm64/kern_k_debug.cpp

@@ -307,8 +307,14 @@ namespace ams::kern::arch::arm64 {
                     R_UNLESS(debug.IsNotNull(), svc::ResultInvalidHandle());
 
                     /* Get the process from the debug object. */
-                    process = debug->GetProcess();
-                    R_UNLESS(process.IsNotNull(), svc::ResultProcessTerminated());
+                    R_UNLESS(debug->IsAttached(),  svc::ResultProcessTerminated());
+                    R_UNLESS(debug->OpenProcess(), svc::ResultProcessTerminated());
+
+                    /* Close the process when we're done. */
+                    ON_SCOPE_EXIT { debug->CloseProcess(); };
+
+                    /* Get the proces. */
+                    KProcess * const process = debug->GetProcessUnsafe();
 
                     /* Set the value to be the context id. */
                     value = process->GetId() & 0xFFFFFFFF;

libraries/libmesosphere/source/arch/arm64/kern_k_page_table.cpp

@@ -166,7 +166,7 @@ namespace ams::kern::arch::arm64 {
     Result KPageTable::InitializeForKernel(void *table, KVirtualAddress start, KVirtualAddress end) {
         /* Initialize basic fields. */
         m_asid = 0;
-        m_manager = std::addressof(Kernel::GetPageTableManager());
+        m_manager = std::addressof(Kernel::GetSystemPageTableManager());
 
         /* Allocate a page for ttbr. */
         const u64 asid_tag = (static_cast<u64>(m_asid) << 48ul);
@@ -247,7 +247,7 @@ namespace ams::kern::arch::arm64 {
                         cur_entry.block_size += next_entry.block_size;
                     } else {
                         if (cur_valid && IsHeapPhysicalAddressForFinalize(cur_entry.phys_addr)) {
-                            mm.Close(GetHeapVirtualAddress(cur_entry.phys_addr), cur_entry.block_size / PageSize);
+                            mm.Close(cur_entry.phys_addr, cur_entry.block_size / PageSize);
                         }
 
                         /* Update tracking variables. */
@@ -265,7 +265,7 @@ namespace ams::kern::arch::arm64 {
 
                 /* Handle the last block. */
                 if (cur_valid && IsHeapPhysicalAddressForFinalize(cur_entry.phys_addr)) {
-                    mm.Close(GetHeapVirtualAddress(cur_entry.phys_addr), cur_entry.block_size / PageSize);
+                    mm.Close(cur_entry.phys_addr, cur_entry.block_size / PageSize);
                 }
             }
 
@@ -319,7 +319,7 @@ namespace ams::kern::arch::arm64 {
         return ResultSuccess();
     }
 
-    Result KPageTable::Operate(PageLinkedList *page_list, KProcessAddress virt_addr, size_t num_pages, KPhysicalAddress phys_addr, bool is_pa_valid, const KPageProperties properties, OperationType operation, bool reuse_ll) {
+    Result KPageTable::OperateImpl(PageLinkedList *page_list, KProcessAddress virt_addr, size_t num_pages, KPhysicalAddress phys_addr, bool is_pa_valid, const KPageProperties properties, OperationType operation, bool reuse_ll) {
         /* Check validity of parameters. */
         MESOSPHERE_ASSERT(this->IsLockedByCurrentThread());
         MESOSPHERE_ASSERT(num_pages > 0);
@@ -350,7 +350,7 @@ namespace ams::kern::arch::arm64 {
         }
     }
 
-    Result KPageTable::Operate(PageLinkedList *page_list, KProcessAddress virt_addr, size_t num_pages, const KPageGroup &page_group, const KPageProperties properties, OperationType operation, bool reuse_ll) {
+    Result KPageTable::OperateImpl(PageLinkedList *page_list, KProcessAddress virt_addr, size_t num_pages, const KPageGroup &page_group, const KPageProperties properties, OperationType operation, bool reuse_ll) {
         /* Check validity of parameters. */
         MESOSPHERE_ASSERT(this->IsLockedByCurrentThread());
         MESOSPHERE_ASSERT(util::IsAligned(GetInteger(virt_addr), PageSize));
@@ -579,7 +579,7 @@ namespace ams::kern::arch::arm64 {
 
         /* Ensure that any pages we track close on exit. */
         KPageGroup pages_to_close(this->GetBlockInfoManager());
-        ON_SCOPE_EXIT { pages_to_close.Close(); };
+        ON_SCOPE_EXIT { pages_to_close.CloseAndReset(); };
 
         /* Begin traversal. */
         TraversalContext context;
@@ -601,8 +601,9 @@ namespace ams::kern::arch::arm64 {
             if (next_entry.block_size > remaining_pages * PageSize) {
                 MESOSPHERE_ABORT_UNLESS(force);
                 MESOSPHERE_R_ABORT_UNLESS(this->SeparatePages(virt_addr, remaining_pages * PageSize, page_list, reuse_ll));
-                next_valid = impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), virt_addr);
-                MESOSPHERE_ASSERT(next_valid);
+                const bool new_valid = impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), virt_addr);
+                MESOSPHERE_ASSERT(new_valid);
+                MESOSPHERE_UNUSED(new_valid);
             }
 
             /* Check that our state is coherent. */
@@ -641,6 +642,7 @@ namespace ams::kern::arch::arm64 {
                                 *l1_entry = InvalidL1PageTableEntry;
                                 this->NoteUpdated();
                                 this->FreePageTable(page_list, l2_virt);
+                                pages_to_close.CloseAndReset();
                             }
                         }
                     }
@@ -684,6 +686,7 @@ namespace ams::kern::arch::arm64 {
                                 }
 
                                 this->FreePageTable(page_list, l3_virt);
+                                pages_to_close.CloseAndReset();
                             }
                         }
                     }
@@ -693,11 +696,11 @@ namespace ams::kern::arch::arm64 {
 
             /* Close the blocks. */
             if (!force && IsHeapPhysicalAddress(next_entry.phys_addr)) {
-                const KVirtualAddress block_virt_addr = GetHeapVirtualAddress(next_entry.phys_addr);
                 const size_t block_num_pages = next_entry.block_size / PageSize;
-                if (R_FAILED(pages_to_close.AddBlock(block_virt_addr, block_num_pages))) {
+                if (R_FAILED(pages_to_close.AddBlock(next_entry.phys_addr, block_num_pages))) {
                     this->NoteUpdated();
-                    Kernel::GetMemoryManager().Close(block_virt_addr, block_num_pages);
+                    Kernel::GetMemoryManager().Close(next_entry.phys_addr, block_num_pages);
+                    pages_to_close.CloseAndReset();
                 }
             }
 
@@ -788,7 +791,7 @@ namespace ams::kern::arch::arm64 {
 
         /* Open references to the pages, if we should. */
         if (IsHeapPhysicalAddress(orig_phys_addr)) {
-            Kernel::GetMemoryManager().Open(GetHeapVirtualAddress(orig_phys_addr), num_pages);
+            Kernel::GetMemoryManager().Open(orig_phys_addr, num_pages);
         }
 
         return ResultSuccess();
@@ -811,7 +814,7 @@ namespace ams::kern::arch::arm64 {
 
             if (num_pages < ContiguousPageSize / PageSize) {
                 for (const auto &block : pg) {
-                    const KPhysicalAddress block_phys_addr = GetLinearMappedPhysicalAddress(block.GetAddress());
+                    const KPhysicalAddress block_phys_addr = block.GetAddress();
                     const size_t cur_pages = block.GetNumPages();
                     R_TRY(this->Map(virt_addr, block_phys_addr, cur_pages, entry_template, disable_head_merge && virt_addr == orig_virt_addr, L3BlockSize, page_list, reuse_ll));
 
@@ -823,7 +826,7 @@ namespace ams::kern::arch::arm64 {
                 AlignedMemoryBlock virt_block(GetInteger(virt_addr), num_pages, L1BlockSize);
                 for (const auto &block : pg) {
                     /* Create a block representing this physical group, synchronize its alignment to our virtual block. */
-                    const KPhysicalAddress block_phys_addr = GetLinearMappedPhysicalAddress(block.GetAddress());
+                    const KPhysicalAddress block_phys_addr = block.GetAddress();
                     size_t cur_pages = block.GetNumPages();
 
                     AlignedMemoryBlock phys_block(GetInteger(block_phys_addr), cur_pages, virt_block.GetAlignment());
@@ -1426,7 +1429,7 @@ namespace ams::kern::arch::arm64 {
         return ResultSuccess();
     }
 
-    void KPageTable::FinalizeUpdate(PageLinkedList *page_list) {
+    void KPageTable::FinalizeUpdateImpl(PageLinkedList *page_list) {
         while (page_list->Peek()) {
             KVirtualAddress page = KVirtualAddress(page_list->Pop());
             MESOSPHERE_ASSERT(this->GetPageTableManager().IsInPageTableHeap(page));

libraries/libmesosphere/source/arch/arm64/svc/kern_svc_exception_asm.s

@@ -89,6 +89,12 @@ _ZN3ams4kern3svc14RestoreContextEm:
     ldp     x30, x8,  [sp, #(EXCEPTION_CONTEXT_X30_SP)]
     ldp     x9,  x10, [sp, #(EXCEPTION_CONTEXT_PC_PSR)]
     ldr     x11,      [sp, #(EXCEPTION_CONTEXT_TPIDR)]
+
+    #if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP)
+    /* Since we're returning from an exception, set SPSR.SS so that we advance an instruction if single-stepping. */
+    orr x10, x10, #(1 << 21)
+    #endif
+
     msr     sp_el0, x8
     msr     elr_el1, x9
     msr     spsr_el1, x10