docs: add changelog for 1.0.0

subrepo:
  subdir:   "emummc"
  merged:   "7a3db0fb"
upstream:
  origin:   "https://github.com/m4xw/emuMMC"
  branch:   "develop"
  commit:   "f6608731"
git-subrepo:
  version:  "0.4.1"
  origin:   "???"
  commit:   "???"

docs/changelog.md

@@ -1,4 +1,20 @@
 # Changelog
+## 1.1.0
++ Support was implemented for 13.0.0.
+  + `mesosphère` was updated to reflect the latest official kernel behavior.
+  + `ncm` was updated to reflect the latest official behaviors.
+  + `erpt` was  updated to reflect the latest official behaviors.
+    + Two new services ("sprofile") were added to `erpt`, and have been fully reimplemented.
+    + **Please Note**: These services provide a way for settings to be pushed to consoles over the internet without system update.
+      + Because there appear to be no settings pushed out yet, this implementation fundamentally cannot be fully tested right now, but hopefully there are no issues once settings begin being distributed.
++ The `LogManager` system module was reimplemented.
+  + This system module provides services that some games use for logging.
+  + Atmosphere's reimplementation supports logging to the SD card (if `lm!enable_sd_card_logging` is true) and to ams.TMA.
+    + To control the directory where logs are saved, modify the `lm!sd_card_log_output_directory` setting.
+  + Atmosphere's reimplementation is disabled by default (in order to save memory), but can be enabled by setting `lm!enable_log_manager` to true.
+    + This will allow reading over logs from games which use the services (or potentially logging from homebrew in the future), which can be useful to developers.
+    + Please note that when TMA is fully implemented in the future, enabling TMA will forcibly enable `LogManager`.
++ General system stability improvements to enhance the user's experience.
 ## 1.0.0
 + `fusee` was completely re-written in C++ to use the same atmosphere-libs APIs as the rest of atmosphere's code.
   + The rewrite was performed with a big emphasis on ensuring a good boot speed, and generally boot should be much faster than it was previously.

emummc/.gitrepo

@@ -6,7 +6,7 @@
 [subrepo]
 	remote = https://github.com/m4xw/emuMMC
 	branch = develop
-	commit = cbc294c390ed73bb281bc1028a8899c053427112
+	commit = f66087313546161a000ee196a788f0626caf80fa
 	parent = 38f9a76ba028995ed3274da3a45b0254f09d1f59
 	method = rebase
 	cmdver = 0.4.1

emummc/README.md

@@ -2,7 +2,7 @@
 *A SDMMC driver replacement for Nintendo's Filesystem Services, by **m4xw***
 
 ### Supported Horizon Versions
-**1.0.0 - 11.0.0**
+**1.0.0 - 13.0.0**
 
 ## Features
 * Arbitrary SDMMC backend selection

emummc/source/FS/FS.h

@@ -37,4 +37,15 @@
 #define BOOT_PARTITION_SIZE 0x2000
 #define FS_READ_WRITE_ERROR 1048
 
+#define NAND_PATROL_SECTOR 0xC20
+#define NAND_PATROL_OFFSET 0x184000
+
+typedef struct _fs_nand_patrol_t
+{
+	uint8_t hmac[0x20];
+	unsigned int offset;
+	unsigned int count;
+	uint8_t rsvd[0x1D8];
+} fs_nand_patrol_t;
+
 #endif /* __FS_H__ */

emummc/source/FS/FS_offsets.c

@@ -55,6 +55,8 @@
 #include "offsets/1200_exfat.h"
 #include "offsets/1203.h"
 #include "offsets/1203_exfat.h"
+#include "offsets/1300.h"
+#include "offsets/1300_exfat.h"
 #include "../utils/fatal.h"
 
 #define GET_OFFSET_STRUCT_NAME(vers) g_offsets##vers
@@ -121,6 +123,8 @@ DEFINE_OFFSET_STRUCT(_1200);
 DEFINE_OFFSET_STRUCT(_1200_EXFAT);
 DEFINE_OFFSET_STRUCT(_1203);
 DEFINE_OFFSET_STRUCT(_1203_EXFAT);
+DEFINE_OFFSET_STRUCT(_1300);
+DEFINE_OFFSET_STRUCT(_1300_EXFAT);
 
 const fs_offsets_t *get_fs_offsets(enum FS_VER version) {
     switch (version) {
@@ -202,6 +206,10 @@ const fs_offsets_t *get_fs_offsets(enum FS_VER version) {
             return &(GET_OFFSET_STRUCT_NAME(_1203));
         case FS_VER_12_0_3_EXFAT:
             return &(GET_OFFSET_STRUCT_NAME(_1203_EXFAT));
+        case FS_VER_13_0_0:
+            return &(GET_OFFSET_STRUCT_NAME(_1300));
+        case FS_VER_13_0_0_EXFAT:
+            return &(GET_OFFSET_STRUCT_NAME(_1300_EXFAT));
         default:
             fatal_abort(Fatal_UnknownVersion);
     }

emummc/source/FS/FS_versions.h

@@ -80,6 +80,9 @@ enum FS_VER
     FS_VER_12_0_3,
     FS_VER_12_0_3_EXFAT,
 
+    FS_VER_13_0_0,
+    FS_VER_13_0_0_EXFAT,
+
     FS_VER_MAX,
 };
 

emummc/source/FS/offsets/1300.h

@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2019 m4xw <m4x@m4xw.net>
+ * Copyright (c) 2019 Atmosphere-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef __FS_1300_H__
+#define __FS_1300_H__
+
+// Accessor vtable getters
+#define FS_OFFSET_1300_SDMMC_ACCESSOR_GC   0x158C80
+#define FS_OFFSET_1300_SDMMC_ACCESSOR_SD   0x15AA90
+#define FS_OFFSET_1300_SDMMC_ACCESSOR_NAND 0x1591B0
+
+// Hooks
+#define FS_OFFSET_1300_SDMMC_WRAPPER_READ  0x154620
+#define FS_OFFSET_1300_SDMMC_WRAPPER_WRITE 0x1546E0
+#define FS_OFFSET_1300_RTLD                0x688
+#define FS_OFFSET_1300_RTLD_DESTINATION    ((uintptr_t)(INT64_C(-0x3C)))
+
+#define FS_OFFSET_1300_CLKRST_SET_MIN_V_CLK_RATE 0x153820
+
+// Misc funcs
+#define FS_OFFSET_1300_LOCK_MUTEX          0x29690
+#define FS_OFFSET_1300_UNLOCK_MUTEX        0x296E0
+
+#define FS_OFFSET_1300_SDMMC_WRAPPER_CONTROLLER_OPEN  0x154500
+#define FS_OFFSET_1300_SDMMC_WRAPPER_CONTROLLER_CLOSE 0x154590
+
+// Misc Data
+#define FS_OFFSET_1300_SD_MUTEX            0xE133E8
+#define FS_OFFSET_1300_NAND_MUTEX          0xE0E768
+#define FS_OFFSET_1300_ACTIVE_PARTITION    0xE0E7A8
+#define FS_OFFSET_1300_SDMMC_DAS_HANDLE    0xDF6E18
+
+// NOPs
+#define FS_OFFSET_1300_SD_DAS_INIT         0x27744
+
+// Nintendo Paths
+#define FS_OFFSET_1300_NINTENDO_PATHS \
+{ \
+	{.opcode_reg = 3, .adrp_offset = 0x0006EBE0, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 3, .adrp_offset = 0x0007BEEC, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 4, .adrp_offset = 0x00082294, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 4, .adrp_offset = 0x0009422C, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 0, .adrp_offset = 0, .add_rel_offset = 0}, \
+}
+
+#endif // __FS_1300_H__

emummc/source/FS/offsets/1300_exfat.h

@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2019 m4xw <m4x@m4xw.net>
+ * Copyright (c) 2019 Atmosphere-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef __FS_1300_EXFAT_H__
+#define __FS_1300_EXFAT_H__
+
+// Accessor vtable getters
+#define FS_OFFSET_1300_EXFAT_SDMMC_ACCESSOR_GC   0x158C80
+#define FS_OFFSET_1300_EXFAT_SDMMC_ACCESSOR_SD   0x15AA90
+#define FS_OFFSET_1300_EXFAT_SDMMC_ACCESSOR_NAND 0x1591B0
+
+// Hooks
+#define FS_OFFSET_1300_EXFAT_SDMMC_WRAPPER_READ  0x154620
+#define FS_OFFSET_1300_EXFAT_SDMMC_WRAPPER_WRITE 0x1546E0
+#define FS_OFFSET_1300_EXFAT_RTLD                0x688
+#define FS_OFFSET_1300_EXFAT_RTLD_DESTINATION    ((uintptr_t)(INT64_C(-0x3C)))
+
+#define FS_OFFSET_1300_EXFAT_CLKRST_SET_MIN_V_CLK_RATE 0x153820
+
+// Misc funcs
+#define FS_OFFSET_1300_EXFAT_LOCK_MUTEX          0x29690
+#define FS_OFFSET_1300_EXFAT_UNLOCK_MUTEX        0x296E0
+
+#define FS_OFFSET_1300_EXFAT_SDMMC_WRAPPER_CONTROLLER_OPEN  0x154500
+#define FS_OFFSET_1300_EXFAT_SDMMC_WRAPPER_CONTROLLER_CLOSE 0x154590
+
+// Misc Data
+#define FS_OFFSET_1300_EXFAT_SD_MUTEX            0xE203E8
+#define FS_OFFSET_1300_EXFAT_NAND_MUTEX          0xE1B768
+#define FS_OFFSET_1300_EXFAT_ACTIVE_PARTITION    0xE1B7A8
+#define FS_OFFSET_1300_EXFAT_SDMMC_DAS_HANDLE    0xE03E18
+
+// NOPs
+#define FS_OFFSET_1300_EXFAT_SD_DAS_INIT         0x27744
+
+// Nintendo Paths
+#define FS_OFFSET_1300_EXFAT_NINTENDO_PATHS \
+{ \
+	{.opcode_reg = 3, .adrp_offset = 0x0006EBE0, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 3, .adrp_offset = 0x0007BEEC, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 4, .adrp_offset = 0x00082294, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 4, .adrp_offset = 0x0009422C, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 0, .adrp_offset = 0, .add_rel_offset = 0}, \
+}
+
+#endif // __FS_1300_EXFAT_H__

emummc/source/emuMMC/emummc.c

@@ -89,7 +89,7 @@ static void _sdmmc_ensure_initialized(void)
     }
 }
 
-static void _file_based_update_filename(char *outFilename, u32 sd_path_len, u32 part_idx)
+static void _file_based_update_filename(char *outFilename, unsigned int sd_path_len, unsigned int part_idx)
 {
     snprintf(outFilename + sd_path_len, 3, "%02d", part_idx);
 }
@@ -103,9 +103,7 @@ static void _file_based_emmc_finalize(void)
         f_close(&f_emu.fp_boot1);
 
         for (int i = 0; i < f_emu.parts; i++)
-        {
             f_close(&f_emu.fp_gpp[i]);
-        }
 
         // Force unmount FAT volume.
         f_mount(NULL, "", 1);
@@ -114,12 +112,59 @@ static void _file_based_emmc_finalize(void)
     }
 }
 
+static void _nand_patrol_ensure_integrity(void)
+{
+    fs_nand_patrol_t nand_patrol;
+    static bool nand_patrol_checked = false;
+
+    if (!nand_patrol_checked)
+    {
+        if (emuMMC_ctx.EMMC_Type == emuMMC_SD_Raw)
+        {
+            unsigned int nand_patrol_sector = emuMMC_ctx.EMMC_StoragePartitionOffset + NAND_PATROL_SECTOR;
+            if (!sdmmc_storage_read(&sd_storage, nand_patrol_sector, 1, &nand_patrol))
+                goto out;
+
+            // Clear nand patrol if last offset exceeds storage.
+            if (nand_patrol.offset > sd_storage.sec_cnt)
+            {
+                memset(&nand_patrol, 0, sizeof(fs_nand_patrol_t));
+                sdmmc_storage_write(&sd_storage, nand_patrol_sector, 1, &nand_patrol);
+            }
+        }
+        else if (emuMMC_ctx.EMMC_Type == emuMMC_SD_File && fat_mounted)
+        {
+            FIL *fp = &f_emu.fp_boot0;
+            if (f_lseek(fp, NAND_PATROL_OFFSET) != FR_OK)
+                goto out;
+
+            if (f_read_fast(fp, &nand_patrol, sizeof(fs_nand_patrol_t)) != FR_OK)
+                goto out;
+
+            // Clear nand patrol if last offset exceeds total file based size.
+            if (nand_patrol.offset > f_emu.total_sect)
+            {
+                memset(&nand_patrol, 0, sizeof(fs_nand_patrol_t));
+
+                if (f_lseek(fp, NAND_PATROL_OFFSET) != FR_OK)
+                    goto out;
+
+                if (f_write_fast(fp, &nand_patrol, sizeof(fs_nand_patrol_t)) != FR_OK)
+                    goto out;
+
+                f_sync(fp);
+            }
+        }
+
+out:
+        nand_patrol_checked = true;
+    }
+}
+
 void sdmmc_finalize(void)
 {
     if (!sdmmc_storage_end(&sd_storage))
-    {
         fatal_abort(Fatal_InitSD);
-    }
 
     storageSDinitialized = false;
 }
@@ -137,14 +182,14 @@ static void _file_based_emmc_initialize(void)
     memcpy(path + path_len, "BOOT0", 6);
     if (f_open(&f_emu.fp_boot0, path, FA_READ | FA_WRITE) != FR_OK)
         fatal_abort(Fatal_FatfsFileOpen);
-    if (!f_expand_cltbl(&f_emu.fp_boot0, 0x400, f_emu.clmt_boot0, f_size(&f_emu.fp_boot0)))
+    if (!f_expand_cltbl(&f_emu.fp_boot0, EMUMMC_FP_CLMT_COUNT, f_emu.clmt_boot0, f_size(&f_emu.fp_boot0)))
         fatal_abort(Fatal_FatfsMemExhaustion);
 
     // Open BOOT1 physical partition.
     memcpy(path + path_len, "BOOT1", 6);
     if (f_open(&f_emu.fp_boot1, path, FA_READ | FA_WRITE) != FR_OK)
         fatal_abort(Fatal_FatfsFileOpen);
-    if (!f_expand_cltbl(&f_emu.fp_boot1, 0x400, f_emu.clmt_boot1, f_size(&f_emu.fp_boot1)))
+    if (!f_expand_cltbl(&f_emu.fp_boot1, EMUMMC_FP_CLMT_COUNT, f_emu.clmt_boot1, f_size(&f_emu.fp_boot1)))
         fatal_abort(Fatal_FatfsMemExhaustion);
 
     // Open handles for GPP physical partition files.
@@ -152,15 +197,14 @@ static void _file_based_emmc_initialize(void)
 
     if (f_open(&f_emu.fp_gpp[0], path, FA_READ | FA_WRITE) != FR_OK)
         fatal_abort(Fatal_FatfsFileOpen);
-    if (!f_expand_cltbl(&f_emu.fp_gpp[0], 0x400, &f_emu.clmt_gpp[0], f_size(&f_emu.fp_gpp[0])))
+    if (!f_expand_cltbl(&f_emu.fp_gpp[0], EMUMMC_FP_CLMT_COUNT, &f_emu.clmt_gpp[0], f_size(&f_emu.fp_gpp[0])))
         fatal_abort(Fatal_FatfsMemExhaustion);
 
-    f_emu.part_size = f_size(&f_emu.fp_gpp[0]) >> 9;
+    f_emu.part_size = (uint64_t)f_size(&f_emu.fp_gpp[0]) >> 9;
+    f_emu.total_sect = f_emu.part_size;
 
     // Iterate folder for split parts and stop if next doesn't exist.
-    // Supports up to 32 parts of any size.
+    for (f_emu.parts = 1; f_emu.parts < EMUMMC_FILE_MAX_PARTS; f_emu.parts++)
-    // TODO: decide on max parts and define them. (hekate produces up to 30 parts on 1GB mode.)
-    for (f_emu.parts = 1; f_emu.parts < 32; f_emu.parts++)
     {
         _file_based_update_filename(path, path_len, f_emu.parts);
 
@@ -173,8 +217,13 @@ static void _file_based_emmc_initialize(void)
             return;
         }
 
-        if (!f_expand_cltbl(&f_emu.fp_gpp[f_emu.parts], 0x400, &f_emu.clmt_gpp[f_emu.parts * 0x400], f_size(&f_emu.fp_gpp[f_emu.parts])))
+        if (!f_expand_cltbl(&f_emu.fp_gpp[f_emu.parts], EMUMMC_FP_CLMT_COUNT,
+            &f_emu.clmt_gpp[f_emu.parts * EMUMMC_FP_CLMT_COUNT], f_size(&f_emu.fp_gpp[f_emu.parts])))
+        {
             fatal_abort(Fatal_FatfsMemExhaustion);
+        }
+
+        f_emu.total_sect += (uint64_t)f_size(&f_emu.fp_gpp[f_emu.parts]) >> 9;
     }
 }
 
@@ -189,7 +238,7 @@ bool sdmmc_initialize(void)
             {
                 storageSDinitialized = true;
 
-                // File based emummc.
+                // Init file based emummc.
                 if ((emuMMC_ctx.EMMC_Type == emuMMC_SD_File) && !fat_mounted)
                 {
                     if (f_mount(&f_emu.sd_fs, "", 1) != FR_OK)
@@ -200,6 +249,9 @@ bool sdmmc_initialize(void)
                     _file_based_emmc_initialize();
                 }
 
+                // Check if nand patrol offset is inside limits.
+                _nand_patrol_ensure_integrity();
+
                 break;
             }
 
@@ -207,9 +259,7 @@ bool sdmmc_initialize(void)
         }
 
         if (!storageSDinitialized)
-        {
             fatal_abort(Fatal_InitSD);
-        }
     }
 
     return storageSDinitialized;
@@ -239,19 +289,17 @@ sdmmc_accessor_t *sdmmc_accessor_get(int mmc_id)
 void mutex_lock_handler(int mmc_id)
 {
     if (custom_driver)
-    {
         lock_mutex(sd_mutex);
-    }
+
     lock_mutex(nand_mutex);
 }
 
 void mutex_unlock_handler(int mmc_id)
 {
     unlock_mutex(nand_mutex);
+
     if (custom_driver)
-    {
         unlock_mutex(sd_mutex);
-    }
 }
 
 int sdmmc_nand_get_active_partition_index()
@@ -271,12 +319,16 @@ int sdmmc_nand_get_active_partition_index()
 
 static uint64_t emummc_read_write_inner(void *buf, unsigned int sector, unsigned int num_sectors, bool is_write)
 {
-    if ((emuMMC_ctx.EMMC_Type == emuMMC_SD_Raw))
+    if (emuMMC_ctx.EMMC_Type == emuMMC_SD_Raw)
     {
         // raw partition sector offset: emuMMC_ctx.EMMC_StoragePartitionOffset.
         sector += emuMMC_ctx.EMMC_StoragePartitionOffset;
         // Set physical partition offset.
         sector += (sdmmc_nand_get_active_partition_index() * BOOT_PARTITION_SIZE);
+
+        if (__builtin_expect(sector + num_sectors > sd_storage.sec_cnt, 0))
+            return 0; // Out of bounds. Can only happen with Nand Patrol if resized.
+
         if (!is_write)
             return sdmmc_storage_read(&sd_storage, sector, num_sectors, buf);
         else
@@ -290,6 +342,9 @@ static uint64_t emummc_read_write_inner(void *buf, unsigned int sector, unsigned
     case FS_EMMC_PARTITION_GPP:
         if (f_emu.parts)
         {
+            if (__builtin_expect(sector + num_sectors > f_emu.total_sect, 0))
+                return 0; // Out of bounds. Can only happen with Nand Patrol if resized.
+
             fp = &f_emu.fp_gpp[sector / f_emu.part_size];
             sector = sector % f_emu.part_size;
 
@@ -300,21 +355,21 @@ static uint64_t emummc_read_write_inner(void *buf, unsigned int sector, unsigned
                 while (remaining > 0) {
                     const unsigned int cur_sectors = MIN(remaining, f_emu.part_size - sector);
 
-                    if (f_lseek(fp, (u64)sector << 9) != FR_OK)
+                    if (f_lseek(fp, (uint64_t)sector << 9) != FR_OK)
                         return 0; // Out of bounds.
 
-                    if (is_write)
+                    if (!is_write)
                     {
-                        if (f_write_fast(fp, buf, (u64)cur_sectors << 9) != FR_OK)
+                        if (f_read_fast(fp, buf, (uint64_t)cur_sectors << 9) != FR_OK)
                             return 0;
                     }
                     else
                     {
-                        if (f_read_fast(fp, buf, (u64)cur_sectors << 9) != FR_OK)
+                        if (f_write_fast(fp, buf, (uint64_t)cur_sectors << 9) != FR_OK)
                             return 0;
                     }
 
-                    buf = (char *)buf + ((u64)cur_sectors << 9);
+                    buf = (char *)buf + ((uint64_t)cur_sectors << 9);
                     remaining -= cur_sectors;
                     sector = 0;
                     ++fp;
@@ -324,28 +379,25 @@ static uint64_t emummc_read_write_inner(void *buf, unsigned int sector, unsigned
             }
         }
         else
-        {
             fp = &f_emu.fp_gpp[0];
-        }
         break;
+
     case FS_EMMC_PARTITION_BOOT1:
         fp = &f_emu.fp_boot1;
         break;
+
     case FS_EMMC_PARTITION_BOOT0:
         fp = &f_emu.fp_boot0;
         break;
     }
 
-    if (f_lseek(fp, (u64)sector << 9) != FR_OK)
+    if (f_lseek(fp, (uint64_t)sector << 9) != FR_OK)
-        return 0; // Out of bounds.
+        return 0; // Out of bounds. Can only happen with Nand Patrol if resized.
 
-    uint64_t res = 0;
     if (!is_write)
-        res = !f_read_fast(fp, buf, (u64)num_sectors << 9);
+        return !f_read_fast(fp, buf, (uint64_t)num_sectors << 9);
     else
-        res = !f_write_fast(fp, buf, (u64)num_sectors << 9);
+        return !f_write_fast(fp, buf, (uint64_t)num_sectors << 9);
-
-    return res;
 }
 
 // Controller open wrapper
@@ -382,9 +434,7 @@ uint64_t sdmmc_wrapper_controller_close(int mmc_id)
     if (_this != NULL)
     {
         if (mmc_id == FS_SDMMC_SD)
-        {
             return 0;
-        }
 
         if (mmc_id == FS_SDMMC_EMMC)
         {
@@ -504,8 +554,6 @@ uint64_t sdmmc_wrapper_write(int mmc_id, unsigned int sector, unsigned int num_s
             mutex_lock_handler(mmc_id);
             _current_accessor = _this;
 
-            sector += 0;
-
             // Call hekates driver.
             if (sdmmc_storage_write(&sd_storage, sector, num_sectors, buf))
             {

emummc/source/emuMMC/emummc.h

@@ -36,6 +36,9 @@ extern "C" {
 #include "../FS/FS.h"
 #include "../libs/fatfs/ff.h"
 
+#define EMUMMC_FILE_MAX_PARTS 32
+#define EMUMMC_FP_CLMT_COUNT  1024
+
 // FS typedefs
 typedef sdmmc_accessor_t *(*_sdmmc_accessor_gc)();
 typedef sdmmc_accessor_t *(*_sdmmc_accessor_sd)();
@@ -63,11 +66,12 @@ typedef struct _file_based_ctxt
 	uint64_t parts;
 	uint64_t part_size;
 	FIL fp_boot0;
-	DWORD clmt_boot0[0x400];
+	DWORD clmt_boot0[EMUMMC_FP_CLMT_COUNT];
 	FIL fp_boot1;
-	DWORD clmt_boot1[0x400];
+	DWORD clmt_boot1[EMUMMC_FP_CLMT_COUNT];
-	FIL fp_gpp[32];
+	FIL fp_gpp[EMUMMC_FILE_MAX_PARTS];
-	DWORD clmt_gpp[0x8000];
+	DWORD clmt_gpp[EMUMMC_FILE_MAX_PARTS * EMUMMC_FP_CLMT_COUNT];
+	uint64_t total_sect;
 } file_based_ctxt;
 
 #ifdef __cplusplus

exosphere/program/source/boot/secmon_boot_key_data.s

@@ -85,10 +85,10 @@ _ZN3ams6secmon4boot15VolatileKeyDataE:
 /* We can get away with only including latest because exosphere supports newer-than-expected master key in engine. */
 /* TODO: Update on next change of keys. */
 /* Mariko Development Master Kek Source. */
-.byte 0x75, 0x2D, 0x2E, 0xF3, 0x2F, 0x3F, 0xFE, 0x65, 0xF4, 0xA9, 0x83, 0xB4, 0xED, 0x42, 0x63, 0xBA
+.byte 0x4D, 0x5A, 0xB2, 0xC9, 0xE9, 0xE4, 0x4E, 0xA4, 0xD3, 0xBF, 0x94, 0x12, 0x36, 0x30, 0xD0, 0x7F
 
 /* Mariko Production Master Kek Source. */
-.byte 0xE5, 0x41, 0xAC, 0xEC, 0xD1, 0xA7, 0xD1, 0xAB, 0xED, 0x03, 0x77, 0xF1, 0x27, 0xCA, 0xF8, 0xF1
+.byte 0x52, 0x71, 0x9B, 0xDF, 0xA7, 0x8B, 0x61, 0xD8, 0xD5, 0x85, 0x11, 0xE4, 0x8E, 0x4F, 0x74, 0xC6
 
 /* Development Master Key Vectors. */
 .byte 0x46, 0x22, 0xB4, 0x51, 0x9A, 0x7E, 0xA7, 0x7F, 0x62, 0xA1, 0x1F, 0x8F, 0xC5, 0x3A, 0xDB, 0xFE /* Zeroes encrypted with Master Key 00. */
@@ -103,6 +103,7 @@ _ZN3ams6secmon4boot15VolatileKeyDataE:
 .byte 0x43, 0x3D, 0xC5, 0x3B, 0xEF, 0x91, 0x02, 0x21, 0x61, 0x54, 0x63, 0x8A, 0x35, 0xE7, 0xCA, 0xEE /* Master key 08 encrypted with Master key 09. */
 .byte 0x6C, 0x2E, 0xCD, 0xB3, 0x34, 0x61, 0x77, 0xF5, 0xF9, 0xB1, 0xDD, 0x61, 0x98, 0x19, 0x3E, 0xD4 /* Master key 09 encrypted with Master key 0A. */
 .byte 0x21, 0x88, 0x6B, 0x10, 0x9E, 0x83, 0xD6, 0x52, 0xAB, 0x08, 0xDB, 0x6D, 0x39, 0xFF, 0x1C, 0x9C /* Master key 0A encrypted with Master key 0B. */
+.byte 0x8A, 0xCE, 0xC4, 0x7F, 0xBE, 0x08, 0x61, 0x88, 0xD3, 0x73, 0x64, 0x51, 0xE2, 0xB6, 0x53, 0x15 /* Master key 0B encrypted with Master key 0C. */
 
 /* Production Master Key Vectors. */
 .byte 0x0C, 0xF0, 0x59, 0xAC, 0x85, 0xF6, 0x26, 0x65, 0xE1, 0xE9, 0x19, 0x55, 0xE6, 0xF2, 0x67, 0x3D /* Zeroes encrypted with Master Key 00. */
@@ -117,6 +118,7 @@ _ZN3ams6secmon4boot15VolatileKeyDataE:
 .byte 0x4D, 0xD9, 0x98, 0x42, 0x45, 0x0D, 0xB1, 0x3C, 0x52, 0x0C, 0x9A, 0x44, 0xBB, 0xAD, 0xAF, 0x80 /* Master key 08 encrypted with Master key 09. */
 .byte 0xB8, 0x96, 0x9E, 0x4A, 0x00, 0x0D, 0xD6, 0x28, 0xB3, 0xD1, 0xDB, 0x68, 0x5F, 0xFB, 0xE1, 0x2A /* Master key 09 encrypted with Master key 0A. */
 .byte 0xC1, 0x8D, 0x16, 0xBB, 0x2A, 0xE4, 0x1D, 0xD4, 0xC2, 0xC1, 0xB6, 0x40, 0x94, 0x35, 0x63, 0x98 /* Master key 0A encrypted with Master key 0B. */
+.byte 0xA3, 0x24, 0x65, 0x75, 0xEA, 0xCC, 0x6E, 0x8D, 0xFB, 0x5A, 0x16, 0x50, 0x74, 0xD2, 0x15, 0x06 /* Master key 0B encrypted with Master key 0C. */
 
 /* Device Master Key Source Sources. */
 .byte 0x8B, 0x4E, 0x1C, 0x22, 0x42, 0x07, 0xC8, 0x73, 0x56, 0x94, 0x08, 0x8B, 0xCC, 0x47, 0x0F, 0x5D /* 4.0.0  Device Master Key Source Source. */
@@ -128,6 +130,7 @@ _ZN3ams6secmon4boot15VolatileKeyDataE:
 .byte 0x4A, 0xC3, 0x4E, 0x14, 0x8B, 0x96, 0x4A, 0xD5, 0xD4, 0x99, 0x73, 0xC4, 0x45, 0xAB, 0x8B, 0x49 /* 9.0.0  Device Master Key Source Source. */
 .byte 0x14, 0xB8, 0x74, 0x12, 0xCB, 0xBD, 0x0B, 0x8F, 0x20, 0xFB, 0x30, 0xDA, 0x27, 0xE4, 0x58, 0x94 /* 9.1.0  Device Master Key Source Source. */
 .byte 0xAA, 0xFD, 0xBC, 0xBB, 0x25, 0xC3, 0xA4, 0xEF, 0xE3, 0xEE, 0x58, 0x53, 0xB7, 0xF8, 0xDD, 0xD6 /* 12.1.0 Device Master Key Source Source. */
+.byte 0xE4, 0xF3, 0x45, 0x6F, 0x18, 0xA1, 0x89, 0xF8, 0xDA, 0x4C, 0x64, 0x75, 0x68, 0xE6, 0xBD, 0x4F /* 13.0.0 Device Master Key Source Source. */
 
 /* Development Device Master Kek Sources. */
 .byte 0xD6, 0xBD, 0x9F, 0xC6, 0x18, 0x09, 0xE1, 0x96, 0x20, 0x39, 0x60, 0xD2, 0x89, 0x83, 0x31, 0x34 /* 4.0.0  Device Master Kek Source. */
@@ -139,6 +142,7 @@ _ZN3ams6secmon4boot15VolatileKeyDataE:
 .byte 0x61, 0x6A, 0x88, 0x21, 0xA3, 0x52, 0xB0, 0x19, 0x16, 0x25, 0xA4, 0xE3, 0x4C, 0x54, 0x02, 0x0F /* 9.0.0  Device Master Kek Source. */
 .byte 0x9D, 0xB1, 0xAE, 0xCB, 0xF6, 0xF6, 0xE3, 0xFE, 0xAB, 0x6F, 0xCB, 0xAF, 0x38, 0x03, 0xFC, 0x7B /* 9.1.0  Device Master Kek Source. */
 .byte 0xC4, 0xBB, 0xF3, 0x9F, 0xA3, 0xAA, 0x00, 0x99, 0x7C, 0x97, 0xAD, 0x91, 0x8F, 0xE8, 0x45, 0xCB /* 12.1.0 Device Master Kek Source. */
+.byte 0x20, 0x20, 0xAA, 0xFB, 0x89, 0xC2, 0xF0, 0x70, 0xB5, 0xE0, 0xA3, 0x11, 0x8A, 0x29, 0x8D, 0x0F /* 13.0.0 Device Master Kek Source. */
 
 /* Production Device Master Kek Sources. */
 .byte 0x88, 0x62, 0x34, 0x6E, 0xFA, 0xF7, 0xD8, 0x3F, 0xE1, 0x30, 0x39, 0x50, 0xF0, 0xB7, 0x5D, 0x5D /* 4.0.0  Device Master Kek Source. */
@@ -150,3 +154,4 @@ _ZN3ams6secmon4boot15VolatileKeyDataE:
 .byte 0x03, 0xE7, 0xEB, 0x43, 0x1B, 0xCF, 0x5F, 0xB5, 0xED, 0xDC, 0x97, 0xAE, 0x21, 0x8D, 0x19, 0xED /* 9.0.0  Device Master Kek Source. */
 .byte 0xCE, 0xFE, 0x41, 0x0F, 0x46, 0x9A, 0x30, 0xD6, 0xF2, 0xE9, 0x0C, 0x6B, 0xB7, 0x15, 0x91, 0x36 /* 9.1.0  Device Master Kek Source. */
 .byte 0xC2, 0x65, 0x34, 0x6E, 0xC7, 0xC6, 0x5D, 0x97, 0x3E, 0x34, 0x5C, 0x6B, 0xB3, 0x7E, 0xC6, 0xE3 /* 12.1.0 Device Master Kek Source. */
+.byte 0x77, 0x52, 0x92, 0xF0, 0xAA, 0xE3, 0xFB, 0xE0, 0x60, 0x16, 0xB3, 0x78, 0x68, 0x53, 0xF7, 0xA8 /* 13.0.0 Device Master Kek Source. */

exosphere/program/source/boot/secmon_package2.cpp

@@ -94,7 +94,7 @@ namespace ams::secmon::boot {
         }
 
         /* Check that the key generation is one that we can use. */
-        static_assert(pkg1::KeyGeneration_Count == 12);
+        static_assert(pkg1::KeyGeneration_Count == 13);
         if (key_generation >= pkg1::KeyGeneration_Count) {
             return false;
         }

exosphere/program/source/smc/secmon_smc_info.cpp

@@ -33,14 +33,15 @@ namespace ams::secmon::smc {
             using PhysicalMemorySize = util::BitPack32::Field<16, 2>;
 
             /* Kernel view, from libmesosphere. */
-            using DebugFillMemory             = util::BitPack32::Field<0,                                 1, bool>;
+            using DebugFillMemory              = util::BitPack32::Field<0,                                  1, bool>;
-            using EnableUserExceptionHandlers = util::BitPack32::Field<DebugFillMemory::Next,             1, bool>;
+            using EnableUserExceptionHandlers  = util::BitPack32::Field<DebugFillMemory::Next,              1, bool>;
-            using EnableUserPmuAccess         = util::BitPack32::Field<EnableUserExceptionHandlers::Next, 1, bool>;
+            using EnableUserPmuAccess          = util::BitPack32::Field<EnableUserExceptionHandlers::Next,  1, bool>;
-            using IncreaseThreadResourceLimit = util::BitPack32::Field<EnableUserPmuAccess::Next,         1, bool>;
+            using IncreaseThreadResourceLimit  = util::BitPack32::Field<EnableUserPmuAccess::Next,          1, bool>;
-            using Reserved4                   = util::BitPack32::Field<IncreaseThreadResourceLimit::Next, 4, u32>;
+            using DisableDynamicResourceLimits = util::BitPack32::Field<IncreaseThreadResourceLimit::Next,  1, bool>;
-            using UseSecureMonitorPanicCall   = util::BitPack32::Field<Reserved4::Next,                   1, bool>;
+            using Reserved5                    = util::BitPack32::Field<DisableDynamicResourceLimits::Next, 3, u32>;
-            using Reserved9                   = util::BitPack32::Field<UseSecureMonitorPanicCall::Next,   7, u32>;
+            using UseSecureMonitorPanicCall    = util::BitPack32::Field<Reserved5::Next,                    1, bool>;
-            using MemorySize                  = util::BitPack32::Field<Reserved9::Next,                   2, u32>; /* smc::MemorySize = pkg1::MemorySize */
+            using Reserved9                    = util::BitPack32::Field<UseSecureMonitorPanicCall::Next,    7, u32>;
+            using MemorySize                   = util::BitPack32::Field<Reserved9::Next,                    2, u32>; /* smc::MemorySize = pkg1::MemorySize */
         };
 
         constexpr const pkg1::MemorySize DramIdToMemorySize[fuse::DramId_Count] = {
@@ -50,7 +51,7 @@ namespace ams::secmon::smc {
             [fuse::DramId_IowaHynix1y4GB]     = pkg1::MemorySize_4GB,
             [fuse::DramId_IcosaSamsung6GB]    = pkg1::MemorySize_6GB,
             [fuse::DramId_HoagHynix1y4GB]     = pkg1::MemorySize_4GB,
-            [fuse::DramId_CopperMicron4GB]    = pkg1::MemorySize_4GB,
+            [fuse::DramId_AulaHynix1y4GB]     = pkg1::MemorySize_4GB,
             [fuse::DramId_IowaX1X2Samsung4GB] = pkg1::MemorySize_4GB,
             [fuse::DramId_IowaSansung4GB]     = pkg1::MemorySize_4GB,
             [fuse::DramId_IowaSamsung8GB]     = pkg1::MemorySize_8GB,

fusee/program/source/fusee_key_derivation.cpp

@@ -23,17 +23,17 @@ namespace ams::nxboot {
 
         alignas(se::AesBlockSize) constexpr inline const u8 MarikoMasterKekSource[se::AesBlockSize] = {
             /* TODO: Update on next change of keys. */
-            0xE5, 0x41, 0xAC, 0xEC, 0xD1, 0xA7, 0xD1, 0xAB, 0xED, 0x03, 0x77, 0xF1, 0x27, 0xCA, 0xF8, 0xF1
+            0x52, 0x71, 0x9B, 0xDF, 0xA7, 0x8B, 0x61, 0xD8, 0xD5, 0x85, 0x11, 0xE4, 0x8E, 0x4F, 0x74, 0xC6
         };
 
         alignas(se::AesBlockSize) constexpr inline const u8 MarikoMasterKekSourceDev[se::AesBlockSize] = {
             /* TODO: Update on next change of keys. */
-            0x75, 0x2D, 0x2E, 0xF3, 0x2F, 0x3F, 0xFE, 0x65, 0xF4, 0xA9, 0x83, 0xB4, 0xED, 0x42, 0x63, 0xBA
+            0x4D, 0x5A, 0xB2, 0xC9, 0xE9, 0xE4, 0x4E, 0xA4, 0xD3, 0xBF, 0x94, 0x12, 0x36, 0x30, 0xD0, 0x7F
         };
 
         alignas(se::AesBlockSize) constexpr inline const u8 EristaMasterKekSource[se::AesBlockSize] = {
             /* TODO: Update on next change of keys. */
-            0x84, 0x67, 0xB6, 0x7F, 0x13, 0x11, 0xAE, 0xE6, 0x58, 0x9B, 0x19, 0xAF, 0x13, 0x6C, 0x80, 0x7A
+            0x68, 0x3B, 0xCA, 0x54, 0xB8, 0x6F, 0x92, 0x48, 0xC3, 0x05, 0x76, 0x87, 0x88, 0x70, 0x79, 0x23
         };
 
         alignas(se::AesBlockSize) constexpr inline const u8 KeyblobKeySource[se::AesBlockSize] = {
@@ -66,6 +66,7 @@ namespace ams::nxboot {
             { 0x4A, 0xC3, 0x4E, 0x14, 0x8B, 0x96, 0x4A, 0xD5, 0xD4, 0x99, 0x73, 0xC4, 0x45, 0xAB, 0x8B, 0x49 }, /* 9.0.0  Device Master Key Source Source. */
             { 0x14, 0xB8, 0x74, 0x12, 0xCB, 0xBD, 0x0B, 0x8F, 0x20, 0xFB, 0x30, 0xDA, 0x27, 0xE4, 0x58, 0x94 }, /* 9.1.0  Device Master Key Source Source. */
             { 0xAA, 0xFD, 0xBC, 0xBB, 0x25, 0xC3, 0xA4, 0xEF, 0xE3, 0xEE, 0x58, 0x53, 0xB7, 0xF8, 0xDD, 0xD6 }, /* 12.1.0 Device Master Key Source Source. */
+            { 0xE4, 0xF3, 0x45, 0x6F, 0x18, 0xA1, 0x89, 0xF8, 0xDA, 0x4C, 0x64, 0x75, 0x68, 0xE6, 0xBD, 0x4F }, /* 13.0.0 Device Master Key Source Source. */
         };
 
         alignas(se::AesBlockSize) constexpr inline const u8 DeviceMasterKekSources[pkg1::OldDeviceMasterKeyCount][se::AesBlockSize] = {
@@ -78,6 +79,7 @@ namespace ams::nxboot {
             { 0x03, 0xE7, 0xEB, 0x43, 0x1B, 0xCF, 0x5F, 0xB5, 0xED, 0xDC, 0x97, 0xAE, 0x21, 0x8D, 0x19, 0xED }, /* 9.0.0  Device Master Kek Source. */
             { 0xCE, 0xFE, 0x41, 0x0F, 0x46, 0x9A, 0x30, 0xD6, 0xF2, 0xE9, 0x0C, 0x6B, 0xB7, 0x15, 0x91, 0x36 }, /* 9.1.0  Device Master Kek Source. */
             { 0xC2, 0x65, 0x34, 0x6E, 0xC7, 0xC6, 0x5D, 0x97, 0x3E, 0x34, 0x5C, 0x6B, 0xB3, 0x7E, 0xC6, 0xE3 }, /* 12.1.0 Device Master Kek Source. */
+            { 0x77, 0x52, 0x92, 0xF0, 0xAA, 0xE3, 0xFB, 0xE0, 0x60, 0x16, 0xB3, 0x78, 0x68, 0x53, 0xF7, 0xA8 }, /* 13.0.0 Device Master Kek Source. */
         };
 
         alignas(se::AesBlockSize) constexpr inline const u8 DeviceMasterKekSourcesDev[pkg1::OldDeviceMasterKeyCount][se::AesBlockSize] = {
@@ -90,6 +92,7 @@ namespace ams::nxboot {
             { 0x61, 0x6A, 0x88, 0x21, 0xA3, 0x52, 0xB0, 0x19, 0x16, 0x25, 0xA4, 0xE3, 0x4C, 0x54, 0x02, 0x0F }, /* 9.0.0  Device Master Kek Source. */
             { 0x9D, 0xB1, 0xAE, 0xCB, 0xF6, 0xF6, 0xE3, 0xFE, 0xAB, 0x6F, 0xCB, 0xAF, 0x38, 0x03, 0xFC, 0x7B }, /* 9.1.0  Device Master Kek Source. */
             { 0xC4, 0xBB, 0xF3, 0x9F, 0xA3, 0xAA, 0x00, 0x99, 0x7C, 0x97, 0xAD, 0x91, 0x8F, 0xE8, 0x45, 0xCB }, /* 12.1.0 Device Master Kek Source. */
+            { 0x20, 0x20, 0xAA, 0xFB, 0x89, 0xC2, 0xF0, 0x70, 0xB5, 0xE0, 0xA3, 0x11, 0x8A, 0x29, 0x8D, 0x0F }, /* 13.0.0 Device Master Kek Source. */
         };
 
         alignas(se::AesBlockSize) constexpr inline const u8 MasterKeySources[pkg1::KeyGeneration_Count][se::AesBlockSize] = {
@@ -105,6 +108,7 @@ namespace ams::nxboot {
             { 0x4D, 0xD9, 0x98, 0x42, 0x45, 0x0D, 0xB1, 0x3C, 0x52, 0x0C, 0x9A, 0x44, 0xBB, 0xAD, 0xAF, 0x80 }, /* Master key 08 encrypted with Master key 09. */
             { 0xB8, 0x96, 0x9E, 0x4A, 0x00, 0x0D, 0xD6, 0x28, 0xB3, 0xD1, 0xDB, 0x68, 0x5F, 0xFB, 0xE1, 0x2A }, /* Master key 09 encrypted with Master key 0A. */
             { 0xC1, 0x8D, 0x16, 0xBB, 0x2A, 0xE4, 0x1D, 0xD4, 0xC2, 0xC1, 0xB6, 0x40, 0x94, 0x35, 0x63, 0x98 }, /* Master key 0A encrypted with Master key 0B. */
+            { 0xA3, 0x24, 0x65, 0x75, 0xEA, 0xCC, 0x6E, 0x8D, 0xFB, 0x5A, 0x16, 0x50, 0x74, 0xD2, 0x15, 0x06 }, /* Master key 0B encrypted with Master key 0C. */
         };
 
         alignas(se::AesBlockSize) constexpr inline const u8 MasterKeySourcesDev[pkg1::KeyGeneration_Count][se::AesBlockSize] = {
@@ -120,6 +124,7 @@ namespace ams::nxboot {
             { 0x43, 0x3D, 0xC5, 0x3B, 0xEF, 0x91, 0x02, 0x21, 0x61, 0x54, 0x63, 0x8A, 0x35, 0xE7, 0xCA, 0xEE }, /* Master key 08 encrypted with Master key 09. */
             { 0x6C, 0x2E, 0xCD, 0xB3, 0x34, 0x61, 0x77, 0xF5, 0xF9, 0xB1, 0xDD, 0x61, 0x98, 0x19, 0x3E, 0xD4 }, /* Master key 09 encrypted with Master key 0A. */
             { 0x21, 0x88, 0x6B, 0x10, 0x9E, 0x83, 0xD6, 0x52, 0xAB, 0x08, 0xDB, 0x6D, 0x39, 0xFF, 0x1C, 0x9C }, /* Master key 0A encrypted with Master key 0B. */
+            { 0x8A, 0xCE, 0xC4, 0x7F, 0xBE, 0x08, 0x61, 0x88, 0xD3, 0x73, 0x64, 0x51, 0xE2, 0xB6, 0x53, 0x15 }, /* Master key 0B encrypted with Master key 0C. */
         };
 
         alignas(se::AesBlockSize) constinit u8 MasterKeys[pkg1::OldMasterKeyCount][se::AesBlockSize] = {};

fusee/program/source/fusee_package2.cpp

@@ -80,7 +80,7 @@ namespace ams::nxboot {
             }
 
             /* Check that the key generation is one that we can use. */
-            static_assert(pkg1::KeyGeneration_Count == 12);
+            static_assert(pkg1::KeyGeneration_Count == 13);
             if (key_generation >= pkg1::KeyGeneration_Count) {
                 return false;
             }

fusee/program/source/fusee_setup_horizon.cpp

@@ -270,6 +270,8 @@ namespace ams::nxboot {
                         target_firmware = ams::TargetFirmware_12_0_2;
                     } else if (std::memcmp(package1 + 0x10, "20210607", 8) == 0) {
                         target_firmware = ams::TargetFirmware_12_1_0;
+                    } else if (std::memcmp(package1 + 0x10, "20210805", 8) == 0) {
+                        target_firmware = ams::TargetFirmware_13_0_0;
                     } else {
                         ShowFatalError("Unable to identify package1!\n");
                     }
@@ -281,7 +283,9 @@ namespace ams::nxboot {
 
             #define CHECK_NCA(NCA_ID, VERSION) do { if (IsNcaExist(NCA_ID)) { return ams::TargetFirmware_##VERSION; } } while(0)
 
-            if (target_firmware >= ams::TargetFirmware_12_1_0) {
+            if (target_firmware >= ams::TargetFirmware_13_0_0) {
+                CHECK_NCA("bf2337ee88bd9f963a33b3ecbbc3732a", 13_0_0);
+            } else if (target_firmware >= ams::TargetFirmware_12_1_0) {
                 CHECK_NCA("9d9d83d68d9517f245f3e8cd7f93c416", 12_1_0);
             } else if (target_firmware >= ams::TargetFirmware_12_0_2) {
                 CHECK_NCA("a1863a5c0e1cedd442f5e60b0422dc15", 12_0_3);

fusee/program/source/fusee_stratosphere.cpp

@@ -147,6 +147,9 @@ namespace ams::nxboot {
             FsVersion_12_0_3,
             FsVersion_12_0_3_Exfat,
 
+            FsVersion_13_0_0,
+            FsVersion_13_0_0_Exfat,
+
             FsVersion_Count,
         };
 
@@ -209,6 +212,9 @@ namespace ams::nxboot {
 
             { 0xC8, 0x67, 0x62, 0xBE, 0x19, 0xA5, 0x1F, 0xA0 }, /* FsVersion_12_0_3 */
             { 0xE1, 0xE8, 0xD3, 0xD6, 0xA2, 0xFE, 0x0B, 0x10 }, /* FsVersion_12_0_3_Exfat */
+
+            { 0x7D, 0x20, 0x05, 0x47, 0x17, 0x8A, 0x83, 0x6A }, /* FsVersion_13_0_0 */
+            { 0x51, 0xEB, 0xFA, 0x9C, 0xCF, 0x66, 0xC0, 0x9E }, /* FsVersion_13_0_0_Exfat */
         };
 
         const InitialProcessBinaryHeader *FindInitialProcessBinary(const pkg2::Package2Header *header, const u8 *data, ams::TargetFirmware target_firmware) {
@@ -592,6 +598,10 @@ namespace ams::nxboot {
                 case FsVersion_12_0_3_Exfat:
                     AddPatch(fs_meta, 0x155579, NogcPatch0, sizeof(NogcPatch0));
                     AddPatch(fs_meta, 0x13EC34, NogcPatch1, sizeof(NogcPatch1));
+                case FsVersion_13_0_0:
+                case FsVersion_13_0_0_Exfat:
+                    AddPatch(fs_meta, 0x159119, NogcPatch0, sizeof(NogcPatch0));
+                    AddPatch(fs_meta, 0x1426D0, NogcPatch1, sizeof(NogcPatch1));
                     break;
                 default:
                     break;

fusee/program/source/mtc/fusee_mtc.cpp

@@ -37,7 +37,7 @@ namespace ams::nxboot {
             /* DramId_MarikoIowaHynix1y4gb     */ 0x10,
             /* DramId_EristaIcosaSamsung6gb    */ 0x01,
             /* DramId_MarikoHoagHynix1y4gb     */ 0x10,
-            /* DramId_EristaCopperMicron4gb    */ MemoryTrainingTableIndex_Invalid,
+            /* DramId_MarikoAulaHynix1y4gb     */ 0x10,
             /* DramId_MarikoIowax1x2Samsung4gb */ 0x00,
             /* DramId_MarikoIowaSamsung4gb     */ 0x05,
             /* DramId_MarikoIowaSamsung8gb     */ 0x06,

fusee/program/source/sdram/fusee_sdram.cpp

@@ -83,6 +83,8 @@ namespace ams::nxboot {
                         break;
                 switch (dram_id) {
                     HANDLE_DRAM_CASE( 3, 12)
+                    HANDLE_DRAM_CASE( 5, 12)
+                    HANDLE_DRAM_CASE( 6, 12)
                     HANDLE_DRAM_CASE( 7,  0)
                     HANDLE_DRAM_CASE( 8,  1)
                     HANDLE_DRAM_CASE( 9,  2)

libraries/libexosphere/include/exosphere/fuse.hpp

@@ -54,7 +54,7 @@ namespace ams::fuse {
         DramId_IowaHynix1y4GB     =  3,
         DramId_IcosaSamsung6GB    =  4,
         DramId_HoagHynix1y4GB     =  5,
-        DramId_CopperMicron4GB    =  6,
+        DramId_AulaHynix1y4GB     =  6,
         DramId_IowaX1X2Samsung4GB =  7,
         DramId_IowaSansung4GB     =  8,
         DramId_IowaSamsung8GB     =  9,

libraries/libexosphere/include/exosphere/pkg1/pkg1_key_generation.hpp

@@ -32,6 +32,7 @@ namespace ams::pkg1 {
         KeyGeneration_9_0_0  = 0x09,
         KeyGeneration_9_1_0  = 0x0A,
         KeyGeneration_12_1_0 = 0x0B,
+        KeyGeneration_13_0_0 = 0x0C,
 
         KeyGeneration_Count,
 

libraries/libmesosphere/Makefile

@@ -142,7 +142,7 @@ $(OFILES_SRC)	: $(HFILES_BIN)
 
 kern_libc_generic.o: CFLAGS += -fno-builtin
 
-kern_k_auto_object.o: CXXFLAGS += -fno-lto
+kern_k_auto_object.o kern_k_debug_base_process_holder.o: CXXFLAGS += -fno-lto
 
 #---------------------------------------------------------------------------------
 %_bin.h %.bin.o	:	%.bin

libraries/libmesosphere/include/mesosphere.hpp

@@ -78,8 +78,7 @@
 #include <mesosphere/kern_select_debug.hpp>
 #include <mesosphere/kern_k_process.hpp>
 #include <mesosphere/kern_k_resource_limit.hpp>
-#include <mesosphere/kern_k_alpha.hpp>
+#include <mesosphere/kern_k_io_pool.hpp>
-#include <mesosphere/kern_k_beta.hpp>
 
 /* More Miscellaneous objects. */
 #include <mesosphere/kern_k_object_name.hpp>

libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp

@@ -293,7 +293,7 @@ namespace ams::kern::arch::arm64::init {
                 }
 
                 /* Swap the mappings. */
-                const u64 attr_preserve_mask  = (negative_block_size_for_mask | 0xFFFF000000000000ul) ^ ((1ul << 48) - 1);
+                const u64 attr_preserve_mask  = (block_size - 1) | 0xFFFF000000000000ul;
                 const size_t shift_for_contig = contig ? 4 : 0;
                 size_t advanced_size = 0;
                 const u64 src_attr_val = src_saved.GetRawAttributesUnsafeForSwap() & attr_preserve_mask;
@@ -726,8 +726,8 @@ namespace ams::kern::arch::arm64::init {
                 m_state.end_address  = address;
             }
 
-            ALWAYS_INLINE void InitializeFromState(uintptr_t state_val) {
+            ALWAYS_INLINE void InitializeFromState(const State *state) {
-                m_state = *reinterpret_cast<State *>(state_val);
+                m_state = *state;
             }
 
             ALWAYS_INLINE void GetFinalState(State *out) {

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_assembly_offsets.h

@@ -156,7 +156,9 @@
 
 /* ams::kern::KScheduler (::SchedulingState), https://github.com/Atmosphere-NX/Atmosphere/blob/master/libraries/libmesosphere/include/mesosphere/kern_k_scheduler.hpp */
 /* NOTE: Due to constraints on ldarb relative offsets, KSCHEDULER_NEEDS_SCHEDULING cannot trivially be changed, and will require assembly edits. */
-#define KSCHEDULER_NEEDS_SCHEDULING               0x00
+#define KSCHEDULER_NEEDS_SCHEDULING        0x00
-#define KSCHEDULER_INTERRUPT_TASK_THREAD_RUNNABLE 0x01
+#define KSCHEDULER_INTERRUPT_TASK_RUNNABLE 0x01
-#define KSCHEDULER_HIGHEST_PRIORITY_THREAD        0x10
+#define KSCHEDULER_HIGHEST_PRIORITY_THREAD 0x10
-#define KSCHEDULER_IDLE_THREAD_STACK              0x18
+#define KSCHEDULER_IDLE_THREAD_STACK       0x18
+#define KSCHEDULER_PREVIOUS_THREAD         0x20
+#define KSCHEDULER_INTERRUPT_TASK_MANAGER  0x28

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_debug.hpp

@@ -31,7 +31,6 @@ namespace ams::kern::arch::arm64 {
         MESOSPHERE_AUTOOBJECT_TRAITS(KDebug, KSynchronizationObject);
         public:
             explicit KDebug() { /* ... */ }
-            virtual ~KDebug() { /* ... */ }
 
             static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
         public:

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_process_page_table.hpp

@@ -96,6 +96,14 @@ namespace ams::kern::arch::arm64 {
                 return m_page_table.MapIo(phys_addr, size, perm);
             }
 
+            Result MapIoRegion(KProcessAddress dst_address, KPhysicalAddress phys_addr, size_t size, ams::svc::MemoryMapping mapping, ams::svc::MemoryPermission perm) {
+                return m_page_table.MapIoRegion(dst_address, phys_addr, size, mapping, perm);
+            }
+
+            Result UnmapIoRegion(KProcessAddress dst_address, KPhysicalAddress phys_addr, size_t size) {
+                return m_page_table.UnmapIoRegion(dst_address, phys_addr, size);
+            }
+
             Result MapStatic(KPhysicalAddress phys_addr, size_t size, KMemoryPermission perm) {
                 return m_page_table.MapStatic(phys_addr, size, perm);
             }
@@ -164,8 +172,8 @@ namespace ams::kern::arch::arm64 {
                 return m_page_table.UnlockForDeviceAddressSpace(address, size);
             }
 
-            Result UnlockForDeviceAddressSpacePartialMap(KProcessAddress address, size_t size, size_t mapped_size) {
+            Result UnlockForDeviceAddressSpacePartialMap(KProcessAddress address, size_t size) {
-                return m_page_table.UnlockForDeviceAddressSpacePartialMap(address, size, mapped_size);
+                return m_page_table.UnlockForDeviceAddressSpacePartialMap(address, size);
             }
 
             Result OpenMemoryRangeForMapDeviceAddressSpace(KPageTableBase::MemoryRange *out, KProcessAddress address, size_t size, KMemoryPermission perm, bool is_aligned) {

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_slab_heap_impl.hpp

@@ -16,6 +16,7 @@
 #pragma once
 #include <vapours.hpp>
 #include <mesosphere/kern_select_cpu.hpp>
+#include <mesosphere/kern_select_interrupt_manager.hpp>
 
 namespace ams::kern::arch::arm64 {
 
@@ -24,6 +25,32 @@ namespace ams::kern::arch::arm64 {
         { t.next } -> std::convertible_to<T *>;
     };
 
+    ALWAYS_INLINE bool IsSlabAtomicValid() {
+        /* Without careful consideration, slab heaps atomics are vulnerable to */
+        /* the ABA problem, when doing compare and swap of node pointers. */
+        /* We resolve this by using the ARM exclusive monitor; we bundle the */
+        /* load and store of the relevant values into a single exclusive monitor */
+        /* hold, preventing the ABA problem. */
+        /* However, our assembly must do both a load and a store under a single */
+        /* hold, at different memory addresses. Considering the case where the */
+        /* addresses are distinct but resolve to the same cache set (by chance), */
+        /* we can note that under a 1-way associative (direct-mapped) cache */
+        /* we would have as a guarantee that the second access would evict the */
+        /* cache line from the first access, invalidating our exclusive monitor */
+        /* hold. Thus, we require that the cache is not 1-way associative, for */
+        /* our implementation to be correct. */
+        {
+            /* Disable interrupts. */
+            KScopedInterruptDisable di;
+
+            /* Select L1 cache. */
+            cpu::SetCsselrEl1(0);
+
+            /* Check that the L1 cache is not direct-mapped. */
+            return cpu::CacheSizeIdRegisterAccessor().GetAssociativity() != 0;
+        }
+    }
+
     template<typename T> requires SlabHeapNode<T>
     ALWAYS_INLINE T *AllocateFromSlabAtomic(T **head) {
         u32 tmp;
@@ -36,10 +63,7 @@ namespace ams::kern::arch::arm64 {
             "    ldr    %[next], [%[node]]\n"
             "    stlxr  %w[tmp], %[next], [%[head]]\n"
             "    cbnz   %w[tmp], 1b\n"
-            "    b      3f\n"
             "2:\n"
-            "    clrex\n"
-            "3:\n"
             : [tmp]"=&r"(tmp), [node]"=&r"(node), [next]"=&r"(next), [head]"+&r"(head)
             :
             : "cc", "memory"
@@ -59,7 +83,6 @@ namespace ams::kern::arch::arm64 {
             "    str    %[next], [%[node]]\n"
             "    stlxr  %w[tmp], %[node], [%[head]]\n"
             "    cbnz   %w[tmp], 1b\n"
-            "2:\n"
             : [tmp]"=&r"(tmp), [node]"+&r"(node), [next]"=&r"(next), [head]"+&r"(head)
             :
             : "cc", "memory"

libraries/libmesosphere/include/mesosphere/board/nintendo/nx/kern_k_device_page_table.hpp

@@ -69,16 +69,16 @@ namespace ams::kern::board::nintendo::nx {
             Result Attach(ams::svc::DeviceName device_name, u64 space_address, u64 space_size);
             Result Detach(ams::svc::DeviceName device_name);
 
-            Result Map(size_t *out_mapped_size, KProcessPageTable *page_table, KProcessAddress process_address, size_t size, KDeviceVirtualAddress device_address, ams::svc::MemoryPermission device_perm, bool is_aligned, bool refresh_mappings);
+            Result Map(KProcessPageTable *page_table, KProcessAddress process_address, size_t size, KDeviceVirtualAddress device_address, ams::svc::MemoryPermission device_perm, bool is_aligned);
             Result Unmap(KProcessPageTable *page_table, KProcessAddress process_address, size_t size, KDeviceVirtualAddress device_address);
 
             void Unmap(KDeviceVirtualAddress device_address, size_t size) {
                 return this->UnmapImpl(device_address, size, false);
             }
         private:
-            Result MapDevicePage(size_t *out_mapped_size, s32 &num_pt, s32 max_pt, KPhysicalAddress phys_addr, u64 size, KDeviceVirtualAddress address, ams::svc::MemoryPermission device_perm);
+            Result MapDevicePage(KPhysicalAddress phys_addr, u64 size, KDeviceVirtualAddress address, ams::svc::MemoryPermission device_perm);
 
-            Result MapImpl(size_t *out_mapped_size, s32 &num_pt, s32 max_pt, KProcessPageTable *page_table, KProcessAddress process_address, size_t size, KDeviceVirtualAddress device_address, ams::svc::MemoryPermission device_perm, bool is_aligned);
+            Result MapImpl(KProcessPageTable *page_table, KProcessAddress process_address, size_t size, KDeviceVirtualAddress device_address, ams::svc::MemoryPermission device_perm, bool is_aligned);
             void UnmapImpl(KDeviceVirtualAddress address, u64 size, bool force);
 
             bool IsFree(KDeviceVirtualAddress address, u64 size) const;

libraries/libmesosphere/include/mesosphere/board/nintendo/nx/kern_k_system_control.hpp

@@ -16,6 +16,12 @@
 #pragma once
 #include <mesosphere/kern_common.hpp>
 
+namespace ams::kern {
+
+    struct InitialProcessBinaryLayout;
+
+}
+
 namespace ams::kern::board::nintendo::nx {
 
     class KSystemControl {
@@ -25,7 +31,7 @@ namespace ams::kern::board::nintendo::nx {
                     /* Initialization. */
                     static size_t GetIntendedMemorySize();
                     static KPhysicalAddress GetKernelPhysicalBaseAddress(uintptr_t base_address);
-                    static KPhysicalAddress GetInitialProcessBinaryPhysicalAddress();
+                    static void GetInitialProcessBinaryLayout(InitialProcessBinaryLayout *out);
                     static bool ShouldIncreaseThreadResourceLimit();
                     static void CpuOn(u64 core_id, uintptr_t entrypoint, uintptr_t arg);
                     static size_t GetApplicationPoolSize();

libraries/libmesosphere/include/mesosphere/init/kern_init_slab_setup.hpp

@@ -34,8 +34,8 @@ namespace ams::kern::init {
         size_t num_KObjectName;
         size_t num_KResourceLimit;
         size_t num_KDebug;
-        size_t num_KAlpha;
+        size_t num_KIoPool;
-        size_t num_KBeta;
+        size_t num_KIoRegion;
     };
 
     NOINLINE void InitializeSlabResourceCounts();

libraries/libmesosphere/include/mesosphere/kern_debug_log.hpp

@@ -28,6 +28,8 @@ namespace ams::kern {
             static NOINLINE void Printf(const char *format, ...) __attribute__((format(printf, 1, 2)));
             static NOINLINE void VPrintf(const char *format, ::std::va_list vl);
 
+            static NOINLINE void LogException(const char *str);
+
             static NOINLINE Result PrintUserString(ams::kern::svc::KUserPointer<const char *> user_str, size_t len);
 
             /* Functionality for preserving across sleep. */
@@ -49,6 +51,8 @@ namespace ams::kern {
 
 #endif
 
+#define MESOSPHERE_EXCEPTION_LOG(str) ::ams::kern::KDebugLog::LogException(str)
+
 #define MESOSPHERE_RELEASE_LOG(fmt, ...) ::ams::kern::KDebugLog::Printf((fmt), ## __VA_ARGS__)
 #define MESOSPHERE_RELEASE_VLOG(fmt, vl) ::ams::kern::KDebugLog::VPrintf((fmt), (vl))
 

libraries/libmesosphere/include/mesosphere/kern_initial_process.hpp

@@ -29,14 +29,19 @@ namespace ams::kern {
         u32 reserved;
     };
 
-    NOINLINE size_t CopyInitialProcessBinaryToKernelMemory();
+    struct InitialProcessBinaryLayout {
-    NOINLINE void CreateAndRunInitialProcesses();
+        uintptr_t address;
+        uintptr_t _08;
+    };
+
+    KPhysicalAddress GetInitialProcessBinaryPhysicalAddress();
+    void SetInitialProcessBinaryPhysicalAddress(KPhysicalAddress phys_addr);
 
     u64 GetInitialProcessIdMin();
     u64 GetInitialProcessIdMax();
-    KVirtualAddress GetInitialProcessBinaryAddress();
     size_t GetInitialProcessesSecureMemorySize();
 
-    void LoadInitialProcessBinaryHeaderDeprecated(KPhysicalAddress pool_end);
+    NOINLINE size_t CopyInitialProcessBinaryToKernelMemory();
+    NOINLINE void CreateAndRunInitialProcesses();
 
 }

libraries/libmesosphere/include/mesosphere/kern_k_alpha.hpp

@@ -1,41 +0,0 @@
-/*
- * Copyright (c) 2018-2020 Atmosphère-NX
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-#pragma once
-#include <mesosphere/kern_common.hpp>
-#include <mesosphere/kern_k_auto_object.hpp>
-#include <mesosphere/kern_slab_helpers.hpp>
-
-namespace ams::kern {
-
-    class KAlpha final : public KAutoObjectWithSlabHeapAndContainer<KAlpha, KAutoObjectWithList> {
-        MESOSPHERE_AUTOOBJECT_TRAITS(KAlpha, KAutoObject);
-        private:
-            /* NOTE: Official KAlpha has size 0x50, corresponding to 0x20 bytes of fields. */
-            /* TODO: Add these fields, if KAlpha is ever instantiable in the NX kernel. */
-        public:
-            explicit KAlpha() {
-                /* ... */
-            }
-
-            virtual ~KAlpha() { /* ... */ }
-
-            /* virtual void Finalize() override; */
-
-            virtual bool IsInitialized() const override { return false /* TODO */; }
-            static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
-    };
-
-}

libraries/libmesosphere/include/mesosphere/kern_k_auto_object.hpp

@@ -75,7 +75,6 @@ namespace ams::kern {
             static KAutoObject *Create(KAutoObject *ptr);
         public:
             constexpr ALWAYS_INLINE explicit KAutoObject() : m_next_closed_object(nullptr), m_ref_count(0) { MESOSPHERE_ASSERT_THIS(); }
-            virtual ~KAutoObject() { MESOSPHERE_ASSERT_THIS(); }
 
             /* Destroy is responsible for destroying the auto object's resources when ref_count hits zero. */
             virtual void Destroy() { MESOSPHERE_ASSERT_THIS(); }
@@ -139,8 +138,10 @@ namespace ams::kern {
         private:
             friend class KAutoObjectWithListContainer;
         private:
-            util::IntrusiveRedBlackTreeNode list_node;
+            util::IntrusiveRedBlackTreeNode m_list_node;
         public:
+            constexpr ALWAYS_INLINE KAutoObjectWithList() : m_list_node() { /* ... */ }
+
             static ALWAYS_INLINE int Compare(const KAutoObjectWithList &lhs, const KAutoObjectWithList &rhs) {
                 const u64 lid = lhs.GetId();
                 const u64 rid = rhs.GetId();

libraries/libmesosphere/include/mesosphere/kern_k_auto_object_container.hpp

@@ -24,7 +24,7 @@ namespace ams::kern {
         NON_COPYABLE(KAutoObjectWithListContainer);
         NON_MOVEABLE(KAutoObjectWithListContainer);
         public:
-            using ListType = util::IntrusiveRedBlackTreeMemberTraits<&KAutoObjectWithList::list_node>::TreeType<KAutoObjectWithList>;
+            using ListType = util::IntrusiveRedBlackTreeMemberTraits<&KAutoObjectWithList::m_list_node>::TreeType<KAutoObjectWithList>;
         public:
             class ListAccessor : public KScopedLightLock {
                 private:

libraries/libmesosphere/include/mesosphere/kern_k_class_token.hpp

@@ -104,17 +104,14 @@ namespace ams::kern {
                 KSession,
                 KSharedMemory,
                 KEvent,
-                KWritableEvent,
                 KLightClientSession,
                 KLightServerSession,
                 KTransferMemory,
                 KDeviceAddressSpace,
                 KSessionRequest,
                 KCodeMemory,
-
+                KIoPool,
-                /* NOTE: True order for these has not been determined yet. */
+                KIoRegion,
-                KAlpha,
-                KBeta,
 
                 FinalClassesEnd = FinalClassesStart + NumFinalClasses,
             };

libraries/libmesosphere/include/mesosphere/kern_k_client_port.hpp

@@ -34,7 +34,6 @@ namespace ams::kern {
             KPort *m_parent;
         public:
             constexpr KClientPort() : m_num_sessions(), m_peak_sessions(), m_max_sessions(), m_parent() { /* ... */ }
-            virtual ~KClientPort() { /* ... */ }
 
             void Initialize(KPort *parent, s32 max_sessions);
             void OnSessionFinalized();

libraries/libmesosphere/include/mesosphere/kern_k_client_session.hpp

@@ -20,14 +20,14 @@
 namespace ams::kern {
 
     class KSession;
+    class KEvent;
 
-    class KClientSession final : public KAutoObjectWithSlabHeapAndContainer<KClientSession, KAutoObjectWithList> {
+    class KClientSession final : public KAutoObject {
         MESOSPHERE_AUTOOBJECT_TRAITS(KClientSession, KAutoObject);
         private:
             KSession *m_parent;
         public:
             constexpr KClientSession() : m_parent() { /* ... */ }
-            virtual ~KClientSession() { /* ... */ }
 
             void Initialize(KSession *parent) {
                 /* Set member variables. */
@@ -40,7 +40,7 @@ namespace ams::kern {
             constexpr KSession *GetParent() const { return m_parent; }
 
             Result SendSyncRequest(uintptr_t address, size_t size);
-            Result SendAsyncRequest(KWritableEvent *event, uintptr_t address, size_t size);
+            Result SendAsyncRequest(KEvent *event, uintptr_t address, size_t size);
 
             void OnServerClosed();
     };

libraries/libmesosphere/include/mesosphere/kern_k_code_memory.hpp

@@ -35,8 +35,6 @@ namespace ams::kern {
                 /* ... */
             }
 
-            virtual ~KCodeMemory() { /* ... */ }
-
             Result Initialize(KProcessAddress address, size_t size);
             virtual void Finalize() override;
 

libraries/libmesosphere/include/mesosphere/kern_k_condition_variable.hpp

@@ -29,8 +29,8 @@ namespace ams::kern {
             constexpr KConditionVariable() : m_tree() { /* ... */ }
 
             /* Arbitration. */
-            Result SignalToAddress(KProcessAddress addr);
+            static Result SignalToAddress(KProcessAddress addr);
-            Result WaitForAddress(ams::svc::Handle handle, KProcessAddress addr, u32 value);
+            static Result WaitForAddress(ams::svc::Handle handle, KProcessAddress addr, u32 value);
 
             /* Condition variable. */
             void Signal(uintptr_t cv_key, s32 count);

libraries/libmesosphere/include/mesosphere/kern_k_debug_base.hpp

@@ -25,15 +25,31 @@ namespace ams::kern {
     class KDebugBase : public KSynchronizationObject {
         protected:
             using DebugEventList = util::IntrusiveListBaseTraits<KEventInfo>::ListType;
+        private:
+            class ProcessHolder {
+                private:
+                    friend class KDebugBase;
+                private:
+                    KProcess *m_process;
+                    std::atomic<u32> m_ref_count;
+                private:
+                    explicit ProcessHolder() : m_process(nullptr) { /* ... */ }
+
+                    void Attach(KProcess *process);
+                    void Detach();
+
+                    bool Open();
+                    void Close();
+            };
         private:
             DebugEventList m_event_info_list;
             u32 m_continue_flags;
-            KProcess *m_process;
+            ProcessHolder m_process_holder;
             KLightLock m_lock;
             KProcess::State m_old_process_state;
+            bool m_is_attached;
         public:
-            explicit KDebugBase() { /* ... */ }
+            explicit KDebugBase() : m_event_info_list(), m_process_holder(), m_lock() { /* ... */ }
-            virtual ~KDebugBase() { /* ... */ }
         protected:
             bool Is64Bit() const;
         public:
@@ -60,7 +76,21 @@ namespace ams::kern {
             Result GetDebugEventInfo(ams::svc::lp64::DebugEventInfo *out);
             Result GetDebugEventInfo(ams::svc::ilp32::DebugEventInfo *out);
 
-            KScopedAutoObject<KProcess> GetProcess();
+            ALWAYS_INLINE bool IsAttached() const {
+                return m_is_attached;
+            }
+
+            ALWAYS_INLINE bool OpenProcess() {
+                return m_process_holder.Open();
+            }
+
+            ALWAYS_INLINE void CloseProcess() {
+                return m_process_holder.Close();
+            }
+
+            ALWAYS_INLINE KProcess *GetProcessUnsafe() const {
+                return m_process_holder.m_process;
+            }
         private:
             void PushDebugEvent(ams::svc::DebugEvent event, uintptr_t param0 = 0, uintptr_t param1 = 0, uintptr_t param2 = 0, uintptr_t param3 = 0, uintptr_t param4 = 0);
             void EnqueueDebugEventInfo(KEventInfo *info);

libraries/libmesosphere/include/mesosphere/kern_k_device_address_space.hpp

@@ -31,7 +31,6 @@ namespace ams::kern {
             bool m_is_initialized;
         public:
             constexpr KDeviceAddressSpace() : m_lock(), m_table(), m_space_address(), m_space_size(), m_is_initialized() { /* ... */ }
-            virtual ~KDeviceAddressSpace() { /* ... */ }
 
             Result Initialize(u64 address, u64 size);
             virtual void Finalize() override;
@@ -42,18 +41,17 @@ namespace ams::kern {
             Result Attach(ams::svc::DeviceName device_name);
             Result Detach(ams::svc::DeviceName device_name);
 
-            Result Map(size_t *out_mapped_size, KProcessPageTable *page_table, KProcessAddress process_address, size_t size, u64 device_address, ams::svc::MemoryPermission device_perm, bool refresh_mappings) {
+            Result MapByForce(KProcessPageTable *page_table, KProcessAddress process_address, size_t size, u64 device_address, ams::svc::MemoryPermission device_perm) {
-                return this->Map(out_mapped_size, page_table, process_address, size, device_address, device_perm, false, refresh_mappings);
+                return this->Map(page_table, process_address, size, device_address, device_perm, false);
             }
 
             Result MapAligned(KProcessPageTable *page_table, KProcessAddress process_address, size_t size, u64 device_address, ams::svc::MemoryPermission device_perm) {
-                size_t dummy;
+                return this->Map(page_table, process_address, size, device_address, device_perm, true);
-                return this->Map(std::addressof(dummy), page_table, process_address, size, device_address, device_perm, true, false);
             }
 
             Result Unmap(KProcessPageTable *page_table, KProcessAddress process_address, size_t size, u64 device_address);
         private:
-            Result Map(size_t *out_mapped_size, KProcessPageTable *page_table, KProcessAddress process_address, size_t size, u64 device_address, ams::svc::MemoryPermission device_perm, bool is_aligned, bool refresh_mappings);
+            Result Map(KProcessPageTable *page_table, KProcessAddress process_address, size_t size, u64 device_address, ams::svc::MemoryPermission device_perm, bool is_aligned);
         public:
             static void Initialize();
     };

libraries/libmesosphere/include/mesosphere/kern_k_dynamic_resource_manager.hpp

@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_k_dynamic_slab_heap.hpp>
+
+namespace ams::kern {
+
+    template<typename T, bool ClearNode = false>
+    class KDynamicResourceManager {
+        NON_COPYABLE(KDynamicResourceManager);
+        NON_MOVEABLE(KDynamicResourceManager);
+        public:
+            using DynamicSlabType = KDynamicSlabHeap<T, ClearNode>;
+        private:
+            KDynamicPageManager *m_page_allocator{};
+            DynamicSlabType *m_slab_heap{};
+        public:
+            constexpr KDynamicResourceManager() = default;
+
+            constexpr ALWAYS_INLINE KVirtualAddress GetAddress() const { return m_slab_heap->GetAddress(); }
+            constexpr ALWAYS_INLINE size_t GetSize() const { return m_slab_heap->GetSize(); }
+            constexpr ALWAYS_INLINE size_t GetUsed() const { return m_slab_heap->GetUsed(); }
+            constexpr ALWAYS_INLINE size_t GetPeak() const { return m_slab_heap->GetPeak(); }
+            constexpr ALWAYS_INLINE size_t GetCount() const { return m_slab_heap->GetCount(); }
+
+            ALWAYS_INLINE void Initialize(KDynamicPageManager *page_allocator, DynamicSlabType *slab_heap) {
+                m_page_allocator = page_allocator;
+                m_slab_heap      = slab_heap;
+            }
+
+            T *Allocate() const {
+                return m_slab_heap->Allocate(m_page_allocator);
+            }
+
+            void Free(T *t) const {
+                m_slab_heap->Free(t);
+            }
+    };
+
+    class KBlockInfoManager       : public KDynamicResourceManager<KBlockInfo>{};
+    class KMemoryBlockSlabManager : public KDynamicResourceManager<KMemoryBlock>{};
+
+    using KBlockInfoSlabHeap   = typename KBlockInfoManager::DynamicSlabType;
+    using KMemoryBlockSlabHeap = typename KMemoryBlockSlabManager::DynamicSlabType;
+
+}

libraries/libmesosphere/include/mesosphere/kern_k_dynamic_slab_heap.hpp

@@ -23,95 +23,71 @@
 namespace ams::kern {
 
     template<typename T, bool ClearNode = false>
-    class KDynamicSlabHeap {
+    class KDynamicSlabHeap : protected impl::KSlabHeapImpl {
         NON_COPYABLE(KDynamicSlabHeap);
         NON_MOVEABLE(KDynamicSlabHeap);
         private:
-            using Impl       = impl::KSlabHeapImpl;
             using PageBuffer = KDynamicPageManager::PageBuffer;
         private:
-            Impl m_impl;
+            std::atomic<size_t> m_used{};
-            KDynamicPageManager *m_page_allocator;
+            std::atomic<size_t> m_peak{};
-            std::atomic<size_t> m_used;
+            std::atomic<size_t> m_count{};
-            std::atomic<size_t> m_peak;
+            KVirtualAddress m_address{};
-            std::atomic<size_t> m_count;
+            size_t m_size{};
-            KVirtualAddress m_address;
-            size_t m_size;
-        private:
-            ALWAYS_INLINE Impl *GetImpl() {
-                return std::addressof(m_impl);
-            }
-            ALWAYS_INLINE const Impl *GetImpl() const {
-                return std::addressof(m_impl);
-            }
         public:
-            constexpr KDynamicSlabHeap() : m_impl(), m_page_allocator(), m_used(), m_peak(), m_count(), m_address(), m_size() { /* ... */ }
+            constexpr KDynamicSlabHeap() = default;
 
-            constexpr KVirtualAddress GetAddress() const { return m_address; }
+            constexpr ALWAYS_INLINE KVirtualAddress GetAddress() const { return m_address; }
-            constexpr size_t GetSize() const { return m_size; }
+            constexpr ALWAYS_INLINE size_t GetSize() const { return m_size; }
-            constexpr size_t GetUsed() const { return m_used.load(); }
+            constexpr ALWAYS_INLINE size_t GetUsed() const { return m_used.load(); }
-            constexpr size_t GetPeak() const { return m_peak.load(); }
+            constexpr ALWAYS_INLINE size_t GetPeak() const { return m_peak.load(); }
-            constexpr size_t GetCount() const { return m_count.load(); }
+            constexpr ALWAYS_INLINE size_t GetCount() const { return m_count.load(); }
 
-            constexpr bool IsInRange(KVirtualAddress addr) const {
+            constexpr ALWAYS_INLINE bool IsInRange(KVirtualAddress addr) const {
                 return this->GetAddress() <= addr && addr <= this->GetAddress() + this->GetSize() - 1;
             }
 
-            void Initialize(KVirtualAddress memory, size_t sz) {
+            ALWAYS_INLINE void Initialize(KDynamicPageManager *page_allocator, size_t num_objects) {
-                /* Set tracking fields. */
-                m_address = memory;
-                m_count   = sz / sizeof(T);
-                m_size    = m_count * sizeof(T);
-
-                /* Free blocks to memory. */
-                u8 *cur = GetPointer<u8>(m_address + m_size);
-                for (size_t i = 0; i < sz / sizeof(T); i++) {
-                    cur -= sizeof(T);
-                    this->GetImpl()->Free(cur);
-                }
-            }
-
-            void Initialize(KDynamicPageManager *page_allocator) {
-                m_page_allocator = page_allocator;
-                m_address        = m_page_allocator->GetAddress();
-                m_size           = m_page_allocator->GetSize();
-            }
-
-            void Initialize(KDynamicPageManager *page_allocator, size_t num_objects) {
                 MESOSPHERE_ASSERT(page_allocator != nullptr);
 
                 /* Initialize members. */
-                this->Initialize(page_allocator);
+                m_address = page_allocator->GetAddress();
+                m_size    = page_allocator->GetSize();
+
+                /* Initialize the base allocator. */
+                KSlabHeapImpl::Initialize();
 
                 /* Allocate until we have the correct number of objects. */
                 while (m_count.load() < num_objects) {
-                    auto *allocated = reinterpret_cast<T *>(m_page_allocator->Allocate());
+                    auto *allocated = reinterpret_cast<T *>(page_allocator->Allocate());
                     MESOSPHERE_ABORT_UNLESS(allocated != nullptr);
+
                     for (size_t i = 0; i < sizeof(PageBuffer) / sizeof(T); i++) {
-                        this->GetImpl()->Free(allocated + i);
+                        KSlabHeapImpl::Free(allocated + i);
                     }
+
                     m_count.fetch_add(sizeof(PageBuffer) / sizeof(T));
                 }
             }
 
-            T *Allocate() {
+            ALWAYS_INLINE T *Allocate(KDynamicPageManager *page_allocator) {
-                T *allocated = reinterpret_cast<T *>(this->GetImpl()->Allocate());
+                T *allocated = static_cast<T *>(KSlabHeapImpl::Allocate());
 
                 /* If we successfully allocated and we should clear the node, do so. */
                 if constexpr (ClearNode) {
                     if (AMS_LIKELY(allocated != nullptr)) {
-                        reinterpret_cast<Impl::Node *>(allocated)->next = nullptr;
+                        reinterpret_cast<KSlabHeapImpl::Node *>(allocated)->next = nullptr;
                     }
                 }
 
                 /* If we fail to allocate, try to get a new page from our next allocator. */
-                if (AMS_UNLIKELY(allocated == nullptr)) {
+                if (AMS_UNLIKELY(allocated == nullptr) ) {
-                    if (m_page_allocator != nullptr) {
+                    if (page_allocator != nullptr) {
-                        allocated = reinterpret_cast<T *>(m_page_allocator->Allocate());
+                        allocated = reinterpret_cast<T *>(page_allocator->Allocate());
                         if (allocated != nullptr) {
                             /* If we succeeded in getting a page, free the rest to our slab. */
                             for (size_t i = 1; i < sizeof(PageBuffer) / sizeof(T); i++) {
-                                this->GetImpl()->Free(allocated + i);
+                                KSlabHeapImpl::Free(allocated + i);
                             }
                             m_count.fetch_add(sizeof(PageBuffer) / sizeof(T));
                         }
@@ -135,13 +111,10 @@ namespace ams::kern {
                 return allocated;
             }
 
-            void Free(T *t) {
+            ALWAYS_INLINE void Free(T *t) {
-                this->GetImpl()->Free(t);
+                KSlabHeapImpl::Free(t);
                 m_used.fetch_sub(1);
             }
     };
 
-    class KBlockInfoManager       : public KDynamicSlabHeap<KBlockInfo>{};
-    class KMemoryBlockSlabManager : public KDynamicSlabHeap<KMemoryBlock>{};
-
 }

libraries/libmesosphere/include/mesosphere/kern_k_event.hpp

@@ -18,26 +18,23 @@
 #include <mesosphere/kern_k_auto_object.hpp>
 #include <mesosphere/kern_slab_helpers.hpp>
 #include <mesosphere/kern_k_readable_event.hpp>
-#include <mesosphere/kern_k_writable_event.hpp>
 
 namespace ams::kern {
 
-    class KEvent final : public KAutoObjectWithSlabHeapAndContainer<KEvent, KAutoObjectWithList> {
+    class KEvent final : public KAutoObjectWithSlabHeapAndContainer<KEvent, KAutoObjectWithList, true> {
         MESOSPHERE_AUTOOBJECT_TRAITS(KEvent, KAutoObject);
         private:
             KReadableEvent m_readable_event;
-            KWritableEvent m_writable_event;
             KProcess *m_owner;
             bool m_initialized;
+            bool m_readable_event_destroyed;
         public:
             constexpr KEvent()
-                : m_readable_event(), m_writable_event(), m_owner(), m_initialized()
+                : m_readable_event(), m_owner(), m_initialized(), m_readable_event_destroyed()
             {
                 /* ... */
             }
 
-            virtual ~KEvent() { /* ... */ }
-
             void Initialize();
             virtual void Finalize() override;
 
@@ -49,7 +46,11 @@ namespace ams::kern {
             virtual KProcess *GetOwner() const override { return m_owner; }
 
             KReadableEvent &GetReadableEvent() { return m_readable_event; }
-            KWritableEvent &GetWritableEvent() { return m_writable_event; }
+
+            Result Signal();
+            Result Clear();
+
+            ALWAYS_INLINE void OnReadableEventDestroyed() { m_readable_event_destroyed = true; }
     };
 
 }

libraries/libmesosphere/include/mesosphere/kern_k_handle_table.hpp

@@ -54,14 +54,10 @@ namespace ams::kern {
             }
 
             union EntryInfo {
-                struct {
+                u16 linear_id;
-                    u16 linear_id;
+                s16 next_free_index;
-                    u16 type;
-                } info;
-                s32 next_free_index;
 
-                constexpr ALWAYS_INLINE u16 GetLinearId() const { return info.linear_id; }
+                constexpr ALWAYS_INLINE u16 GetLinearId() const { return linear_id; }
-                constexpr ALWAYS_INLINE u16 GetType() const { return info.type; }
                 constexpr ALWAYS_INLINE s32 GetNextFreeIndex() const { return next_free_index; }
             };
         private:
@@ -187,17 +183,8 @@ namespace ams::kern {
             NOINLINE Result Reserve(ams::svc::Handle *out_handle);
             NOINLINE void Unreserve(ams::svc::Handle handle);
 
-            template<typename T>
+            NOINLINE Result Add(ams::svc::Handle *out_handle, KAutoObject *obj);
-            ALWAYS_INLINE Result Add(ams::svc::Handle *out_handle, T *obj) {
+            NOINLINE void Register(ams::svc::Handle handle, KAutoObject *obj);
-                static_assert(std::is_base_of<KAutoObject, T>::value);
-                return this->Add(out_handle, obj, obj->GetTypeObj().GetClassToken());
-            }
-
-            template<typename T>
-            ALWAYS_INLINE void Register(ams::svc::Handle handle, T *obj) {
-                static_assert(std::is_base_of<KAutoObject, T>::value);
-                return this->Register(handle, obj, obj->GetTypeObj().GetClassToken());
-            }
 
             template<typename T>
             ALWAYS_INLINE bool GetMultipleObjects(T **out, const ams::svc::Handle *handles, size_t num_handles) const {
@@ -242,8 +229,6 @@ namespace ams::kern {
                 return false;
             }
         private:
-            NOINLINE Result Add(ams::svc::Handle *out_handle, KAutoObject *obj, u16 type);
-            NOINLINE void Register(ams::svc::Handle handle, KAutoObject *obj, u16 type);
 
             constexpr ALWAYS_INLINE s32 AllocateEntry() {
                 MESOSPHERE_ASSERT_THIS();

libraries/libmesosphere/include/mesosphere/kern_k_interrupt_event.hpp

@@ -32,7 +32,6 @@ namespace ams::kern {
             bool m_is_initialized;
         public:
             constexpr KInterruptEvent() : m_interrupt_id(-1), m_core_id(-1), m_is_initialized(false) { /* ... */ }
-            virtual ~KInterruptEvent() { /* ... */ }
 
             Result Initialize(int32_t interrupt_name, ams::svc::InterruptType type);
             virtual void Finalize() override;
@@ -49,13 +48,10 @@ namespace ams::kern {
     class KInterruptEventTask : public KSlabAllocated<KInterruptEventTask>, public KInterruptTask {
         private:
             KInterruptEvent *m_event;
-            KLightLock m_lock;
         public:
-            constexpr KInterruptEventTask() : m_event(nullptr), m_lock() { /* ... */ }
+            constexpr KInterruptEventTask() : m_event(nullptr) { /* ... */ }
             ~KInterruptEventTask() { /* ... */ }
 
-            KLightLock &GetLock() { return m_lock; }
-
             virtual KInterruptTask *OnInterrupt(s32 interrupt_id) override;
             virtual void DoTask() override;
 

libraries/libmesosphere/include/mesosphere/kern_k_interrupt_task_manager.hpp

@@ -27,28 +27,25 @@ namespace ams::kern {
                     KInterruptTask *m_head;
                     KInterruptTask *m_tail;
                 public:
-                    constexpr TaskQueue() : m_head(nullptr), m_tail(nullptr) { /* ... */ }
+                    constexpr ALWAYS_INLINE TaskQueue() : m_head(nullptr), m_tail(nullptr) { /* ... */ }
 
-                    constexpr KInterruptTask *GetHead() { return m_head; }
+                    constexpr ALWAYS_INLINE KInterruptTask *GetHead() { return m_head; }
-                    constexpr bool IsEmpty() const { return m_head == nullptr; }
+                    constexpr ALWAYS_INLINE bool IsEmpty() const { return m_head == nullptr; }
-                    constexpr void Clear() { m_head = nullptr; m_tail = nullptr; }
+                    constexpr ALWAYS_INLINE void Clear() { m_head = nullptr; m_tail = nullptr; }
 
                     void Enqueue(KInterruptTask *task);
                     void Dequeue();
             };
         private:
             TaskQueue m_task_queue;
-            KThread *m_thread;
+            s64 m_cpu_time;
-        private:
-            static void ThreadFunction(uintptr_t arg);
-            void ThreadFunctionImpl();
         public:
-            constexpr KInterruptTaskManager() : m_task_queue(), m_thread(nullptr) { /* ... */ }
+            constexpr KInterruptTaskManager() : m_task_queue(), m_cpu_time(0) { /* ... */ }
 
-            constexpr KThread *GetThread() const { return m_thread; }
+            constexpr ALWAYS_INLINE s64 GetCpuTime() const { return m_cpu_time; }
 
-            NOINLINE void Initialize();
             void EnqueueTask(KInterruptTask *task);
+            void DoTasks();
     };
 
 }

libraries/libmesosphere/include/mesosphere/kern_k_io_pool.hpp

@@ -17,32 +17,34 @@
 #include <mesosphere/kern_common.hpp>
 #include <mesosphere/kern_k_auto_object.hpp>
 #include <mesosphere/kern_slab_helpers.hpp>
+#include <mesosphere/kern_k_io_region.hpp>
 
 namespace ams::kern {
 
-    class KProcess;
+    class KIoPool final : public KAutoObjectWithSlabHeapAndContainer<KIoPool, KAutoObjectWithList> {
-
+        MESOSPHERE_AUTOOBJECT_TRAITS(KIoPool, KAutoObject);
-    class KBeta final : public KAutoObjectWithSlabHeapAndContainer<KBeta, KAutoObjectWithList> {
-        MESOSPHERE_AUTOOBJECT_TRAITS(KBeta, KAutoObject);
         private:
-            friend class KProcess;
+            using IoRegionList = util::IntrusiveListMemberTraits<&KIoRegion::m_pool_list_node>::ListType;
         private:
-            /* NOTE: Official KBeta has size 0x88, corresponding to 0x58 bytes of fields. */
+            KLightLock m_lock;
-            /* TODO: Add these fields, if KBeta is ever instantiable in the NX kernel. */
+            IoRegionList m_io_region_list;
-            util::IntrusiveListNode m_process_list_node;
+            ams::svc::IoPoolType m_pool_type;
+            bool m_is_initialized;
         public:
-            explicit KBeta()
+            static bool IsValidIoPoolType(ams::svc::IoPoolType pool_type);
-                : m_process_list_node()
+        public:
-            {
+            explicit KIoPool() : m_lock(), m_io_region_list(), m_is_initialized(false) {
                 /* ... */
             }
 
-            virtual ~KBeta() { /* ... */ }
+            Result Initialize(ams::svc::IoPoolType pool_type);
+            virtual void Finalize() override;
 
-            /* virtual void Finalize() override; */
+            virtual bool IsInitialized() const override { return m_is_initialized; }
-
-            virtual bool IsInitialized() const override { return false /* TODO */; }
             static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
+
+            Result AddIoRegion(KIoRegion *region);
+            void RemoveIoRegion(KIoRegion *region);
     };
 
 }

libraries/libmesosphere/include/mesosphere/kern_k_io_region.hpp

@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_k_auto_object.hpp>
+#include <mesosphere/kern_slab_helpers.hpp>
+
+namespace ams::kern {
+
+    class KProcess;
+    class KIoPool;
+
+    class KIoRegion final : public KAutoObjectWithSlabHeapAndContainer<KIoRegion, KAutoObjectWithList> {
+        MESOSPHERE_AUTOOBJECT_TRAITS(KIoRegion, KAutoObject);
+        private:
+            friend class KProcess;
+            friend class KIoPool;
+        private:
+            KLightLock m_lock;
+            KIoPool *m_pool;
+            KPhysicalAddress m_physical_address;
+            size_t m_size;
+            ams::svc::MemoryMapping m_mapping;
+            ams::svc::MemoryPermission m_perm;
+            bool m_is_initialized;
+            bool m_is_mapped;
+            util::IntrusiveListNode m_process_list_node;
+            util::IntrusiveListNode m_pool_list_node;
+        public:
+            explicit KIoRegion()
+                : m_lock(), m_pool(nullptr), m_is_initialized(false), m_process_list_node(), m_pool_list_node()
+            {
+                /* ... */
+            }
+
+            Result Initialize(KIoPool *pool, KPhysicalAddress phys_addr, size_t size, ams::svc::MemoryMapping mapping, ams::svc::MemoryPermission perm);
+            virtual void Finalize() override;
+
+            virtual bool IsInitialized() const override { return m_is_initialized; }
+            static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
+
+            Result Map(KProcessAddress address, size_t size, ams::svc::MemoryPermission map_perm);
+            Result Unmap(KProcessAddress address, size_t size);
+
+            bool Overlaps(KPhysicalAddress address, size_t size) const {
+                return m_physical_address <= (address + size - 1) && address <= (m_physical_address + m_size - 1);
+            }
+
+            ALWAYS_INLINE KPhysicalAddress GetAddress() const { return m_physical_address; }
+            ALWAYS_INLINE size_t GetSize() const { return m_size; }
+    };
+
+}

libraries/libmesosphere/include/mesosphere/kern_k_light_client_session.hpp

@@ -21,13 +21,12 @@ namespace ams::kern {
 
     class KLightSession;
 
-    class KLightClientSession final : public KAutoObjectWithSlabHeapAndContainer<KLightClientSession, KAutoObjectWithList> {
+    class KLightClientSession final : public KAutoObject {
         MESOSPHERE_AUTOOBJECT_TRAITS(KLightClientSession, KAutoObject);
         private:
             KLightSession *m_parent;
         public:
             constexpr KLightClientSession() : m_parent() { /* ... */ }
-            virtual ~KLightClientSession() { /* ... */ }
 
             void Initialize(KLightSession *parent) {
                 /* Set member variables. */

libraries/libmesosphere/include/mesosphere/kern_k_light_condition_variable.hpp

@@ -27,59 +27,9 @@ namespace ams::kern {
             KThread::WaiterList m_wait_list;
         public:
             constexpr ALWAYS_INLINE KLightConditionVariable() : m_wait_list() { /* ... */ }
-        private:
-            void WaitImpl(KLightLock *lock, s64 timeout, bool allow_terminating_thread) {
-                KThread *owner = GetCurrentThreadPointer();
-                KHardwareTimer *timer;
-
-                /* Sleep the thread. */
-                {
-                    KScopedSchedulerLockAndSleep lk(&timer, owner, timeout);
-
-                    if (!allow_terminating_thread && owner->IsTerminationRequested()) {
-                        lk.CancelSleep();
-                        return;
-                    }
-
-                    lock->Unlock();
-
-
-                    /* Set the thread as waiting. */
-                    GetCurrentThread().SetState(KThread::ThreadState_Waiting);
-
-                    /* Add the thread to the queue. */
-                    m_wait_list.push_back(GetCurrentThread());
-                }
-
-                /* Remove the thread from the wait list. */
-                {
-                    KScopedSchedulerLock sl;
-
-                    m_wait_list.erase(m_wait_list.iterator_to(GetCurrentThread()));
-                }
-
-                /* Cancel the task that the sleep setup. */
-                if (timer != nullptr) {
-                    timer->CancelTask(owner);
-                }
-
-                /* Re-acquire the lock. */
-                lock->Lock();
-            }
         public:
-            void Wait(KLightLock *lock, s64 timeout = -1ll, bool allow_terminating_thread = true) {
+            void Wait(KLightLock *lock, s64 timeout = -1ll, bool allow_terminating_thread = true);
-                this->WaitImpl(lock, timeout, allow_terminating_thread);
+            void Broadcast();
-            }
-
-            void Broadcast() {
-                KScopedSchedulerLock lk;
-
-                /* Signal all threads. */
-                for (auto &thread : m_wait_list) {
-                    thread.SetState(KThread::ThreadState_Runnable);
-                }
-            }
-
     };
 
 }

libraries/libmesosphere/include/mesosphere/kern_k_light_server_session.hpp

@@ -23,17 +23,16 @@ namespace ams::kern {
 
     class KLightSession;
 
-    class KLightServerSession final : public KAutoObjectWithSlabHeapAndContainer<KLightServerSession, KAutoObjectWithList>, public util::IntrusiveListBaseNode<KLightServerSession> {
+    class KLightServerSession final : public KAutoObject, public util::IntrusiveListBaseNode<KLightServerSession> {
         MESOSPHERE_AUTOOBJECT_TRAITS(KLightServerSession, KAutoObject);
         private:
             KLightSession *m_parent;
-            KThreadQueue m_request_queue;
+            KThread::WaiterList m_request_list;
-            KThreadQueue m_server_queue;
             KThread *m_current_request;
+            u64 m_server_thread_id;
             KThread *m_server_thread;
         public:
-            constexpr KLightServerSession() : m_parent(), m_request_queue(), m_server_queue(), m_current_request(), m_server_thread() { /* ... */ }
+            constexpr KLightServerSession() : m_parent(), m_request_list(), m_current_request(), m_server_thread_id(), m_server_thread() { /* ... */ }
-            virtual ~KLightServerSession() { /* ... */ }
 
             void Initialize(KLightSession *parent) {
                 /* Set member variables. */

libraries/libmesosphere/include/mesosphere/kern_k_light_session.hpp

@@ -25,7 +25,7 @@ namespace ams::kern {
     class KClientPort;
     class KProcess;
 
-    class KLightSession final : public KAutoObjectWithSlabHeapAndContainer<KLightSession, KAutoObjectWithList> {
+    class KLightSession final : public KAutoObjectWithSlabHeapAndContainer<KLightSession, KAutoObjectWithList, true> {
         MESOSPHERE_AUTOOBJECT_TRAITS(KLightSession, KAutoObject);
         private:
             enum class State : u8 {
@@ -52,8 +52,6 @@ namespace ams::kern {
                 /* ... */
             }
 
-            virtual ~KLightSession() { /* ... */ }
-
             void Initialize(KClientPort *client_port, uintptr_t name);
             virtual void Finalize() override;
 

libraries/libmesosphere/include/mesosphere/kern_k_memory_block.hpp

@@ -105,6 +105,8 @@ namespace ams::kern {
 
         KMemoryState_GeneratedCode      = ams::svc::MemoryState_GeneratedCode       | KMemoryState_FlagMapped | KMemoryState_FlagReferenceCounted    | KMemoryState_FlagCanDebug,
         KMemoryState_CodeOut            = ams::svc::MemoryState_CodeOut             | KMemoryState_FlagMapped | KMemoryState_FlagReferenceCounted,
+
+        KMemoryState_Coverage           = ams::svc::MemoryState_Coverage            | KMemoryState_FlagMapped,
     };
 
 #if 1
@@ -130,6 +132,7 @@ namespace ams::kern {
     static_assert(KMemoryState_Kernel           == 0x00002013);
     static_assert(KMemoryState_GeneratedCode    == 0x00402214);
     static_assert(KMemoryState_CodeOut          == 0x00402015);
+    static_assert(KMemoryState_Coverage         == 0x00002016);
 #endif
 
     enum KMemoryPermission : u8 {

libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp

@@ -144,10 +144,12 @@ namespace ams::kern {
             static NOINLINE const KMemoryRegion &GetPageTableHeapRegion()  { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_VirtualDramKernelPtHeap)); }
             static NOINLINE const KMemoryRegion &GetKernelStackRegion()    { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_KernelStack)); }
             static NOINLINE const KMemoryRegion &GetTempRegion()           { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_KernelTemp)); }
+            static NOINLINE const KMemoryRegion &GetSlabRegion()           { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_KernelSlab)); }
 
             static NOINLINE const KMemoryRegion &GetKernelTraceBufferRegion() { return Dereference(GetVirtualLinearMemoryRegionTree().FindByType(KMemoryRegionType_VirtualDramKernelTraceBuffer)); }
 
             static NOINLINE const KMemoryRegion &GetVirtualLinearRegion(KVirtualAddress address) { return Dereference(FindLinear(address)); }
+            static NOINLINE const KMemoryRegion &GetPhysicalLinearRegion(KPhysicalAddress address) { return Dereference(FindLinear(address)); }
 
             static NOINLINE const KMemoryRegion *GetPhysicalKernelTraceBufferRegion() { return GetPhysicalMemoryRegionTree().FindFirstDerived(KMemoryRegionType_KernelTraceBuffer); }
             static NOINLINE const KMemoryRegion *GetPhysicalOnMemoryBootImageRegion() { return GetPhysicalMemoryRegionTree().FindFirstDerived(KMemoryRegionType_OnMemoryBootImage); }

libraries/libmesosphere/include/mesosphere/kern_k_memory_manager.hpp

@@ -70,37 +70,37 @@ namespace ams::kern {
                 public:
                     Impl() : m_heap(), m_page_reference_counts(), m_management_region(), m_pool(), m_next(), m_prev() { /* ... */ }
 
-                    size_t Initialize(uintptr_t address, size_t size, KVirtualAddress management, KVirtualAddress management_end, Pool p);
+                    size_t Initialize(KPhysicalAddress address, size_t size, KVirtualAddress management, KVirtualAddress management_end, Pool p);
 
-                    KVirtualAddress AllocateBlock(s32 index, bool random) { return m_heap.AllocateBlock(index, random); }
+                    KPhysicalAddress AllocateBlock(s32 index, bool random) { return m_heap.AllocateBlock(index, random); }
-                    void Free(KVirtualAddress addr, size_t num_pages) { m_heap.Free(addr, num_pages); }
+                    void Free(KPhysicalAddress addr, size_t num_pages) { m_heap.Free(addr, num_pages); }
 
                     void SetInitialUsedHeapSize(size_t reserved_size) { m_heap.SetInitialUsedSize(reserved_size); }
 
                     void InitializeOptimizedMemory() { std::memset(GetVoidPointer(m_management_region), 0, CalculateOptimizedProcessOverheadSize(m_heap.GetSize())); }
 
-                    void TrackUnoptimizedAllocation(KVirtualAddress block, size_t num_pages);
+                    void TrackUnoptimizedAllocation(KPhysicalAddress block, size_t num_pages);
-                    void TrackOptimizedAllocation(KVirtualAddress block, size_t num_pages);
+                    void TrackOptimizedAllocation(KPhysicalAddress block, size_t num_pages);
 
-                    bool ProcessOptimizedAllocation(KVirtualAddress block, size_t num_pages, u8 fill_pattern);
+                    bool ProcessOptimizedAllocation(KPhysicalAddress block, size_t num_pages, u8 fill_pattern);
 
                     constexpr Pool GetPool() const { return m_pool; }
                     constexpr size_t GetSize() const { return m_heap.GetSize(); }
-                    constexpr KVirtualAddress GetEndAddress() const { return m_heap.GetEndAddress(); }
+                    constexpr KPhysicalAddress GetEndAddress() const { return m_heap.GetEndAddress(); }
 
                     size_t GetFreeSize() const { return m_heap.GetFreeSize(); }
 
                     void DumpFreeList() const { return m_heap.DumpFreeList(); }
 
-                    constexpr size_t GetPageOffset(KVirtualAddress address)      const { return m_heap.GetPageOffset(address); }
+                    constexpr size_t GetPageOffset(KPhysicalAddress address)      const { return m_heap.GetPageOffset(address); }
-                    constexpr size_t GetPageOffsetToEnd(KVirtualAddress address) const { return m_heap.GetPageOffsetToEnd(address); }
+                    constexpr size_t GetPageOffsetToEnd(KPhysicalAddress address) const { return m_heap.GetPageOffsetToEnd(address); }
 
                     constexpr void SetNext(Impl *n) { m_next = n; }
                     constexpr void SetPrev(Impl *n) { m_prev = n; }
                     constexpr Impl *GetNext() const { return m_next; }
                     constexpr Impl *GetPrev() const { return m_prev; }
 
-                    void OpenFirst(KVirtualAddress address, size_t num_pages) {
+                    void OpenFirst(KPhysicalAddress address, size_t num_pages) {
                         size_t index = this->GetPageOffset(address);
                         const size_t end = index + num_pages;
                         while (index < end) {
@@ -111,7 +111,7 @@ namespace ams::kern {
                         }
                     }
 
-                    void Open(KVirtualAddress address, size_t num_pages) {
+                    void Open(KPhysicalAddress address, size_t num_pages) {
                         size_t index = this->GetPageOffset(address);
                         const size_t end = index + num_pages;
                         while (index < end) {
@@ -122,7 +122,7 @@ namespace ams::kern {
                         }
                     }
 
-                    void Close(KVirtualAddress address, size_t num_pages) {
+                    void Close(KPhysicalAddress address, size_t num_pages) {
                         size_t index = this->GetPageOffset(address);
                         const size_t end = index + num_pages;
 
@@ -164,12 +164,12 @@ namespace ams::kern {
             u64 m_optimized_process_ids[Pool_Count];
             bool m_has_optimized_process[Pool_Count];
         private:
-            Impl &GetManager(KVirtualAddress address) {
+            Impl &GetManager(KPhysicalAddress address) {
-                return m_managers[KMemoryLayout::GetVirtualLinearRegion(address).GetAttributes()];
+                return m_managers[KMemoryLayout::GetPhysicalLinearRegion(address).GetAttributes()];
             }
 
-            const Impl &GetManager(KVirtualAddress address) const {
+            const Impl &GetManager(KPhysicalAddress address) const {
-                return m_managers[KMemoryLayout::GetVirtualLinearRegion(address).GetAttributes()];
+                return m_managers[KMemoryLayout::GetPhysicalLinearRegion(address).GetAttributes()];
             }
 
             constexpr Impl *GetFirstManager(Pool pool, Direction dir) {
@@ -197,15 +197,15 @@ namespace ams::kern {
             NOINLINE Result InitializeOptimizedMemory(u64 process_id, Pool pool);
             NOINLINE void FinalizeOptimizedMemory(u64 process_id, Pool pool);
 
-            NOINLINE KVirtualAddress AllocateAndOpenContinuous(size_t num_pages, size_t align_pages, u32 option);
+            NOINLINE KPhysicalAddress AllocateAndOpenContinuous(size_t num_pages, size_t align_pages, u32 option);
             NOINLINE Result AllocateAndOpen(KPageGroup *out, size_t num_pages, u32 option);
             NOINLINE Result AllocateAndOpenForProcess(KPageGroup *out, size_t num_pages, u32 option, u64 process_id, u8 fill_pattern);
 
-            Pool GetPool(KVirtualAddress address) const {
+            Pool GetPool(KPhysicalAddress address) const {
                 return this->GetManager(address).GetPool();
             }
 
-            void Open(KVirtualAddress address, size_t num_pages) {
+            void Open(KPhysicalAddress address, size_t num_pages) {
                 /* Repeatedly open references until we've done so for all pages. */
                 while (num_pages) {
                     auto &manager = this->GetManager(address);
@@ -221,7 +221,7 @@ namespace ams::kern {
                 }
             }
 
-            void Close(KVirtualAddress address, size_t num_pages) {
+            void Close(KPhysicalAddress address, size_t num_pages) {
                 /* Repeatedly close references until we've done so for all pages. */
                 while (num_pages) {
                     auto &manager = this->GetManager(address);

libraries/libmesosphere/include/mesosphere/kern_k_memory_region.hpp

@@ -237,12 +237,6 @@ namespace ams::kern {
         public:
             NOINLINE void InsertDirectly(uintptr_t address, uintptr_t last_address, u32 attr = 0, u32 type_id = 0);
             NOINLINE bool Insert(uintptr_t address, size_t size, u32 type_id, u32 new_attr = 0, u32 old_attr = 0);
-
-            NOINLINE KVirtualAddress GetRandomAlignedRegion(size_t size, size_t alignment, u32 type_id);
-
-            ALWAYS_INLINE KVirtualAddress GetRandomAlignedRegionWithGuard(size_t size, size_t alignment, u32 type_id, size_t guard_size) {
-                return this->GetRandomAlignedRegion(size + 2 * guard_size, alignment, type_id) + guard_size;
-            }
         public:
             /* Iterator accessors. */
             iterator begin() {

libraries/libmesosphere/include/mesosphere/kern_k_page_group.hpp

@@ -22,77 +22,121 @@ namespace ams::kern {
 
     class KBlockInfoManager;
 
-    class KBlockInfo : public util::IntrusiveListBaseNode<KBlockInfo> {
+    class KPageGroup;
+
+    class KBlockInfo {
         private:
-            KVirtualAddress m_address;
+            friend class KPageGroup;
-            size_t m_num_pages;
+        private:
+            KBlockInfo *m_next{};
+            u32 m_page_index{};
+            u32 m_num_pages{};
         public:
-            constexpr KBlockInfo() : util::IntrusiveListBaseNode<KBlockInfo>(), m_address(), m_num_pages() { /* ... */ }
+            constexpr KBlockInfo() = default;
 
-            constexpr void Initialize(KVirtualAddress addr, size_t np) {
+            constexpr ALWAYS_INLINE void Initialize(KPhysicalAddress addr, size_t np) {
-                m_address = addr;
+                MESOSPHERE_ASSERT(util::IsAligned(GetInteger(addr), PageSize));
-                m_num_pages = np;
+                MESOSPHERE_ASSERT(static_cast<u32>(np) == np);
+
+                m_page_index = GetInteger(addr) / PageSize;
+                m_num_pages  = np;
             }
 
-            constexpr KVirtualAddress GetAddress() const { return m_address; }
+            constexpr ALWAYS_INLINE KPhysicalAddress GetAddress() const { return m_page_index * PageSize; }
-            constexpr size_t GetNumPages() const { return m_num_pages; }
+            constexpr ALWAYS_INLINE size_t GetNumPages() const { return m_num_pages; }
-            constexpr size_t GetSize() const { return this->GetNumPages() * PageSize; }
+            constexpr ALWAYS_INLINE size_t GetSize() const { return this->GetNumPages() * PageSize; }
-            constexpr KVirtualAddress GetEndAddress() const { return this->GetAddress() + this->GetSize(); }
+            constexpr ALWAYS_INLINE KPhysicalAddress GetEndAddress() const { return (m_page_index + m_num_pages) * PageSize; }
-            constexpr KVirtualAddress GetLastAddress() const { return this->GetEndAddress() - 1; }
+            constexpr ALWAYS_INLINE KPhysicalAddress GetLastAddress() const { return this->GetEndAddress() - 1; }
 
-            constexpr bool IsEquivalentTo(const KBlockInfo &rhs) const {
+            constexpr ALWAYS_INLINE KBlockInfo *GetNext() const { return m_next; }
-                return m_address == rhs.m_address && m_num_pages == rhs.m_num_pages;
+
+            constexpr ALWAYS_INLINE bool IsEquivalentTo(const KBlockInfo &rhs) const {
+                return m_page_index == rhs.m_page_index && m_num_pages == rhs.m_num_pages;
             }
 
-            constexpr bool operator==(const KBlockInfo &rhs) const {
+            constexpr ALWAYS_INLINE bool operator==(const KBlockInfo &rhs) const {
                 return this->IsEquivalentTo(rhs);
             }
 
-            constexpr bool operator!=(const KBlockInfo &rhs) const {
+            constexpr ALWAYS_INLINE bool operator!=(const KBlockInfo &rhs) const {
                 return !(*this == rhs);
             }
 
-            constexpr bool IsStrictlyBefore(KVirtualAddress addr) const {
+            constexpr ALWAYS_INLINE bool IsStrictlyBefore(KPhysicalAddress addr) const {
-                const KVirtualAddress end = this->GetEndAddress();
+                const KPhysicalAddress end = this->GetEndAddress();
 
-                if (m_address != Null<KVirtualAddress> && end == Null<KVirtualAddress>) {
+                if (m_page_index != 0 && end == Null<KPhysicalAddress>) {
                     return false;
                 }
 
                 return end < addr;
             }
 
-            constexpr bool operator<(KVirtualAddress addr) const {
+            constexpr ALWAYS_INLINE bool operator<(KPhysicalAddress addr) const {
                 return this->IsStrictlyBefore(addr);
             }
 
-            constexpr bool TryConcatenate(KVirtualAddress addr, size_t np) {
+            constexpr ALWAYS_INLINE bool TryConcatenate(KPhysicalAddress addr, size_t np) {
-                if (addr != Null<KVirtualAddress> && addr == this->GetEndAddress()) {
+                if (addr != Null<KPhysicalAddress> && addr == this->GetEndAddress()) {
                     m_num_pages += np;
                     return true;
                 }
                 return false;
             }
+        private:
+            constexpr ALWAYS_INLINE void SetNext(KBlockInfo *next) {
+                m_next = next;
+            }
     };
+    static_assert(sizeof(KBlockInfo) <= 0x10);
 
     class KPageGroup {
         public:
-            using BlockInfoList = util::IntrusiveListBaseTraits<KBlockInfo>::ListType;
+            class Iterator {
-            using iterator = BlockInfoList::const_iterator;
+                public:
+                    using iterator_category = std::forward_iterator_tag;
+                    using value_type        = const KBlockInfo;
+                    using difference_type   = std::ptrdiff_t;
+                    using pointer           = value_type *;
+                    using reference         = value_type &;
+                private:
+                    pointer m_node;
+                public:
+                    constexpr explicit ALWAYS_INLINE Iterator(pointer n) : m_node(n) { /* ... */ }
+
+                    constexpr ALWAYS_INLINE bool operator==(const Iterator &rhs) const { return m_node == rhs.m_node; }
+                    constexpr ALWAYS_INLINE bool operator!=(const Iterator &rhs) const { return !(*this == rhs); }
+
+                    constexpr ALWAYS_INLINE pointer operator->() const { return m_node; }
+                    constexpr ALWAYS_INLINE reference operator*() const { return *m_node; }
+
+                    constexpr ALWAYS_INLINE Iterator &operator++() {
+                        m_node = m_node->GetNext();
+                        return *this;
+                    }
+
+                    constexpr ALWAYS_INLINE Iterator operator++(int) {
+                        const Iterator it{*this};
+                        ++(*this);
+                        return it;
+                    }
+            };
         private:
-            BlockInfoList m_block_list;
+            KBlockInfo *m_first_block;
+            KBlockInfo *m_last_block;
             KBlockInfoManager *m_manager;
         public:
-            explicit KPageGroup(KBlockInfoManager *m) : m_block_list(), m_manager(m) { /* ... */ }
+            explicit KPageGroup(KBlockInfoManager *m) : m_first_block(), m_last_block(), m_manager(m) { /* ... */ }
             ~KPageGroup() { this->Finalize(); }
 
+            void CloseAndReset();
             void Finalize();
 
-            iterator begin() const { return m_block_list.begin(); }
+            ALWAYS_INLINE Iterator begin() const { return Iterator{m_first_block}; }
-            iterator end() const { return m_block_list.end(); }
+            ALWAYS_INLINE Iterator end() const { return Iterator{nullptr}; }
-            bool empty() const { return m_block_list.empty(); }
+            ALWAYS_INLINE bool empty() const { return m_first_block == nullptr; }
 
-            Result AddBlock(KVirtualAddress addr, size_t num_pages);
+            Result AddBlock(KPhysicalAddress addr, size_t num_pages);
             void Open() const;
             void Close() const;
 
@@ -100,11 +144,11 @@ namespace ams::kern {
 
             bool IsEquivalentTo(const KPageGroup &rhs) const;
 
-            bool operator==(const KPageGroup &rhs) const {
+            ALWAYS_INLINE bool operator==(const KPageGroup &rhs) const {
                 return this->IsEquivalentTo(rhs);
             }
 
-            bool operator!=(const KPageGroup &rhs) const {
+            ALWAYS_INLINE bool operator!=(const KPageGroup &rhs) const {
                 return !(*this == rhs);
             }
     };

libraries/libmesosphere/include/mesosphere/kern_k_page_heap.hpp

@@ -54,7 +54,7 @@ namespace ams::kern {
             class Block {
                 private:
                     KPageBitmap m_bitmap;
-                    KVirtualAddress m_heap_address;
+                    KPhysicalAddress m_heap_address;
                     uintptr_t m_end_offset;
                     size_t m_block_shift;
                     size_t m_next_block_shift;
@@ -68,13 +68,13 @@ namespace ams::kern {
                     constexpr size_t GetNumFreeBlocks() const { return m_bitmap.GetNumBits(); }
                     constexpr size_t GetNumFreePages() const { return this->GetNumFreeBlocks() * this->GetNumPages(); }
 
-                    u64 *Initialize(KVirtualAddress addr, size_t size, size_t bs, size_t nbs, u64 *bit_storage) {
+                    u64 *Initialize(KPhysicalAddress addr, size_t size, size_t bs, size_t nbs, u64 *bit_storage) {
                         /* Set shifts. */
                         m_block_shift = bs;
                         m_next_block_shift = nbs;
 
                         /* Align up the address. */
-                        KVirtualAddress end = addr + size;
+                        KPhysicalAddress end = addr + size;
                         const size_t align = (m_next_block_shift != 0) ? (u64(1) << m_next_block_shift) : (u64(1) << m_block_shift);
                         addr = util::AlignDown(GetInteger(addr), align);
                         end  = util::AlignUp(GetInteger(end), align);
@@ -84,7 +84,7 @@ namespace ams::kern {
                         return m_bitmap.Initialize(bit_storage, m_end_offset);
                     }
 
-                    KVirtualAddress PushBlock(KVirtualAddress address) {
+                    KPhysicalAddress PushBlock(KPhysicalAddress address) {
                         /* Set the bit for the free block. */
                         size_t offset = (address - m_heap_address) >> this->GetShift();
                         m_bitmap.SetBit(offset);
@@ -99,14 +99,14 @@ namespace ams::kern {
                         }
 
                         /* We couldn't coalesce, or we're already as big as possible. */
-                        return Null<KVirtualAddress>;
+                        return Null<KPhysicalAddress>;
                     }
 
-                    KVirtualAddress PopBlock(bool random) {
+                    KPhysicalAddress PopBlock(bool random) {
                         /* Find a free block. */
                         ssize_t soffset = m_bitmap.FindFreeBlock(random);
                         if (soffset < 0) {
-                            return Null<KVirtualAddress>;
+                            return Null<KPhysicalAddress>;
                         }
                         const size_t offset = static_cast<size_t>(soffset);
 
@@ -123,27 +123,27 @@ namespace ams::kern {
                     }
             };
         private:
-            KVirtualAddress m_heap_address;
+            KPhysicalAddress m_heap_address;
             size_t m_heap_size;
             size_t m_initial_used_size;
             size_t m_num_blocks;
             Block m_blocks[NumMemoryBlockPageShifts];
         private:
-            void Initialize(KVirtualAddress heap_address, size_t heap_size, KVirtualAddress management_address, size_t management_size, const size_t *block_shifts, size_t num_block_shifts);
+            void Initialize(KPhysicalAddress heap_address, size_t heap_size, KVirtualAddress management_address, size_t management_size, const size_t *block_shifts, size_t num_block_shifts);
             size_t GetNumFreePages() const;
 
-            void FreeBlock(KVirtualAddress block, s32 index);
+            void FreeBlock(KPhysicalAddress block, s32 index);
         public:
             KPageHeap() : m_heap_address(), m_heap_size(), m_initial_used_size(), m_num_blocks(), m_blocks() { /* ... */ }
 
-            constexpr KVirtualAddress GetAddress() const { return m_heap_address; }
+            constexpr KPhysicalAddress GetAddress() const { return m_heap_address; }
             constexpr size_t GetSize() const { return m_heap_size; }
-            constexpr KVirtualAddress GetEndAddress() const { return this->GetAddress() + this->GetSize(); }
+            constexpr KPhysicalAddress GetEndAddress() const { return this->GetAddress() + this->GetSize(); }
-            constexpr size_t GetPageOffset(KVirtualAddress block) const { return (block - this->GetAddress()) / PageSize; }
+            constexpr size_t GetPageOffset(KPhysicalAddress block) const { return (block - this->GetAddress()) / PageSize; }
-            constexpr size_t GetPageOffsetToEnd(KVirtualAddress block) const { return (this->GetEndAddress() - block) / PageSize; }
+            constexpr size_t GetPageOffsetToEnd(KPhysicalAddress block) const { return (this->GetEndAddress() - block) / PageSize; }
 
-            void Initialize(KVirtualAddress heap_address, size_t heap_size, KVirtualAddress management_address, size_t management_size) {
+            void Initialize(KPhysicalAddress heap_address, size_t heap_size, KVirtualAddress management_address, size_t management_size) {
-                return Initialize(heap_address, heap_size, management_address, management_size, MemoryBlockPageShifts, NumMemoryBlockPageShifts);
+                return this->Initialize(heap_address, heap_size, management_address, management_size, MemoryBlockPageShifts, NumMemoryBlockPageShifts);
             }
 
             size_t GetFreeSize() const { return this->GetNumFreePages() * PageSize; }
@@ -158,8 +158,8 @@ namespace ams::kern {
                 m_initial_used_size = m_heap_size - free_size - reserved_size;
             }
 
-            KVirtualAddress AllocateBlock(s32 index, bool random);
+            KPhysicalAddress AllocateBlock(s32 index, bool random);
-            void Free(KVirtualAddress addr, size_t num_pages);
+            void Free(KPhysicalAddress addr, size_t num_pages);
         private:
             static size_t CalculateManagementOverheadSize(size_t region_size, const size_t *block_shifts, size_t num_block_shifts);
         public:

libraries/libmesosphere/include/mesosphere/kern_k_page_table_base.hpp

@@ -57,7 +57,7 @@ namespace ams::kern {
             using TraversalContext = KPageTableImpl::TraversalContext;
 
             struct MemoryRange {
-                KVirtualAddress address;
+                KPhysicalAddress address;
                 size_t size;
 
                 void Close();
@@ -178,7 +178,6 @@ namespace ams::kern {
             KResourceLimit *m_resource_limit{};
             const KMemoryRegion *m_cached_physical_linear_region{};
             const KMemoryRegion *m_cached_physical_heap_region{};
-            const KMemoryRegion *m_cached_virtual_heap_region{};
             MemoryFillValue m_heap_fill_value{};
             MemoryFillValue m_ipc_fill_value{};
             MemoryFillValue m_stack_fill_value{};
@@ -257,18 +256,6 @@ namespace ams::kern {
                 return KMemoryLayout::IsHeapPhysicalAddress(m_cached_physical_heap_region, phys_addr);
             }
 
-            ALWAYS_INLINE bool IsHeapVirtualAddress(KVirtualAddress virt_addr) {
-                MESOSPHERE_ASSERT(this->IsLockedByCurrentThread());
-
-                return KMemoryLayout::IsHeapVirtualAddress(m_cached_virtual_heap_region, virt_addr);
-            }
-
-            ALWAYS_INLINE bool IsHeapVirtualAddress(KVirtualAddress virt_addr, size_t size) {
-                MESOSPHERE_ASSERT(this->IsLockedByCurrentThread());
-
-                return KMemoryLayout::IsHeapVirtualAddress(m_cached_virtual_heap_region, virt_addr, size);
-            }
-
             ALWAYS_INLINE bool ContainsPages(KProcessAddress addr, size_t num_pages) const {
                 return (m_address_space_start <= addr) && (num_pages <= (m_address_space_end - m_address_space_start) / PageSize) && (addr + num_pages * PageSize - 1 <= m_address_space_end - 1);
             }
@@ -352,6 +339,8 @@ namespace ams::kern {
             Result MapCodeMemory(KProcessAddress dst_address, KProcessAddress src_address, size_t size);
             Result UnmapCodeMemory(KProcessAddress dst_address, KProcessAddress src_address, size_t size);
             Result MapIo(KPhysicalAddress phys_addr, size_t size, KMemoryPermission perm);
+            Result MapIoRegion(KProcessAddress dst_address, KPhysicalAddress phys_addr, size_t size, ams::svc::MemoryMapping mapping, ams::svc::MemoryPermission perm);
+            Result UnmapIoRegion(KProcessAddress dst_address, KPhysicalAddress phys_addr, size_t size);
             Result MapStatic(KPhysicalAddress phys_addr, size_t size, KMemoryPermission perm);
             Result MapRegion(KMemoryRegionType region_type, KMemoryPermission perm);
 
@@ -388,7 +377,7 @@ namespace ams::kern {
             Result LockForUnmapDeviceAddressSpace(KProcessAddress address, size_t size);
 
             Result UnlockForDeviceAddressSpace(KProcessAddress address, size_t size);
-            Result UnlockForDeviceAddressSpacePartialMap(KProcessAddress address, size_t size, size_t mapped_size);
+            Result UnlockForDeviceAddressSpacePartialMap(KProcessAddress address, size_t size);
 
             Result OpenMemoryRangeForMapDeviceAddressSpace(KPageTableBase::MemoryRange *out, KProcessAddress address, size_t size, KMemoryPermission perm, bool is_aligned);
             Result OpenMemoryRangeForUnmapDeviceAddressSpace(MemoryRange *out, KProcessAddress address, size_t size);

libraries/libmesosphere/include/mesosphere/kern_k_page_table_manager.hpp

@@ -15,58 +15,27 @@
  */
 #pragma once
 #include <mesosphere/kern_common.hpp>
-#include <mesosphere/kern_slab_helpers.hpp>
+#include <mesosphere/kern_k_page_table_slab_heap.hpp>
-#include <mesosphere/kern_k_dynamic_slab_heap.hpp>
+#include <mesosphere/kern_k_dynamic_resource_manager.hpp>
 
 namespace ams::kern {
 
-    namespace impl {
+    class KPageTableManager : public KDynamicResourceManager<impl::PageTablePage, true> {
-
-        class PageTablePage {
-            private:
-                u8 m_buffer[PageSize];
-            public:
-                ALWAYS_INLINE PageTablePage() { /* Do not initialize anything. */ }
-        };
-        static_assert(sizeof(PageTablePage) == PageSize);
-
-    }
-
-    class KPageTableManager : public KDynamicSlabHeap<impl::PageTablePage, true> {
         public:
-            using RefCount = u16;
+            using RefCount = KPageTableSlabHeap::RefCount;
-            static constexpr size_t PageTableSize = sizeof(impl::PageTablePage);
+            static constexpr size_t PageTableSize = KPageTableSlabHeap::PageTableSize;
-            static_assert(PageTableSize == PageSize);
         private:
-            using BaseHeap = KDynamicSlabHeap<impl::PageTablePage, true>;
+            using BaseHeap = KDynamicResourceManager<impl::PageTablePage, true>;
         private:
-            RefCount *m_ref_counts;
+            KPageTableSlabHeap *m_pt_heap{};
         public:
-            static constexpr size_t CalculateReferenceCountSize(size_t size) {
+            constexpr KPageTableManager() = default;
-                return (size / PageSize) * sizeof(RefCount);
-            }
-        public:
-            constexpr KPageTableManager() : BaseHeap(), m_ref_counts() { /* ... */ }
-        private:
-            void Initialize(RefCount *rc) {
-                m_ref_counts = rc;
-                for (size_t i = 0; i < this->GetSize() / PageSize; i++) {
-                    m_ref_counts[i] = 0;
-                }
-            }
 
-            constexpr RefCount *GetRefCountPointer(KVirtualAddress addr) const {
+            ALWAYS_INLINE void Initialize(KDynamicPageManager *page_allocator, KPageTableSlabHeap *pt_heap) {
-                return std::addressof(m_ref_counts[(addr - this->GetAddress()) / PageSize]);
+                m_pt_heap = pt_heap;
-            }
-        public:
-            void Initialize(KDynamicPageManager *page_allocator, RefCount *rc) {
-                BaseHeap::Initialize(page_allocator);
-                this->Initialize(rc);
-            }
 
-            void Initialize(KDynamicPageManager *page_allocator, size_t object_count, RefCount *rc) {
+                static_assert(std::derived_from<KPageTableSlabHeap, DynamicSlabType>);
-                BaseHeap::Initialize(page_allocator, object_count);
+                BaseHeap::Initialize(page_allocator, pt_heap);
-                this->Initialize(rc);
             }
 
             KVirtualAddress Allocate() {
@@ -74,33 +43,23 @@ namespace ams::kern {
             }
 
             void Free(KVirtualAddress addr) {
-                /* Free the page. */
+                return BaseHeap::Free(GetPointer<impl::PageTablePage>(addr));
-                BaseHeap::Free(GetPointer<impl::PageTablePage>(addr));
             }
 
-            RefCount GetRefCount(KVirtualAddress addr) const {
+            ALWAYS_INLINE RefCount GetRefCount(KVirtualAddress addr) const {
-                MESOSPHERE_ASSERT(this->IsInRange(addr));
+                return m_pt_heap->GetRefCount(addr);
-                return *this->GetRefCountPointer(addr);
             }
 
-            void Open(KVirtualAddress addr, int count) {
+            ALWAYS_INLINE void Open(KVirtualAddress addr, int count) {
-                MESOSPHERE_ASSERT(this->IsInRange(addr));
+                return m_pt_heap->Open(addr, count);
-
-                *this->GetRefCountPointer(addr) += count;
-
-                MESOSPHERE_ABORT_UNLESS(this->GetRefCount(addr) > 0);
             }
 
-            bool Close(KVirtualAddress addr, int count) {
+            ALWAYS_INLINE bool Close(KVirtualAddress addr, int count) {
-                MESOSPHERE_ASSERT(this->IsInRange(addr));
+                return m_pt_heap->Close(addr, count);
-                MESOSPHERE_ABORT_UNLESS(this->GetRefCount(addr) >= count);
-
-                *this->GetRefCountPointer(addr) -= count;
-                return this->GetRefCount(addr) == 0;
             }
 
-            constexpr bool IsInPageTableHeap(KVirtualAddress addr) const {
+            constexpr ALWAYS_INLINE bool IsInPageTableHeap(KVirtualAddress addr) const {
-                return this->IsInRange(addr);
+                return m_pt_heap->IsInRange(addr);
             }
     };
 

libraries/libmesosphere/include/mesosphere/kern_k_page_table_slab_heap.hpp

@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_slab_helpers.hpp>
+#include <mesosphere/kern_k_dynamic_slab_heap.hpp>
+
+namespace ams::kern {
+
+    namespace impl {
+
+        class PageTablePage {
+            private:
+                u8 m_buffer[PageSize];
+            public:
+                ALWAYS_INLINE PageTablePage() { /* Do not initialize anything. */ }
+        };
+        static_assert(sizeof(PageTablePage) == PageSize);
+
+    }
+
+    class KPageTableSlabHeap : public KDynamicSlabHeap<impl::PageTablePage, true> {
+        public:
+            using RefCount = u16;
+            static constexpr size_t PageTableSize = sizeof(impl::PageTablePage);
+            static_assert(PageTableSize == PageSize);
+        private:
+            using BaseHeap = KDynamicSlabHeap<impl::PageTablePage, true>;
+        private:
+            RefCount *m_ref_counts{};
+        public:
+            static constexpr ALWAYS_INLINE size_t CalculateReferenceCountSize(size_t size) {
+                return (size / PageSize) * sizeof(RefCount);
+            }
+        public:
+            constexpr KPageTableSlabHeap() = default;
+        private:
+            ALWAYS_INLINE void Initialize(RefCount *rc) {
+                m_ref_counts = rc;
+                for (size_t i = 0; i < this->GetSize() / PageSize; i++) {
+                    m_ref_counts[i] = 0;
+                }
+            }
+
+            constexpr ALWAYS_INLINE RefCount *GetRefCountPointer(KVirtualAddress addr) const {
+                return m_ref_counts + ((addr - this->GetAddress()) / PageSize);
+            }
+        public:
+            ALWAYS_INLINE void Initialize(KDynamicPageManager *page_allocator, size_t object_count, RefCount *rc) {
+                BaseHeap::Initialize(page_allocator, object_count);
+                this->Initialize(rc);
+            }
+
+            ALWAYS_INLINE RefCount GetRefCount(KVirtualAddress addr) const {
+                MESOSPHERE_ASSERT(this->IsInRange(addr));
+                return *this->GetRefCountPointer(addr);
+            }
+
+            ALWAYS_INLINE void Open(KVirtualAddress addr, int count) {
+                MESOSPHERE_ASSERT(this->IsInRange(addr));
+
+                *this->GetRefCountPointer(addr) += count;
+
+                MESOSPHERE_ABORT_UNLESS(this->GetRefCount(addr) > 0);
+            }
+
+            ALWAYS_INLINE bool Close(KVirtualAddress addr, int count) {
+                MESOSPHERE_ASSERT(this->IsInRange(addr));
+                MESOSPHERE_ABORT_UNLESS(this->GetRefCount(addr) >= count);
+
+                *this->GetRefCountPointer(addr) -= count;
+                return this->GetRefCount(addr) == 0;
+            }
+
+            constexpr ALWAYS_INLINE bool IsInPageTableHeap(KVirtualAddress addr) const {
+                return this->IsInRange(addr);
+            }
+    };
+
+}

libraries/libmesosphere/include/mesosphere/kern_k_port.hpp

@@ -42,7 +42,6 @@ namespace ams::kern {
             bool m_is_light;
         public:
             constexpr KPort() : m_server(), m_client(), m_name(), m_state(State::Invalid), m_is_light() { /* ... */ }
-            virtual ~KPort() { /* ... */ }
 
             static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
 

libraries/libmesosphere/include/mesosphere/kern_k_process.hpp

@@ -22,14 +22,14 @@
 #include <mesosphere/kern_k_thread.hpp>
 #include <mesosphere/kern_k_thread_local_page.hpp>
 #include <mesosphere/kern_k_shared_memory_info.hpp>
-#include <mesosphere/kern_k_beta.hpp>
+#include <mesosphere/kern_k_io_region.hpp>
 #include <mesosphere/kern_k_worker_task.hpp>
 #include <mesosphere/kern_select_page_table.hpp>
 #include <mesosphere/kern_k_condition_variable.hpp>
 #include <mesosphere/kern_k_address_arbiter.hpp>
 #include <mesosphere/kern_k_capabilities.hpp>
 #include <mesosphere/kern_k_wait_object.hpp>
-#include <mesosphere/kern_k_dynamic_slab_heap.hpp>
+#include <mesosphere/kern_k_dynamic_resource_manager.hpp>
 #include <mesosphere/kern_k_page_table_manager.hpp>
 
 namespace ams::kern {
@@ -53,7 +53,7 @@ namespace ams::kern {
             static constexpr size_t AslrAlignment = KernelAslrAlignment;
         private:
             using SharedMemoryInfoList = util::IntrusiveListBaseTraits<KSharedMemoryInfo>::ListType;
-            using BetaList = util::IntrusiveListMemberTraits<&KBeta::m_process_list_node>::ListType;
+            using IoRegionList = util::IntrusiveListMemberTraits<&KIoRegion::m_process_list_node>::ListType;
             using TLPTree = util::IntrusiveRedBlackTreeBaseTraits<KThreadLocalPage>::TreeType<KThreadLocalPage>;
             using TLPIterator = TLPTree::iterator;
         private:
@@ -96,7 +96,7 @@ namespace ams::kern {
             KThread                    *m_exception_thread{};
             ThreadList                  m_thread_list{};
             SharedMemoryInfoList        m_shared_memory_list{};
-            BetaList                    m_beta_list{};
+            IoRegionList                m_io_region_list{};
             bool                        m_is_suspended{};
             bool                        m_is_immortal{};
             bool                        m_is_jit_debug{};
@@ -121,6 +121,9 @@ namespace ams::kern {
             KMemoryBlockSlabManager     m_memory_block_slab_manager{};
             KBlockInfoManager           m_block_info_manager{};
             KPageTableManager           m_page_table_manager{};
+            KMemoryBlockSlabHeap        m_memory_block_heap{};
+            KBlockInfoSlabHeap          m_block_info_heap{};
+            KPageTableSlabHeap          m_page_table_heap{};
         private:
             Result Initialize(const ams::svc::CreateProcessParameter &params);
 
@@ -143,7 +146,6 @@ namespace ams::kern {
             }
         public:
             KProcess() { /* ... */ }
-            virtual ~KProcess() { /* ... */ }
 
             Result Initialize(const ams::svc::CreateProcessParameter &params, const KPageGroup &pg, const u32 *caps, s32 num_caps, KResourceLimit *res_limit, KMemoryManager::Pool pool, bool immortal);
             Result Initialize(const ams::svc::CreateProcessParameter &params, svc::KUserPointer<const u32 *> caps, s32 num_caps, KResourceLimit *res_limit, KMemoryManager::Pool pool);
@@ -273,6 +275,9 @@ namespace ams::kern {
             Result AddSharedMemory(KSharedMemory *shmem, KProcessAddress address, size_t size);
             void RemoveSharedMemory(KSharedMemory *shmem, KProcessAddress address, size_t size);
 
+            void AddIoRegion(KIoRegion *io_region);
+            void RemoveIoRegion(KIoRegion *io_region);
+
             Result CreateThreadLocalRegion(KProcessAddress *out);
             Result DeleteThreadLocalRegion(KProcessAddress addr);
             void *GetThreadLocalRegionPointer(KProcessAddress addr);
@@ -342,14 +347,6 @@ namespace ams::kern {
             void UnpinCurrentThread();
             void UnpinThread(KThread *thread);
 
-            Result SignalToAddress(KProcessAddress address) {
-                return m_cond_var.SignalToAddress(address);
-            }
-
-            Result WaitForAddress(ams::svc::Handle handle, KProcessAddress address, u32 tag) {
-                return m_cond_var.WaitForAddress(handle, address, tag);
-            }
-
             void SignalConditionVariable(uintptr_t cv_key, int32_t count) {
                 return m_cond_var.Signal(cv_key, count);
             }

libraries/libmesosphere/include/mesosphere/kern_k_readable_event.hpp

@@ -28,21 +28,16 @@ namespace ams::kern {
             KEvent *m_parent;
         public:
             constexpr explicit KReadableEvent() : KSynchronizationObject(), m_is_signaled(), m_parent() { MESOSPHERE_ASSERT_THIS(); }
-            virtual ~KReadableEvent() { MESOSPHERE_ASSERT_THIS(); }
 
-            constexpr void Initialize(KEvent *parent) {
+            void Initialize(KEvent *parent);
-                MESOSPHERE_ASSERT_THIS();
-                m_is_signaled  = false;
-                m_parent       = parent;
-            }
 
             constexpr KEvent *GetParent() const { return m_parent; }
 
+            Result Signal();
+            Result Clear();
+
             virtual bool IsSignaled() const override;
             virtual void Destroy() override;
-
-            virtual Result Signal();
-            virtual Result Clear();
             virtual Result Reset();
     };
 

libraries/libmesosphere/include/mesosphere/kern_k_resource_limit.hpp

@@ -34,7 +34,6 @@ namespace ams::kern {
             KLightConditionVariable m_cond_var;
         public:
             constexpr ALWAYS_INLINE KResourceLimit() : m_limit_values(), m_current_values(), m_current_hints(), m_peak_values(), m_lock(), m_waiter_count(), m_cond_var() { /* ... */ }
-            virtual ~KResourceLimit() { /* ... */ }
 
             static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
 
@@ -48,6 +47,8 @@ namespace ams::kern {
 
             Result SetLimitValue(ams::svc::LimitableResource which, s64 value);
 
+            void Add(ams::svc::LimitableResource which, s64 value);
+
             bool Reserve(ams::svc::LimitableResource which, s64 value);
             bool Reserve(ams::svc::LimitableResource which, s64 value, s64 timeout);
             void Release(ams::svc::LimitableResource which, s64 value);

libraries/libmesosphere/include/mesosphere/kern_k_scheduler.hpp

@@ -17,6 +17,7 @@
 #include <mesosphere/kern_select_cpu.hpp>
 #include <mesosphere/kern_k_thread.hpp>
 #include <mesosphere/kern_k_priority_queue.hpp>
+#include <mesosphere/kern_k_interrupt_task_manager.hpp>
 #include <mesosphere/kern_k_scheduler_lock.hpp>
 
 namespace ams::kern {
@@ -39,11 +40,13 @@ namespace ams::kern {
 
             struct SchedulingState {
                 std::atomic<u8> needs_scheduling;
-                bool interrupt_task_thread_runnable;
+                bool interrupt_task_runnable;
                 bool should_count_idle;
                 u64  idle_count;
                 KThread *highest_priority_thread;
                 void *idle_thread_stack;
+                KThread *prev_thread;
+                KInterruptTaskManager *interrupt_task_manager;
             };
         private:
             friend class KScopedSchedulerLock;
@@ -53,28 +56,29 @@ namespace ams::kern {
             SchedulingState m_state;
             bool m_is_active;
             s32 m_core_id;
-            KThread *m_prev_thread;
             s64 m_last_context_switch_time;
             KThread *m_idle_thread;
             std::atomic<KThread *> m_current_thread;
         public:
             constexpr KScheduler()
-                : m_state(), m_is_active(false), m_core_id(0), m_prev_thread(nullptr), m_last_context_switch_time(0), m_idle_thread(nullptr), m_current_thread(nullptr)
+                : m_state(), m_is_active(false), m_core_id(0), m_last_context_switch_time(0), m_idle_thread(nullptr), m_current_thread(nullptr)
             {
-                m_state.needs_scheduling = true;
+                m_state.needs_scheduling        = true;
-                m_state.interrupt_task_thread_runnable = false;
+                m_state.interrupt_task_runnable = false;
-                m_state.should_count_idle = false;
+                m_state.should_count_idle       = false;
-                m_state.idle_count = 0;
+                m_state.idle_count              = 0;
-                m_state.idle_thread_stack = nullptr;
+                m_state.idle_thread_stack       = nullptr;
                 m_state.highest_priority_thread = nullptr;
+                m_state.prev_thread             = nullptr;
+                m_state.interrupt_task_manager  = nullptr;
             }
 
             NOINLINE void Initialize(KThread *idle_thread);
             NOINLINE void Activate();
 
             ALWAYS_INLINE void SetInterruptTaskRunnable() {
-                m_state.interrupt_task_thread_runnable = true;
+                m_state.interrupt_task_runnable = true;
-                m_state.needs_scheduling               = true;
+                m_state.needs_scheduling        = true;
             }
 
             ALWAYS_INLINE void RequestScheduleOnInterrupt() {
@@ -94,7 +98,7 @@ namespace ams::kern {
             }
 
             ALWAYS_INLINE KThread *GetPreviousThread() const {
-                return m_prev_thread;
+                return m_state.prev_thread;
             }
 
             ALWAYS_INLINE KThread *GetSchedulerCurrentThread() const {
@@ -108,8 +112,6 @@ namespace ams::kern {
             /* Static private API. */
             static ALWAYS_INLINE KSchedulerPriorityQueue &GetPriorityQueue() { return s_priority_queue; }
             static NOINLINE u64 UpdateHighestPriorityThreadsImpl();
-
-            static NOINLINE void InterruptTaskThreadToRunnable();
         public:
             /* Static public API. */
             static ALWAYS_INLINE bool CanSchedule() { return GetCurrentThread().GetDisableDispatchCount() == 0; }
@@ -124,13 +126,14 @@ namespace ams::kern {
                 GetCurrentThread().DisableDispatch();
             }
 
-            static NOINLINE void EnableScheduling(u64 cores_needing_scheduling) {
+            static ALWAYS_INLINE void EnableScheduling(u64 cores_needing_scheduling) {
                 MESOSPHERE_ASSERT(GetCurrentThread().GetDisableDispatchCount() >= 1);
 
+                GetCurrentScheduler().RescheduleOtherCores(cores_needing_scheduling);
+
                 if (GetCurrentThread().GetDisableDispatchCount() > 1) {
                     GetCurrentThread().EnableDispatch();
                 } else {
-                    GetCurrentScheduler().RescheduleOtherCores(cores_needing_scheduling);
                     GetCurrentScheduler().RescheduleCurrentCore();
                 }
             }
@@ -176,14 +179,23 @@ namespace ams::kern {
 
             ALWAYS_INLINE void RescheduleCurrentCore() {
                 MESOSPHERE_ASSERT(GetCurrentThread().GetDisableDispatchCount() == 1);
-                {
-                    /* Disable interrupts, and then context switch. */
-                    KScopedInterruptDisable intr_disable;
-                    ON_SCOPE_EXIT { GetCurrentThread().EnableDispatch(); };
 
-                    if (m_state.needs_scheduling.load()) {
+                GetCurrentThread().EnableDispatch();
-                        Schedule();
+
-                    }
+                if (m_state.needs_scheduling.load()) {
+                    /* Disable interrupts, and then check again if rescheduling is needed. */
+                    KScopedInterruptDisable intr_disable;
+
+                    GetCurrentScheduler().RescheduleCurrentCoreImpl();
+                }
+            }
+
+            ALWAYS_INLINE void RescheduleCurrentCoreImpl() {
+                /* Check that scheduling is needed. */
+                if (AMS_LIKELY(m_state.needs_scheduling.load())) {
+                    GetCurrentThread().DisableDispatch();
+                    this->Schedule();
+                    GetCurrentThread().EnableDispatch();
                 }
             }
 
@@ -199,10 +211,12 @@ namespace ams::kern {
     };
 
     consteval bool KScheduler::ValidateAssemblyOffsets() {
-        static_assert(__builtin_offsetof(KScheduler, m_state.needs_scheduling)               == KSCHEDULER_NEEDS_SCHEDULING);
+        static_assert(__builtin_offsetof(KScheduler, m_state.needs_scheduling)        == KSCHEDULER_NEEDS_SCHEDULING);
-        static_assert(__builtin_offsetof(KScheduler, m_state.interrupt_task_thread_runnable) == KSCHEDULER_INTERRUPT_TASK_THREAD_RUNNABLE);
+        static_assert(__builtin_offsetof(KScheduler, m_state.interrupt_task_runnable) == KSCHEDULER_INTERRUPT_TASK_RUNNABLE);
-        static_assert(__builtin_offsetof(KScheduler, m_state.highest_priority_thread)        == KSCHEDULER_HIGHEST_PRIORITY_THREAD);
+        static_assert(__builtin_offsetof(KScheduler, m_state.highest_priority_thread) == KSCHEDULER_HIGHEST_PRIORITY_THREAD);
-        static_assert(__builtin_offsetof(KScheduler, m_state.idle_thread_stack)              == KSCHEDULER_IDLE_THREAD_STACK);
+        static_assert(__builtin_offsetof(KScheduler, m_state.idle_thread_stack)       == KSCHEDULER_IDLE_THREAD_STACK);
+        static_assert(__builtin_offsetof(KScheduler, m_state.prev_thread)             == KSCHEDULER_PREVIOUS_THREAD);
+        static_assert(__builtin_offsetof(KScheduler, m_state.interrupt_task_manager)  == KSCHEDULER_INTERRUPT_TASK_MANAGER);
 
         return true;
     }

libraries/libmesosphere/include/mesosphere/kern_k_scheduler_lock.hpp

@@ -45,7 +45,7 @@ namespace ams::kern {
                 return m_owner_thread == GetCurrentThreadPointer();
             }
 
-            void Lock() {
+            NOINLINE void Lock() {
                 MESOSPHERE_ASSERT_THIS();
 
                 if (this->IsLockedByCurrentThread()) {
@@ -67,7 +67,7 @@ namespace ams::kern {
                 }
             }
 
-            void Unlock() {
+            NOINLINE void Unlock() {
                 MESOSPHERE_ASSERT_THIS();
                 MESOSPHERE_ASSERT(this->IsLockedByCurrentThread());
                 MESOSPHERE_ASSERT(m_lock_count > 0);

libraries/libmesosphere/include/mesosphere/kern_k_server_port.hpp

@@ -35,7 +35,6 @@ namespace ams::kern {
             KPort *m_parent;
         public:
             constexpr KServerPort() : m_session_list(), m_light_session_list(), m_parent() { /* ... */ }
-            virtual ~KServerPort() { /* ... */ }
 
             void Initialize(KPort *parent);
             void EnqueueSession(KServerSession *session);

libraries/libmesosphere/include/mesosphere/kern_k_server_session.hpp

@@ -34,7 +34,6 @@ namespace ams::kern {
             KLightLock m_lock;
         public:
             constexpr KServerSession() : m_parent(), m_request_list(), m_current_request(), m_lock() { /* ... */ }
-            virtual ~KServerSession() { /* ... */ }
 
             virtual void Destroy() override;
 

libraries/libmesosphere/include/mesosphere/kern_k_session.hpp

@@ -25,7 +25,7 @@ namespace ams::kern {
     class KClientPort;
     class KProcess;
 
-    class KSession final : public KAutoObjectWithSlabHeapAndContainer<KSession, KAutoObjectWithList> {
+    class KSession final : public KAutoObjectWithSlabHeapAndContainer<KSession, KAutoObjectWithList, true> {
         MESOSPHERE_AUTOOBJECT_TRAITS(KSession, KAutoObject);
         private:
             enum class State : u8 {
@@ -57,8 +57,6 @@ namespace ams::kern {
                 /* ... */
             }
 
-            virtual ~KSession() { /* ... */ }
-
             void Initialize(KClientPort *client_port, uintptr_t name);
             virtual void Finalize() override;
 

libraries/libmesosphere/include/mesosphere/kern_k_session_request.hpp

@@ -17,14 +17,14 @@
 #include <mesosphere/kern_common.hpp>
 #include <mesosphere/kern_k_auto_object.hpp>
 #include <mesosphere/kern_slab_helpers.hpp>
-#include <mesosphere/kern_k_writable_event.hpp>
+#include <mesosphere/kern_k_event.hpp>
 #include <mesosphere/kern_k_thread.hpp>
 #include <mesosphere/kern_k_process.hpp>
 #include <mesosphere/kern_k_memory_block.hpp>
 
 namespace ams::kern {
 
-    class KSessionRequest final : public KSlabAllocated<KSessionRequest>, public KAutoObject, public util::IntrusiveListBaseNode<KSessionRequest> {
+    class KSessionRequest final : public KSlabAllocated<KSessionRequest, true>, public KAutoObject, public util::IntrusiveListBaseNode<KSessionRequest> {
         MESOSPHERE_AUTOOBJECT_TRAITS(KSessionRequest, KAutoObject);
         public:
             class SessionMappings {
@@ -126,12 +126,11 @@ namespace ams::kern {
             SessionMappings m_mappings;
             KThread *m_thread;
             KProcess *m_server;
-            KWritableEvent *m_event;
+            KEvent *m_event;
             uintptr_t m_address;
             size_t m_size;
         public:
             constexpr KSessionRequest() : m_mappings(), m_thread(), m_server(), m_event(), m_address(), m_size() { /* ... */ }
-            virtual ~KSessionRequest() { /* ... */ }
 
             static KSessionRequest *Create() {
                 KSessionRequest *req = KSessionRequest::Allocate();
@@ -141,12 +140,20 @@ namespace ams::kern {
                 return req;
             }
 
+            static KSessionRequest *CreateFromUnusedSlabMemory() {
+                KSessionRequest *req = KSessionRequest::AllocateFromUnusedSlabMemory();
+                if (req != nullptr) {
+                    KAutoObject::Create(req);
+                }
+                return req;
+            }
+
             virtual void Destroy() override {
                 this->Finalize();
                 KSessionRequest::Free(this);
             }
 
-            void Initialize(KWritableEvent *event, uintptr_t address, size_t size) {
+            void Initialize(KEvent *event, uintptr_t address, size_t size) {
                 m_mappings.Initialize();
 
                 m_thread  = std::addressof(GetCurrentThread());
@@ -177,7 +184,7 @@ namespace ams::kern {
             static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
 
             constexpr ALWAYS_INLINE KThread *GetThread() const { return m_thread; }
-            constexpr ALWAYS_INLINE KWritableEvent *GetEvent() const { return m_event; }
+            constexpr ALWAYS_INLINE KEvent *GetEvent() const { return m_event; }
             constexpr ALWAYS_INLINE uintptr_t GetAddress() const { return m_address; }
             constexpr ALWAYS_INLINE size_t GetSize() const { return m_size; }
             constexpr ALWAYS_INLINE KProcess *GetServerProcess() const { return m_server; }

libraries/libmesosphere/include/mesosphere/kern_k_shared_memory.hpp

@@ -35,14 +35,12 @@ namespace ams::kern {
             bool m_is_initialized;
         public:
             explicit KSharedMemory()
-                : m_page_group(std::addressof(Kernel::GetBlockInfoManager())), m_resource_limit(nullptr), m_owner_process_id(std::numeric_limits<u64>::max()),
+                : m_page_group(std::addressof(Kernel::GetSystemBlockInfoManager())), m_resource_limit(nullptr), m_owner_process_id(std::numeric_limits<u64>::max()),
                   m_owner_perm(ams::svc::MemoryPermission_None), m_remote_perm(ams::svc::MemoryPermission_None), m_is_initialized(false)
             {
                 /* ... */
             }
 
-            virtual ~KSharedMemory() { /* ... */ }
-
             Result Initialize(KProcess *owner, size_t size, ams::svc::MemoryPermission own_perm, ams::svc::MemoryPermission rem_perm);
             virtual void Finalize() override;
 

libraries/libmesosphere/include/mesosphere/kern_k_slab_heap.hpp

@@ -16,11 +16,13 @@
 #pragma once
 #include <mesosphere/kern_common.hpp>
 #include <mesosphere/kern_k_typed_address.hpp>
+#include <mesosphere/kern_k_memory_layout.hpp>
 
 #if defined(ATMOSPHERE_ARCH_ARM64)
 
     #include <mesosphere/arch/arm64/kern_k_slab_heap_impl.hpp>
     namespace ams::kern {
+        using ams::kern::arch::arm64::IsSlabAtomicValid;
         using ams::kern::arch::arm64::AllocateFromSlabAtomic;
         using ams::kern::arch::arm64::FreeToSlabAtomic;
     }
@@ -44,78 +46,73 @@ namespace ams::kern {
                     Node *next;
                 };
             private:
-                Node * m_head;
+                Node *m_head{nullptr};
-                size_t m_obj_size;
             public:
-                constexpr KSlabHeapImpl() : m_head(nullptr), m_obj_size(0) { MESOSPHERE_ASSERT_THIS(); }
+                constexpr KSlabHeapImpl() = default;
 
-                void Initialize(size_t size) {
+                void Initialize() {
-                    MESOSPHERE_INIT_ABORT_UNLESS(m_head == nullptr);
+                    MESOSPHERE_ABORT_UNLESS(m_head == nullptr);
-                    m_obj_size = size;
+                    MESOSPHERE_ABORT_UNLESS(IsSlabAtomicValid());
                 }
 
-                Node *GetHead() const {
+                ALWAYS_INLINE Node *GetHead() const {
                     return m_head;
                 }
 
-                size_t GetObjectSize() const {
+                ALWAYS_INLINE void *Allocate() {
-                    return m_obj_size;
-                }
-
-                void *Allocate() {
-                    MESOSPHERE_ASSERT_THIS();
-
                     return AllocateFromSlabAtomic(std::addressof(m_head));
                 }
 
-                void Free(void *obj) {
+                ALWAYS_INLINE void Free(void *obj) {
-                    MESOSPHERE_ASSERT_THIS();
+                    return FreeToSlabAtomic(std::addressof(m_head), static_cast<Node *>(obj));
-
-                    Node *node = reinterpret_cast<Node *>(obj);
-
-                    return FreeToSlabAtomic(std::addressof(m_head), node);
                 }
         };
 
     }
 
-    class KSlabHeapBase {
+    template<bool SupportDynamicExpansion>
+    class KSlabHeapBase : protected impl::KSlabHeapImpl {
         NON_COPYABLE(KSlabHeapBase);
         NON_MOVEABLE(KSlabHeapBase);
         private:
-            using Impl = impl::KSlabHeapImpl;
+            size_t m_obj_size{};
+            uintptr_t m_peak{};
+            uintptr_t m_start{};
+            uintptr_t m_end{};
         private:
-            Impl m_impl;
+            ALWAYS_INLINE void UpdatePeakImpl(uintptr_t obj) {
-            uintptr_t m_peak;
+                static_assert(std::atomic_ref<uintptr_t>::is_always_lock_free);
-            uintptr_t m_start;
+                std::atomic_ref<uintptr_t> peak_ref(m_peak);
-            uintptr_t m_end;
+
-        private:
+                const uintptr_t alloc_peak = obj + this->GetObjectSize();
-            ALWAYS_INLINE Impl *GetImpl() {
+                uintptr_t cur_peak = m_peak;
-                return std::addressof(m_impl);
+                do {
-            }
+                    if (alloc_peak <= cur_peak) {
-            ALWAYS_INLINE const Impl *GetImpl() const {
+                        break;
-                return std::addressof(m_impl);
+                    }
+                } while (!peak_ref.compare_exchange_strong(cur_peak, alloc_peak));
             }
         public:
-            constexpr KSlabHeapBase() : m_impl(), m_peak(0), m_start(0), m_end(0) { MESOSPHERE_ASSERT_THIS(); }
+            constexpr KSlabHeapBase() = default;
 
             ALWAYS_INLINE bool Contains(uintptr_t address) const {
                 return m_start <= address && address < m_end;
             }
 
-            void InitializeImpl(size_t obj_size, void *memory, size_t memory_size) {
+            void Initialize(size_t obj_size, void *memory, size_t memory_size) {
-                MESOSPHERE_ASSERT_THIS();
-
                 /* Ensure we don't initialize a slab using null memory. */
                 MESOSPHERE_ABORT_UNLESS(memory != nullptr);
 
+                /* Set our object size. */
+                m_obj_size = obj_size;
+
                 /* Initialize the base allocator. */
-                this->GetImpl()->Initialize(obj_size);
+                KSlabHeapImpl::Initialize();
 
                 /* Set our tracking variables. */
                 const size_t num_obj = (memory_size / obj_size);
                 m_start = reinterpret_cast<uintptr_t>(memory);
-                m_end = m_start + num_obj * obj_size;
+                m_end  = m_start + num_obj * obj_size;
                 m_peak = m_start;
 
                 /* Free the objects. */
@@ -123,75 +120,91 @@ namespace ams::kern {
 
                 for (size_t i = 0; i < num_obj; i++) {
                     cur -= obj_size;
-                    this->GetImpl()->Free(cur);
+                    KSlabHeapImpl::Free(cur);
                 }
             }
 
-            size_t GetSlabHeapSize() const {
+            ALWAYS_INLINE size_t GetSlabHeapSize() const {
                 return (m_end - m_start) / this->GetObjectSize();
             }
 
-            size_t GetObjectSize() const {
+            ALWAYS_INLINE size_t GetObjectSize() const {
-                return this->GetImpl()->GetObjectSize();
+                return m_obj_size;
             }
 
-            void *AllocateImpl() {
+            ALWAYS_INLINE void *Allocate() {
-                MESOSPHERE_ASSERT_THIS();
+                void *obj = KSlabHeapImpl::Allocate();
-
-                void *obj = this->GetImpl()->Allocate();
 
                 /* Track the allocated peak. */
                 #if defined(MESOSPHERE_BUILD_FOR_DEBUGGING)
                 if (AMS_LIKELY(obj != nullptr)) {
-                    static_assert(std::atomic_ref<uintptr_t>::is_always_lock_free);
+                    if constexpr (SupportDynamicExpansion) {
-                    std::atomic_ref<uintptr_t> peak_ref(m_peak);
+                        if (this->Contains(reinterpret_cast<uintptr_t>(obj))) {
-
+                            this->UpdatePeakImpl(reinterpret_cast<uintptr_t>(obj));
-                    const uintptr_t alloc_peak = reinterpret_cast<uintptr_t>(obj) + this->GetObjectSize();
+                        } else {
-                    uintptr_t cur_peak = m_peak;
+                            this->UpdatePeakImpl(reinterpret_cast<uintptr_t>(m_end) - this->GetObjectSize());
-                    do {
-                        if (alloc_peak <= cur_peak) {
-                            break;
                         }
-                    } while (!peak_ref.compare_exchange_strong(cur_peak, alloc_peak));
+                    } else {
+                        this->UpdatePeakImpl(reinterpret_cast<uintptr_t>(obj));
+                    }
                 }
                 #endif
 
                 return obj;
             }
 
-            void FreeImpl(void *obj) {
+            ALWAYS_INLINE void Free(void *obj) {
-                MESOSPHERE_ASSERT_THIS();
-
                 /* Don't allow freeing an object that wasn't allocated from this heap. */
-                MESOSPHERE_ABORT_UNLESS(this->Contains(reinterpret_cast<uintptr_t>(obj)));
+                const bool contained = this->Contains(reinterpret_cast<uintptr_t>(obj));
+                if constexpr (SupportDynamicExpansion) {
+                    const bool is_slab = KMemoryLayout::GetSlabRegion().Contains(reinterpret_cast<uintptr_t>(obj));
+                    MESOSPHERE_ABORT_UNLESS(contained || is_slab);
+                } else {
+                    MESOSPHERE_ABORT_UNLESS(contained);
+                }
 
-                this->GetImpl()->Free(obj);
+                KSlabHeapImpl::Free(obj);
             }
 
-            size_t GetObjectIndexImpl(const void *obj) const {
+            ALWAYS_INLINE size_t GetObjectIndex(const void *obj) const {
+                if constexpr (SupportDynamicExpansion) {
+                    if (!this->Contains(reinterpret_cast<uintptr_t>(obj))) {
+                        return std::numeric_limits<size_t>::max();
+                    }
+                }
+
                 return (reinterpret_cast<uintptr_t>(obj) - m_start) / this->GetObjectSize();
             }
 
-            size_t GetPeakIndex() const {
+            ALWAYS_INLINE size_t GetPeakIndex() const {
-                return this->GetObjectIndexImpl(reinterpret_cast<const void *>(m_peak));
+                return this->GetObjectIndex(reinterpret_cast<const void *>(m_peak));
             }
 
-            uintptr_t GetSlabHeapAddress() const {
+            ALWAYS_INLINE uintptr_t GetSlabHeapAddress() const {
                 return m_start;
             }
 
-            size_t GetNumRemaining() const {
+            ALWAYS_INLINE size_t GetNumRemaining() const {
                 size_t remaining = 0;
 
                 /* Only calculate the number of remaining objects under debug configuration. */
                 #if defined(MESOSPHERE_BUILD_FOR_DEBUGGING)
                 while (true) {
-                    auto *cur = this->GetImpl()->GetHead();
+                    auto *cur = this->GetHead();
                     remaining = 0;
 
-                    while (this->Contains(reinterpret_cast<uintptr_t>(cur))) {
+                    if constexpr (SupportDynamicExpansion) {
-                        ++remaining;
+                        const auto &slab_region = KMemoryLayout::GetSlabRegion();
-                        cur = cur->next;
+
+                        while (this->Contains(reinterpret_cast<uintptr_t>(cur)) || slab_region.Contains(reinterpret_cast<uintptr_t>(cur))) {
+                            ++remaining;
+                            cur = cur->next;
+                        }
+                    } else {
+                        while (this->Contains(reinterpret_cast<uintptr_t>(cur))) {
+                            ++remaining;
+                            cur = cur->next;
+                        }
                     }
 
                     if (cur == nullptr) {
@@ -204,29 +217,31 @@ namespace ams::kern {
             }
     };
 
-    template<typename T>
+    template<typename T, bool SupportDynamicExpansion>
-    class KSlabHeap : public KSlabHeapBase {
+    class KSlabHeap : public KSlabHeapBase<SupportDynamicExpansion> {
+        private:
+            using BaseHeap = KSlabHeapBase<SupportDynamicExpansion>;
         public:
-            constexpr KSlabHeap() : KSlabHeapBase() { /* ... */ }
+            constexpr KSlabHeap() = default;
 
             void Initialize(void *memory, size_t memory_size) {
-                this->InitializeImpl(sizeof(T), memory, memory_size);
+                BaseHeap::Initialize(sizeof(T), memory, memory_size);
             }
 
-            T *Allocate() {
+            ALWAYS_INLINE T *Allocate() {
-                T *obj = reinterpret_cast<T *>(this->AllocateImpl());
+                T *obj = static_cast<T *>(BaseHeap::Allocate());
                 if (AMS_LIKELY(obj != nullptr)) {
                     std::construct_at(obj);
                 }
                 return obj;
             }
 
-            void Free(T *obj) {
+            ALWAYS_INLINE void Free(T *obj) {
-                this->FreeImpl(obj);
+                BaseHeap::Free(obj);
             }
 
-            size_t GetObjectIndex(const T *obj) const {
+            ALWAYS_INLINE size_t GetObjectIndex(const T *obj) const {
-                return this->GetObjectIndexImpl(obj);
+                return BaseHeap::GetObjectIndex(obj);
             }
     };
 

libraries/libmesosphere/include/mesosphere/kern_k_synchronization_object.hpp

@@ -33,7 +33,6 @@ namespace ams::kern {
             ThreadListNode *m_thread_list_tail;
         protected:
             constexpr ALWAYS_INLINE explicit KSynchronizationObject() : KAutoObjectWithList(), m_thread_list_head(), m_thread_list_tail() { MESOSPHERE_ASSERT_THIS(); }
-            virtual ~KSynchronizationObject() { MESOSPHERE_ASSERT_THIS(); }
 
             virtual void OnFinalizeSynchronizationObject() { MESOSPHERE_ASSERT_THIS(); }
 
@@ -46,7 +45,39 @@ namespace ams::kern {
         public:
             virtual void Finalize() override;
             virtual bool IsSignaled() const { AMS_INFINITE_LOOP(); }
-            virtual void DumpWaiters();
+
+            void DumpWaiters();
+
+            ALWAYS_INLINE void LinkNode(ThreadListNode *node) {
+                /* Link the node to the list. */
+                if (m_thread_list_tail == nullptr) {
+                    m_thread_list_head = node;
+                } else {
+                    m_thread_list_tail->next = node;
+                }
+
+                m_thread_list_tail = node;
+            }
+
+            ALWAYS_INLINE void UnlinkNode(ThreadListNode *node) {
+                /* Unlink the node from the list. */
+                ThreadListNode *prev_ptr = reinterpret_cast<ThreadListNode *>(std::addressof(m_thread_list_head));
+                ThreadListNode *prev_val = nullptr;
+                ThreadListNode *prev, *tail_prev;
+
+                do {
+                    prev      = prev_ptr;
+                    prev_ptr  = prev_ptr->next;
+                    tail_prev = prev_val;
+                    prev_val  = prev_ptr;
+                } while (prev_ptr != node);
+
+                if (m_thread_list_tail == node) {
+                    m_thread_list_tail = tail_prev;
+                }
+
+                prev->next = node->next;
+            }
     };
 
 }

libraries/libmesosphere/include/mesosphere/kern_k_target_system.hpp

@@ -23,12 +23,13 @@ namespace ams::kern {
         private:
             friend class KSystemControl;
         private:
-            static inline bool s_is_debug_mode;
+            static inline constinit bool s_is_debug_mode;
-            static inline bool s_enable_debug_logging;
+            static inline constinit bool s_enable_debug_logging;
-            static inline bool s_enable_user_exception_handlers;
+            static inline constinit bool s_enable_user_exception_handlers;
-            static inline bool s_enable_debug_memory_fill;
+            static inline constinit bool s_enable_debug_memory_fill;
-            static inline bool s_enable_user_pmu_access;
+            static inline constinit bool s_enable_user_pmu_access;
-            static inline bool s_enable_kernel_debugging;
+            static inline constinit bool s_enable_kernel_debugging;
+            static inline constinit bool s_enable_dynamic_resource_limits;
         private:
             static ALWAYS_INLINE void SetIsDebugMode(bool en) { s_is_debug_mode = en; }
             static ALWAYS_INLINE void EnableDebugLogging(bool en) { s_enable_debug_logging = en; }
@@ -36,6 +37,7 @@ namespace ams::kern {
             static ALWAYS_INLINE void EnableDebugMemoryFill(bool en) { s_enable_debug_memory_fill = en; }
             static ALWAYS_INLINE void EnableUserPmuAccess(bool en) { s_enable_user_pmu_access = en; }
             static ALWAYS_INLINE void EnableKernelDebugging(bool en) { s_enable_kernel_debugging = en; }
+            static ALWAYS_INLINE void EnableDynamicResourceLimits(bool en) { s_enable_dynamic_resource_limits = en; }
         public:
             static ALWAYS_INLINE bool IsDebugMode() { return s_is_debug_mode; }
             static ALWAYS_INLINE bool IsDebugLoggingEnabled() { return s_enable_debug_logging; }
@@ -43,6 +45,7 @@ namespace ams::kern {
             static ALWAYS_INLINE bool IsDebugMemoryFillEnabled() { return s_enable_debug_memory_fill; }
             static ALWAYS_INLINE bool IsUserPmuAccessEnabled() { return s_enable_user_pmu_access; }
             static ALWAYS_INLINE bool IsKernelDebuggingEnabled() { return s_enable_kernel_debugging; }
+            static ALWAYS_INLINE bool IsDynamicResourceLimitsEnabled() { return s_enable_dynamic_resource_limits; }
     };
 
 }

libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp

@@ -37,6 +37,7 @@ namespace ams::kern {
             friend class KProcess;
             friend class KConditionVariable;
             friend class KAddressArbiter;
+            friend class KThreadQueue;
         public:
             static constexpr s32 MainThreadPriority = 1;
             static constexpr s32 IdleThreadPriority = 64;
@@ -191,7 +192,6 @@ namespace ams::kern {
             KAffinityMask                   m_physical_affinity_mask{};
             u64                             m_thread_id{};
             std::atomic<s64>                m_cpu_time{};
-            KSynchronizationObject         *m_synced_object{};
             KProcessAddress                 m_address_key{};
             KProcess                       *m_parent{};
             void                           *m_kernel_stack_top{};
@@ -204,9 +204,7 @@ namespace ams::kern {
             s64                             m_last_scheduled_tick{};
             QueueEntry                      m_per_core_priority_queue_entry[cpu::NumCores]{};
             KLightLock                     *m_waiting_lock{};
-
+            KThreadQueue                   *m_wait_queue{};
-            KThreadQueue                   *m_sleeping_queue{};
-
             WaiterList                      m_waiter_list{};
             WaiterList                      m_pinned_waiter_list{};
             KThread                        *m_lock_owner{};
@@ -215,6 +213,7 @@ namespace ams::kern {
             u32                             m_address_key_value{};
             u32                             m_suspend_request_flags{};
             u32                             m_suspend_allowed_flags{};
+            s32                             m_synced_index{};
             Result                          m_wait_result;
             Result                          m_debug_exception_result;
             s32                             m_base_priority{};
@@ -239,10 +238,7 @@ namespace ams::kern {
         public:
             constexpr KThread() : m_wait_result(svc::ResultNoSynchronizationObject()), m_debug_exception_result(ResultSuccess()) { /* ... */ }
 
-            virtual ~KThread() { /* ... */ }
-
             Result Initialize(KThreadFunction func, uintptr_t arg, void *kern_stack_top, KProcessAddress user_stack_top, s32 prio, s32 virt_core, KProcess *owner, ThreadType type);
-
         private:
             static Result InitializeThread(KThread *thread, KThreadFunction func, uintptr_t arg, KProcessAddress user_stack_top, s32 prio, s32 virt_core, KProcess *owner, ThreadType type);
         public:
@@ -377,6 +373,8 @@ namespace ams::kern {
             void FinishTermination();
 
             void IncreaseBasePriority(s32 priority);
+
+            NOINLINE void SetState(ThreadState state);
         public:
             constexpr u64 GetThreadId() const { return m_thread_id; }
 
@@ -393,7 +391,6 @@ namespace ams::kern {
 
             constexpr ThreadState GetState() const { return static_cast<ThreadState>(m_thread_state & ThreadState_Mask); }
             constexpr ThreadState GetRawState() const { return m_thread_state; }
-            NOINLINE void SetState(ThreadState state);
 
             NOINLINE KThreadContext *GetContextForSchedulerLoop();
 
@@ -445,8 +442,6 @@ namespace ams::kern {
             constexpr QueueEntry &GetPriorityQueueEntry(s32 core) { return m_per_core_priority_queue_entry[core]; }
             constexpr const QueueEntry &GetPriorityQueueEntry(s32 core) const { return m_per_core_priority_queue_entry[core]; }
 
-            constexpr void SetSleepingQueue(KThreadQueue *q) { m_sleeping_queue = q; }
-
             constexpr ConditionVariableThreadTree *GetConditionVariableTree() const { return m_condvar_tree; }
 
             constexpr s32 GetNumKernelWaiters() const { return m_num_kernel_waiters; }
@@ -463,29 +458,22 @@ namespace ams::kern {
             constexpr void SetLockOwner(KThread *owner) { m_lock_owner = owner; }
             constexpr KThread *GetLockOwner() const { return m_lock_owner; }
 
-            constexpr void SetSyncedObject(KSynchronizationObject *obj, Result wait_res) {
+            constexpr void ClearWaitQueue() { m_wait_queue = nullptr; }
-                MESOSPHERE_ASSERT_THIS();
 
-                m_synced_object = obj;
+            void BeginWait(KThreadQueue *queue);
-                m_wait_result = wait_res;
+            void NotifyAvailable(KSynchronizationObject *signaled_object, Result wait_result);
-            }
+            void EndWait(Result wait_result);
+            void CancelWait(Result wait_result, bool cancel_timer_task);
 
-            constexpr Result GetWaitResult(KSynchronizationObject **out) const {
+            constexpr void SetSyncedIndex(s32 index) { m_synced_index = index; }
-                MESOSPHERE_ASSERT_THIS();
+            constexpr s32 GetSyncedIndex() const { return m_synced_index; }
 
-                *out = m_synced_object;
+            constexpr void SetWaitResult(Result wait_res) { m_wait_result = wait_res; }
-                return m_wait_result;
+            constexpr Result GetWaitResult() const { return m_wait_result; }
-            }
 
-            constexpr void SetDebugExceptionResult(Result result) {
+            constexpr void SetDebugExceptionResult(Result result) { m_debug_exception_result = result; }
-                MESOSPHERE_ASSERT_THIS();
-                m_debug_exception_result = result;
-            }
 
-            constexpr Result GetDebugExceptionResult() const {
+            constexpr Result GetDebugExceptionResult() const { return m_debug_exception_result; }
-                MESOSPHERE_ASSERT_THIS();
-                return m_debug_exception_result;
-            }
 
             void WaitCancel();
 
@@ -588,8 +576,6 @@ namespace ams::kern {
                 }
             }
 
-            void Wakeup();
-
             void SetBasePriority(s32 priority);
             Result SetPriorityToIdle();
 

libraries/libmesosphere/include/mesosphere/kern_k_thread_queue.hpp

@@ -16,69 +16,28 @@
 #pragma once
 #include <mesosphere/kern_common.hpp>
 #include <mesosphere/kern_k_thread.hpp>
+#include <mesosphere/kern_select_hardware_timer.hpp>
 
 namespace ams::kern {
 
     class KThreadQueue {
         private:
-            KThread::WaiterList m_wait_list;
+            KHardwareTimer *m_hardware_timer;
         public:
-            constexpr ALWAYS_INLINE KThreadQueue() : m_wait_list() { /* ... */ }
+            constexpr ALWAYS_INLINE KThreadQueue() : m_hardware_timer(nullptr) { /* ... */ }
 
-            bool IsEmpty() const { return m_wait_list.empty(); }
+            constexpr void SetHardwareTimer(KHardwareTimer *timer) { m_hardware_timer = timer; }
 
-            KThread::WaiterList::iterator begin() { return m_wait_list.begin(); }
+            virtual void NotifyAvailable(KThread *waiting_thread, KSynchronizationObject *signaled_object, Result wait_result);
-            KThread::WaiterList::iterator end() { return m_wait_list.end(); }
+            virtual void EndWait(KThread *waiting_thread, Result wait_result);
+            virtual void CancelWait(KThread *waiting_thread, Result wait_result, bool cancel_timer_task);
+    };
 
-            bool SleepThread(KThread *t) {
+    class KThreadQueueWithoutEndWait : public KThreadQueue {
-                KScopedSchedulerLock sl;
+        public:
+            constexpr ALWAYS_INLINE KThreadQueueWithoutEndWait() : KThreadQueue() { /* ... */ }
 
-                /* If the thread needs terminating, don't enqueue it. */
+            virtual void EndWait(KThread *waiting_thread, Result wait_result) override final;
-                if (t->IsTerminationRequested()) {
-                    return false;
-                }
-
-                /* Set the thread's queue and mark it as waiting. */
-                t->SetSleepingQueue(this);
-                t->SetState(KThread::ThreadState_Waiting);
-
-                /* Add the thread to the queue. */
-                m_wait_list.push_back(*t);
-
-                return true;
-            }
-
-            void WakeupThread(KThread *t) {
-                KScopedSchedulerLock sl;
-
-                /* Remove the thread from the queue. */
-                m_wait_list.erase(m_wait_list.iterator_to(*t));
-
-                /* Mark the thread as no longer sleeping. */
-                t->SetState(KThread::ThreadState_Runnable);
-                t->SetSleepingQueue(nullptr);
-            }
-
-            KThread *WakeupFrontThread() {
-                KScopedSchedulerLock sl;
-
-                if (m_wait_list.empty()) {
-                    return nullptr;
-                } else {
-                    /* Remove the thread from the queue. */
-                    auto it = m_wait_list.begin();
-                    KThread *thread = std::addressof(*it);
-                    m_wait_list.erase(it);
-
-                    MESOSPHERE_ASSERT(thread->GetState() == KThread::ThreadState_Waiting);
-
-                    /* Mark the thread as no longer sleeping. */
-                    thread->SetState(KThread::ThreadState_Runnable);
-                    thread->SetSleepingQueue(nullptr);
-
-                    return thread;
-                }
-            }
     };
 
 }

libraries/libmesosphere/include/mesosphere/kern_k_transfer_memory.hpp

@@ -35,8 +35,6 @@ namespace ams::kern {
                 /* ... */
             }
 
-            virtual ~KTransferMemory() { /* ... */ }
-
             Result Initialize(KProcessAddress addr, size_t size, ams::svc::MemoryPermission own_perm);
             virtual void Finalize() override;
 

libraries/libmesosphere/include/mesosphere/kern_k_unused_slab_memory.hpp

@@ -13,28 +13,15 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
-#include <mesosphere.hpp>
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_k_typed_address.hpp>
 
 namespace ams::kern {
 
-    void KWritableEvent::Initialize(KEvent *p) {
+    /* Utilities to allocate/free memory from the "unused" gaps between slab heaps. */
-        /* Set parent, open a reference to the readable event. */
+    /* See KTargetSystem::IsDynamicResourceLimitsEnabled() usage for more context. */
-        m_parent = p;
+    KVirtualAddress AllocateUnusedSlabMemory(size_t size, size_t alignment);
-        m_parent->GetReadableEvent().Open();
+    void FreeUnusedSlabMemory(KVirtualAddress address, size_t size);
-    }
-
-    Result KWritableEvent::Signal() {
-        return m_parent->GetReadableEvent().Signal();
-    }
-
-    Result KWritableEvent::Clear() {
-        return m_parent->GetReadableEvent().Clear();
-    }
-
-    void KWritableEvent::Destroy() {
-        /* Close our references. */
-        m_parent->GetReadableEvent().Close();
-        m_parent->Close();
-    }
 
 }

libraries/libmesosphere/include/mesosphere/kern_k_wait_object.hpp

@@ -20,14 +20,13 @@
 
 namespace ams::kern {
 
-    class KWaitObject : public KTimerTask {
+    class KWaitObject {
         private:
             KThread::WaiterList m_wait_list;
-            bool  m_timer_used;
+            KThread *m_next_thread;
         public:
-            constexpr KWaitObject() : m_wait_list(), m_timer_used() { /* ... */ }
+            constexpr KWaitObject() : m_wait_list(), m_next_thread() { /* ... */ }
 
-            virtual void OnTimer() override;
             Result Synchronize(s64 timeout);
     };
 

libraries/libmesosphere/include/mesosphere/kern_k_worker_task_manager.hpp

@@ -32,9 +32,7 @@ namespace ams::kern {
         private:
             KWorkerTask *m_head_task;
             KWorkerTask *m_tail_task;
-            KThread *m_thread;
+            KThread *m_waiting_thread;
-            WorkerType m_type;
-            bool m_active;
         private:
             static void ThreadFunction(uintptr_t arg);
             void ThreadFunctionImpl();
@@ -42,9 +40,9 @@ namespace ams::kern {
             KWorkerTask *GetTask();
             void AddTask(KWorkerTask *task);
         public:
-            constexpr KWorkerTaskManager() : m_head_task(), m_tail_task(), m_thread(), m_type(WorkerType_Count), m_active() { /* ... */ }
+            constexpr KWorkerTaskManager() : m_head_task(), m_tail_task(), m_waiting_thread() { /* ... */ }
 
-            NOINLINE void Initialize(WorkerType wt, s32 priority);
+            NOINLINE void Initialize(s32 priority);
             static void AddTask(WorkerType type, KWorkerTask *task);
     };
 

libraries/libmesosphere/include/mesosphere/kern_k_writable_event.hpp

@@ -1,44 +0,0 @@
-/*
- * Copyright (c) 2018-2020 Atmosphère-NX
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-#pragma once
-#include <mesosphere/kern_common.hpp>
-#include <mesosphere/kern_k_auto_object.hpp>
-#include <mesosphere/kern_slab_helpers.hpp>
-
-namespace ams::kern {
-
-    class KEvent;
-
-    class KWritableEvent final : public KAutoObjectWithSlabHeapAndContainer<KWritableEvent, KAutoObjectWithList> {
-        MESOSPHERE_AUTOOBJECT_TRAITS(KWritableEvent, KAutoObject);
-        private:
-            KEvent *m_parent;
-        public:
-            constexpr explicit KWritableEvent() : m_parent(nullptr) { /* ... */ }
-            virtual ~KWritableEvent() { /* ... */ }
-
-            virtual void Destroy() override;
-
-            static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
-
-            void Initialize(KEvent *p);
-            Result Signal();
-            Result Clear();
-
-            constexpr KEvent *GetParent() const { return m_parent; }
-    };
-
-}

libraries/libmesosphere/include/mesosphere/kern_kernel.hpp

@@ -63,14 +63,21 @@ namespace ams::kern {
             static constexpr size_t ApplicationMemoryBlockSlabHeapSize = 20000;
             static constexpr size_t SystemMemoryBlockSlabHeapSize      = 10000;
             static constexpr size_t BlockInfoSlabHeapSize              = 4000;
+            static constexpr size_t ReservedDynamicPageCount           = 70;
         private:
             static State s_state;
             static KResourceLimit s_system_resource_limit;
             static KMemoryManager s_memory_manager;
-            static KPageTableManager s_page_table_manager;
+            static KPageTableSlabHeap s_page_table_heap;
+            static KMemoryBlockSlabHeap s_app_memory_block_heap;
+            static KMemoryBlockSlabHeap s_sys_memory_block_heap;
+            static KBlockInfoSlabHeap s_block_info_heap;
+            static KPageTableManager s_app_page_table_manager;
+            static KPageTableManager s_sys_page_table_manager;
             static KMemoryBlockSlabManager s_app_memory_block_manager;
             static KMemoryBlockSlabManager s_sys_memory_block_manager;
-            static KBlockInfoManager s_block_info_manager;
+            static KBlockInfoManager s_app_block_info_manager;
+            static KBlockInfoManager s_sys_block_info_manager;
             static KSupervisorPageTable s_supervisor_page_table;
             static KUnsafeMemory s_unsafe_memory;
             static KWorkerTaskManager s_worker_task_managers[KWorkerTaskManager::WorkerType_Count];
@@ -130,12 +137,20 @@ namespace ams::kern {
                 return s_sys_memory_block_manager;
             }
 
-            static ALWAYS_INLINE KBlockInfoManager &GetBlockInfoManager() {
+            static ALWAYS_INLINE KBlockInfoManager &GetApplicationBlockInfoManager() {
-                return s_block_info_manager;
+                return s_app_block_info_manager;
             }
 
-            static ALWAYS_INLINE KPageTableManager &GetPageTableManager() {
+            static ALWAYS_INLINE KBlockInfoManager &GetSystemBlockInfoManager() {
-                return s_page_table_manager;
+                return s_sys_block_info_manager;
+            }
+
+            static ALWAYS_INLINE KPageTableManager &GetApplicationPageTableManager() {
+                return s_app_page_table_manager;
+            }
+
+            static ALWAYS_INLINE KPageTableManager &GetSystemPageTableManager() {
+                return s_sys_page_table_manager;
             }
 
             static ALWAYS_INLINE KSupervisorPageTable &GetKernelPageTable() {

libraries/libmesosphere/include/mesosphere/kern_slab_helpers.hpp

@@ -18,15 +18,16 @@
 #include <mesosphere/kern_k_auto_object.hpp>
 #include <mesosphere/kern_k_slab_heap.hpp>
 #include <mesosphere/kern_k_auto_object_container.hpp>
+#include <mesosphere/kern_k_unused_slab_memory.hpp>
 
 namespace ams::kern {
 
-    template<class Derived>
+    template<class Derived, bool SupportDynamicExpansion = false>
     class KSlabAllocated {
         private:
-            static inline KSlabHeap<Derived> s_slab_heap;
+            static constinit inline KSlabHeap<Derived, SupportDynamicExpansion> s_slab_heap;
         public:
-            constexpr KSlabAllocated() { /* ... */ }
+            constexpr KSlabAllocated() = default;
 
             size_t GetSlabIndex() const {
                 return s_slab_heap.GetIndex(static_cast<const Derived *>(this));
@@ -36,14 +37,25 @@ namespace ams::kern {
                 s_slab_heap.Initialize(memory, memory_size);
             }
 
-            static ALWAYS_INLINE Derived *Allocate() {
+            static Derived *Allocate() {
                 return s_slab_heap.Allocate();
             }
 
-            static ALWAYS_INLINE void Free(Derived *obj) {
+            static void Free(Derived *obj) {
                 s_slab_heap.Free(obj);
             }
 
+            template<bool Enable = SupportDynamicExpansion, typename = typename std::enable_if<Enable>::type>
+            static Derived *AllocateFromUnusedSlabMemory() {
+                static_assert(Enable == SupportDynamicExpansion);
+
+                Derived * const obj = GetPointer<Derived>(AllocateUnusedSlabMemory(sizeof(Derived), alignof(Derived)));
+                if (AMS_LIKELY(obj != nullptr)) {
+                    std::construct_at(obj);
+                }
+                return obj;
+            }
+
             static size_t GetObjectSize() { return s_slab_heap.GetObjectSize(); }
             static size_t GetSlabHeapSize() { return s_slab_heap.GetSlabHeapSize(); }
             static size_t GetPeakIndex() { return s_slab_heap.GetPeakIndex(); }
@@ -52,12 +64,12 @@ namespace ams::kern {
             static size_t GetNumRemaining() { return s_slab_heap.GetNumRemaining(); }
     };
 
-    template<typename Derived, typename Base>
+    template<typename Derived, typename Base, bool SupportDynamicExpansion = false>
     class KAutoObjectWithSlabHeapAndContainer : public Base {
         static_assert(std::is_base_of<KAutoObjectWithList, Base>::value);
         private:
-            static inline KSlabHeap<Derived> s_slab_heap;
+            static constinit inline KSlabHeap<Derived, SupportDynamicExpansion> s_slab_heap;
-            static inline KAutoObjectWithListContainer s_container;
+            static constinit inline KAutoObjectWithListContainer s_container;
         private:
             static ALWAYS_INLINE Derived *Allocate() {
                 return s_slab_heap.Allocate();
@@ -73,8 +85,7 @@ namespace ams::kern {
                     ALWAYS_INLINE ~ListAccessor() { /* ... */ }
             };
         public:
-            constexpr KAutoObjectWithSlabHeapAndContainer() : Base() { /* ... */ }
+            constexpr KAutoObjectWithSlabHeapAndContainer() = default;
-            virtual ~KAutoObjectWithSlabHeapAndContainer() { /* ... */ }
 
             virtual void Destroy() override {
                 const bool is_initialized = this->IsInitialized();
@@ -110,6 +121,18 @@ namespace ams::kern {
                 return obj;
             }
 
+            template<bool Enable = SupportDynamicExpansion, typename = typename std::enable_if<Enable>::type>
+            static Derived *CreateFromUnusedSlabMemory() {
+                static_assert(Enable == SupportDynamicExpansion);
+
+                Derived * const obj = GetPointer<Derived>(AllocateUnusedSlabMemory(sizeof(Derived), alignof(Derived)));
+                if (AMS_LIKELY(obj != nullptr)) {
+                    std::construct_at(obj);
+                    KAutoObject::Create(obj);
+                }
+                return obj;
+            }
+
             static void Register(Derived *obj) {
                 return s_container.Register(obj);
             }

libraries/libmesosphere/source/arch/arm64/kern_exception_handlers.cpp

@@ -308,7 +308,7 @@ namespace ams::kern::arch::arm64 {
                 {
                     if (ec != EsrEc_SoftwareStepEl0) {
                         /* If the exception wasn't single-step, print details. */
-                        MESOSPHERE_RELEASE_LOG("Exception occurred. %016lx\n", GetCurrentProcess().GetProgramId());
+                        MESOSPHERE_EXCEPTION_LOG("Exception occurred. ");
 
                         {
                             /* Print the current thread's registers. */
@@ -327,7 +327,7 @@ namespace ams::kern::arch::arm64 {
                 #else
                 {
                     /* Print that an exception occurred. */
-                    MESOSPHERE_RELEASE_LOG("Exception occurred. %016lx\n", GetCurrentProcess().GetProgramId());
+                    MESOSPHERE_EXCEPTION_LOG("Exception occurred. ");
 
                     {
                         /* Print the current thread's registers. */
@@ -496,7 +496,7 @@ namespace ams::kern::arch::arm64 {
         }
 
         /* Print that an exception occurred. */
-        MESOSPHERE_RELEASE_LOG("Exception occurred. %016lx\n", GetCurrentProcess().GetProgramId());
+        MESOSPHERE_EXCEPTION_LOG("Exception occurred. ");
 
         /* Exit the current process. */
         GetCurrentProcess().Exit();

libraries/libmesosphere/source/arch/arm64/kern_k_debug.cpp

@@ -307,8 +307,14 @@ namespace ams::kern::arch::arm64 {
                     R_UNLESS(debug.IsNotNull(), svc::ResultInvalidHandle());
 
                     /* Get the process from the debug object. */
-                    process = debug->GetProcess();
+                    R_UNLESS(debug->IsAttached(),  svc::ResultProcessTerminated());
-                    R_UNLESS(process.IsNotNull(), svc::ResultProcessTerminated());
+                    R_UNLESS(debug->OpenProcess(), svc::ResultProcessTerminated());
+
+                    /* Close the process when we're done. */
+                    ON_SCOPE_EXIT { debug->CloseProcess(); };
+
+                    /* Get the proces. */
+                    KProcess * const process = debug->GetProcessUnsafe();
 
                     /* Set the value to be the context id. */
                     value = process->GetId() & 0xFFFFFFFF;

libraries/libmesosphere/source/arch/arm64/kern_k_page_table.cpp

@@ -166,7 +166,7 @@ namespace ams::kern::arch::arm64 {
     Result KPageTable::InitializeForKernel(void *table, KVirtualAddress start, KVirtualAddress end) {
         /* Initialize basic fields. */
         m_asid = 0;
-        m_manager = std::addressof(Kernel::GetPageTableManager());
+        m_manager = std::addressof(Kernel::GetSystemPageTableManager());
 
         /* Allocate a page for ttbr. */
         const u64 asid_tag = (static_cast<u64>(m_asid) << 48ul);
@@ -247,7 +247,7 @@ namespace ams::kern::arch::arm64 {
                         cur_entry.block_size += next_entry.block_size;
                     } else {
                         if (cur_valid && IsHeapPhysicalAddressForFinalize(cur_entry.phys_addr)) {
-                            mm.Close(GetHeapVirtualAddress(cur_entry.phys_addr), cur_entry.block_size / PageSize);
+                            mm.Close(cur_entry.phys_addr, cur_entry.block_size / PageSize);
                         }
 
                         /* Update tracking variables. */
@@ -265,7 +265,7 @@ namespace ams::kern::arch::arm64 {
 
                 /* Handle the last block. */
                 if (cur_valid && IsHeapPhysicalAddressForFinalize(cur_entry.phys_addr)) {
-                    mm.Close(GetHeapVirtualAddress(cur_entry.phys_addr), cur_entry.block_size / PageSize);
+                    mm.Close(cur_entry.phys_addr, cur_entry.block_size / PageSize);
                 }
             }
 
@@ -579,7 +579,7 @@ namespace ams::kern::arch::arm64 {
 
         /* Ensure that any pages we track close on exit. */
         KPageGroup pages_to_close(this->GetBlockInfoManager());
-        ON_SCOPE_EXIT { pages_to_close.Close(); };
+        ON_SCOPE_EXIT { pages_to_close.CloseAndReset(); };
 
         /* Begin traversal. */
         TraversalContext context;
@@ -601,8 +601,9 @@ namespace ams::kern::arch::arm64 {
             if (next_entry.block_size > remaining_pages * PageSize) {
                 MESOSPHERE_ABORT_UNLESS(force);
                 MESOSPHERE_R_ABORT_UNLESS(this->SeparatePages(virt_addr, remaining_pages * PageSize, page_list, reuse_ll));
-                next_valid = impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), virt_addr);
+                const bool new_valid = impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), virt_addr);
-                MESOSPHERE_ASSERT(next_valid);
+                MESOSPHERE_ASSERT(new_valid);
+                MESOSPHERE_UNUSED(new_valid);
             }
 
             /* Check that our state is coherent. */
@@ -641,6 +642,7 @@ namespace ams::kern::arch::arm64 {
                                 *l1_entry = InvalidL1PageTableEntry;
                                 this->NoteUpdated();
                                 this->FreePageTable(page_list, l2_virt);
+                                pages_to_close.CloseAndReset();
                             }
                         }
                     }
@@ -684,6 +686,7 @@ namespace ams::kern::arch::arm64 {
                                 }
 
                                 this->FreePageTable(page_list, l3_virt);
+                                pages_to_close.CloseAndReset();
                             }
                         }
                     }
@@ -693,11 +696,11 @@ namespace ams::kern::arch::arm64 {
 
             /* Close the blocks. */
             if (!force && IsHeapPhysicalAddress(next_entry.phys_addr)) {
-                const KVirtualAddress block_virt_addr = GetHeapVirtualAddress(next_entry.phys_addr);
                 const size_t block_num_pages = next_entry.block_size / PageSize;
-                if (R_FAILED(pages_to_close.AddBlock(block_virt_addr, block_num_pages))) {
+                if (R_FAILED(pages_to_close.AddBlock(next_entry.phys_addr, block_num_pages))) {
                     this->NoteUpdated();
-                    Kernel::GetMemoryManager().Close(block_virt_addr, block_num_pages);
+                    Kernel::GetMemoryManager().Close(next_entry.phys_addr, block_num_pages);
+                    pages_to_close.CloseAndReset();
                 }
             }
 
@@ -788,7 +791,7 @@ namespace ams::kern::arch::arm64 {
 
         /* Open references to the pages, if we should. */
         if (IsHeapPhysicalAddress(orig_phys_addr)) {
-            Kernel::GetMemoryManager().Open(GetHeapVirtualAddress(orig_phys_addr), num_pages);
+            Kernel::GetMemoryManager().Open(orig_phys_addr, num_pages);
         }
 
         return ResultSuccess();
@@ -811,7 +814,7 @@ namespace ams::kern::arch::arm64 {
 
             if (num_pages < ContiguousPageSize / PageSize) {
                 for (const auto &block : pg) {
-                    const KPhysicalAddress block_phys_addr = GetLinearMappedPhysicalAddress(block.GetAddress());
+                    const KPhysicalAddress block_phys_addr = block.GetAddress();
                     const size_t cur_pages = block.GetNumPages();
                     R_TRY(this->Map(virt_addr, block_phys_addr, cur_pages, entry_template, disable_head_merge && virt_addr == orig_virt_addr, L3BlockSize, page_list, reuse_ll));
 
@@ -823,7 +826,7 @@ namespace ams::kern::arch::arm64 {
                 AlignedMemoryBlock virt_block(GetInteger(virt_addr), num_pages, L1BlockSize);
                 for (const auto &block : pg) {
                     /* Create a block representing this physical group, synchronize its alignment to our virtual block. */
-                    const KPhysicalAddress block_phys_addr = GetLinearMappedPhysicalAddress(block.GetAddress());
+                    const KPhysicalAddress block_phys_addr = block.GetAddress();
                     size_t cur_pages = block.GetNumPages();
 
                     AlignedMemoryBlock phys_block(GetInteger(block_phys_addr), cur_pages, virt_block.GetAlignment());

libraries/libmesosphere/source/board/nintendo/nx/kern_k_device_page_table.cpp

@@ -650,11 +650,11 @@ namespace ams::kern::board::nintendo::nx {
         g_memory_controller_address = KMemoryLayout::GetDevicePhysicalAddress(KMemoryRegionType_MemoryController);
 
         /* Allocate a page to use as a reserved/no device table. */
-        const KVirtualAddress table_virt_addr = Kernel::GetPageTableManager().Allocate();
+        const KVirtualAddress table_virt_addr = Kernel::GetSystemPageTableManager().Allocate();
         MESOSPHERE_ABORT_UNLESS(table_virt_addr != Null<KVirtualAddress>);
         const KPhysicalAddress table_phys_addr = GetPageTablePhysicalAddress(table_virt_addr);
         MESOSPHERE_ASSERT(IsValidPhysicalAddress(table_phys_addr));
-        Kernel::GetPageTableManager().Open(table_virt_addr, 1);
+        Kernel::GetSystemPageTableManager().Open(table_virt_addr, 1);
 
         /* Clear the page and save it. */
         /* NOTE: Nintendo does not check the result of StoreDataCache. */
@@ -779,7 +779,7 @@ namespace ams::kern::board::nintendo::nx {
         const size_t end_index   = (space_address + space_size - 1) / DeviceRegionSize;
 
         /* Get the page table manager. */
-        auto &ptm = Kernel::GetPageTableManager();
+        auto &ptm = Kernel::GetSystemPageTableManager();
 
         /* Clear the tables. */
         static_assert(TableCount == (1ul << DeviceVirtualAddressBits) / DeviceRegionSize);
@@ -840,7 +840,7 @@ namespace ams::kern::board::nintendo::nx {
 
     void KDevicePageTable::Finalize() {
         /* Get the page table manager. */
-        auto &ptm = Kernel::GetPageTableManager();
+        auto &ptm = Kernel::GetSystemPageTableManager();
 
         /* Detach from all devices. */
         {
@@ -1006,10 +1006,7 @@ namespace ams::kern::board::nintendo::nx {
         return true;
     }
 
-    Result KDevicePageTable::MapDevicePage(size_t *out_mapped_size, s32 &num_pt, s32 max_pt, KPhysicalAddress phys_addr, u64 size, KDeviceVirtualAddress address, ams::svc::MemoryPermission device_perm) {
+    Result KDevicePageTable::MapDevicePage(KPhysicalAddress phys_addr, u64 size, KDeviceVirtualAddress address, ams::svc::MemoryPermission device_perm) {
-        /* Clear the output size. */
-        *out_mapped_size = 0;
-
         /* Ensure that the physical address is valid. */
         R_UNLESS(IsValidPhysicalAddress(static_cast<u64>(GetInteger(phys_addr)) + size - 1), svc::ResultInvalidCurrentMemory());
         MESOSPHERE_ASSERT((address & ~DeviceVirtualAddressMask) == 0);
@@ -1017,7 +1014,7 @@ namespace ams::kern::board::nintendo::nx {
 
         /* Get the memory manager and page table manager. */
         KMemoryManager &mm     = Kernel::GetMemoryManager();
-        KPageTableManager &ptm = Kernel::GetPageTableManager();
+        KPageTableManager &ptm = Kernel::GetSystemPageTableManager();
 
         /* Cache permissions. */
         const bool read  = (device_perm & ams::svc::MemoryPermission_Read)  != 0;
@@ -1051,16 +1048,13 @@ namespace ams::kern::board::nintendo::nx {
                     SmmuSynchronizationBarrier();
 
                     /* Open references to the pages. */
-                    mm.Open(GetHeapVirtualAddress(phys_addr), DeviceLargePageSize / PageSize);
+                    mm.Open(phys_addr, DeviceLargePageSize / PageSize);
 
                     /* Advance. */
                     phys_addr        += DeviceLargePageSize;
                     address          += DeviceLargePageSize;
-                    *out_mapped_size += DeviceLargePageSize;
                     remaining        -= DeviceLargePageSize;
                     continue;
-                } else if (num_pt == max_pt) {
-                    break;
                 } else {
                     /* Make an l1 table. */
                     const KVirtualAddress table_vaddr = ptm.Allocate();
@@ -1076,9 +1070,6 @@ namespace ams::kern::board::nintendo::nx {
                     InvalidatePtc(GetPageTablePhysicalAddress(KVirtualAddress(std::addressof(l1[l1_index]))));
                     InvalidateTlbSection(m_table_asids[l0_index], address);
                     SmmuSynchronizationBarrier();
-
-                    /* Increment the page table count. */
-                    ++num_pt;
                 }
             }
 
@@ -1112,12 +1103,11 @@ namespace ams::kern::board::nintendo::nx {
                 SmmuSynchronizationBarrier();
 
                 /* Open references to the pages. */
-                mm.Open(GetHeapVirtualAddress(phys_addr), (map_count * DevicePageSize) / PageSize);
+                mm.Open(phys_addr, (map_count * DevicePageSize) / PageSize);
 
                 /* Advance. */
                 phys_addr        += map_count * DevicePageSize;
                 address          += map_count * DevicePageSize;
-                *out_mapped_size += map_count * DevicePageSize;
                 remaining        -= map_count * DevicePageSize;
             }
         }
@@ -1125,14 +1115,7 @@ namespace ams::kern::board::nintendo::nx {
         return ResultSuccess();
     }
 
-    Result KDevicePageTable::MapImpl(size_t *out_mapped_size, s32 &num_pt, s32 max_pt, KProcessPageTable *page_table, KProcessAddress process_address, size_t size, KDeviceVirtualAddress device_address, ams::svc::MemoryPermission device_perm, bool is_aligned) {
+    Result KDevicePageTable::MapImpl(KProcessPageTable *page_table, KProcessAddress process_address, size_t size, KDeviceVirtualAddress device_address, ams::svc::MemoryPermission device_perm, bool is_aligned) {
-        /* Clear the output size. */
-        *out_mapped_size = 0;
-
-        /* Get the size, and validate the address. */
-        MESOSPHERE_ASSERT((device_address & ~DeviceVirtualAddressMask) == 0);
-        MESOSPHERE_ASSERT(((device_address + size - 1) & ~DeviceVirtualAddressMask) == 0);
-
         /* Ensure that the region we're mapping to is free. */
         R_UNLESS(this->IsFree(device_address, size), svc::ResultInvalidCurrentMemory());
 
@@ -1141,31 +1124,28 @@ namespace ams::kern::board::nintendo::nx {
 
         /* Iterate, mapping device pages. */
         KDeviceVirtualAddress cur_addr = device_address;
-        while (true) {
+        size_t mapped_size = 0;
-            /* Get the current contiguous range. */
+        while (mapped_size < size) {
-            KPageTableBase::MemoryRange contig_range = {};
+            /* Map the next contiguous range. */
-            R_TRY(page_table->OpenMemoryRangeForMapDeviceAddressSpace(std::addressof(contig_range), process_address + *out_mapped_size, size - *out_mapped_size, ConvertToKMemoryPermission(device_perm), is_aligned));
+            size_t cur_size;
+            {
+                /* Get the current contiguous range. */
+                KPageTableBase::MemoryRange contig_range = {};
+                R_TRY(page_table->OpenMemoryRangeForMapDeviceAddressSpace(std::addressof(contig_range), process_address + mapped_size, size - mapped_size, ConvertToKMemoryPermission(device_perm), is_aligned));
 
-            /* Ensure we close the range when we're done. */
+                /* Ensure we close the range when we're done. */
-            ON_SCOPE_EXIT { contig_range.Close(); };
+                ON_SCOPE_EXIT { contig_range.Close(); };
 
-            /* Map the device page. */
+                /* Get the current size. */
-            size_t mapped_size = 0;
+                cur_size = contig_range.size;
-            R_TRY(this->MapDevicePage(std::addressof(mapped_size), num_pt, max_pt, GetHeapPhysicalAddress(contig_range.address), contig_range.size, cur_addr, device_perm));
+
+                /* Map the device page. */
+                R_TRY(this->MapDevicePage(contig_range.address, contig_range.size, cur_addr, device_perm));
+            }
 
             /* Advance. */
-            cur_addr         += contig_range.size;
+            cur_addr    += cur_size;
-            *out_mapped_size += mapped_size;
+            mapped_size += cur_size;
-
-            /* If we didn't map as much as we wanted, break. */
-            if (mapped_size < contig_range.size) {
-                break;
-            }
-
-            /* Similarly, if we're done, break. */
-            if (*out_mapped_size >= size) {
-                break;
-            }
         }
 
         /* We're done, so cancel our guard. */
@@ -1181,10 +1161,10 @@ namespace ams::kern::board::nintendo::nx {
 
         /* Get the memory manager and page table manager. */
         KMemoryManager &mm     = Kernel::GetMemoryManager();
-        KPageTableManager &ptm = Kernel::GetPageTableManager();
+        KPageTableManager &ptm = Kernel::GetSystemPageTableManager();
 
         /* Make a page group for the pages we're closing. */
-        KPageGroup pg(std::addressof(Kernel::GetBlockInfoManager()));
+        KPageGroup pg(std::addressof(Kernel::GetSystemBlockInfoManager()));
 
         /* Walk the directory. */
         u64 remaining = size;
@@ -1245,7 +1225,7 @@ namespace ams::kern::board::nintendo::nx {
                             contig_count     = contig_phys_addr != Null<KPhysicalAddress> ? 1 : 0;
                         } else if (phys_addr == Null<KPhysicalAddress> || phys_addr != (contig_phys_addr + (contig_count * DevicePageSize))) {
                             /* If we're no longer contiguous, close the range we've been building. */
-                            mm.Close(GetHeapVirtualAddress(contig_phys_addr), (contig_count * DevicePageSize) / PageSize);
+                            mm.Close(contig_phys_addr, (contig_count * DevicePageSize) / PageSize);
 
                             contig_phys_addr = phys_addr;
                             contig_count     = contig_phys_addr != Null<KPhysicalAddress> ? 1 : 0;
@@ -1255,7 +1235,7 @@ namespace ams::kern::board::nintendo::nx {
                     }
 
                     if (contig_count > 0) {
-                        mm.Close(GetHeapVirtualAddress(contig_phys_addr), (contig_count * DevicePageSize) / PageSize);
+                        mm.Close(contig_phys_addr, (contig_count * DevicePageSize) / PageSize);
                     }
                 }
 
@@ -1294,7 +1274,7 @@ namespace ams::kern::board::nintendo::nx {
                 SmmuSynchronizationBarrier();
 
                 /* Close references. */
-                mm.Close(GetHeapVirtualAddress(phys_addr), DeviceLargePageSize / PageSize);
+                mm.Close(phys_addr, DeviceLargePageSize / PageSize);
 
                 /* Advance. */
                 address   += DeviceLargePageSize;
@@ -1320,7 +1300,7 @@ namespace ams::kern::board::nintendo::nx {
         /* Walk the directory. */
         KProcessAddress cur_process_address = process_address;
         size_t remaining_size               = size;
-        KPhysicalAddress cur_phys_address   = GetHeapPhysicalAddress(contig_range.address);
+        KPhysicalAddress cur_phys_address   = contig_range.address;
         size_t remaining_in_range           = contig_range.size;
         bool first                          = true;
         u32  first_attr                     = 0;
@@ -1367,7 +1347,7 @@ namespace ams::kern::board::nintendo::nx {
                         }
                         range_open = true;
 
-                        cur_phys_address   = GetHeapPhysicalAddress(contig_range.address);
+                        cur_phys_address   = contig_range.address;
                         remaining_in_range = contig_range.size;
                     }
 
@@ -1410,7 +1390,7 @@ namespace ams::kern::board::nintendo::nx {
                     }
                     range_open = true;
 
-                    cur_phys_address   = GetHeapPhysicalAddress(contig_range.address);
+                    cur_phys_address   = contig_range.address;
                     remaining_in_range = contig_range.size;
                 }
 
@@ -1437,13 +1417,13 @@ namespace ams::kern::board::nintendo::nx {
         return true;
     }
 
-    Result KDevicePageTable::Map(size_t *out_mapped_size, KProcessPageTable *page_table, KProcessAddress process_address, size_t size, KDeviceVirtualAddress device_address, ams::svc::MemoryPermission device_perm, bool is_aligned, bool refresh_mappings) {
+    Result KDevicePageTable::Map(KProcessPageTable *page_table, KProcessAddress process_address, size_t size, KDeviceVirtualAddress device_address, ams::svc::MemoryPermission device_perm, bool is_aligned) {
-        /* Clear the output size. */
+        /* Validate address/size. */
-        *out_mapped_size = 0;
+        MESOSPHERE_ASSERT((device_address & ~DeviceVirtualAddressMask) == 0);
+        MESOSPHERE_ASSERT(((device_address + size - 1) & ~DeviceVirtualAddressMask) == 0);
 
         /* Map the pages. */
-        s32 num_pt = 0;
+        return this->MapImpl(page_table, process_address, size, device_address, device_perm, is_aligned);
-        return this->MapImpl(out_mapped_size, num_pt, refresh_mappings ? 1 : std::numeric_limits<s32>::max(), page_table, process_address, size, device_address, device_perm, is_aligned);
     }
 
     Result KDevicePageTable::Unmap(KProcessPageTable *page_table, KProcessAddress process_address, size_t size, KDeviceVirtualAddress device_address) {

libraries/libmesosphere/source/board/nintendo/nx/kern_k_io_pool.board.nintendo_nx.inc

@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+constexpr IoRegionExtents g_io_region_extents[4] = {
+    { KPhysicalAddress(0x12000000), 224_MB }, /* PCIE_A2 */
+    { Null<KPhysicalAddress>, 0 },
+    { Null<KPhysicalAddress>, 0 },
+    { Null<KPhysicalAddress>, 0 },
+};
+
+constexpr bool IsValidIoPoolTypeImpl(ams::svc::IoPoolType pool_type) {
+    return pool_type == ams::svc::IoPoolType_PcieA2;
+}

libraries/libmesosphere/source/board/nintendo/nx/kern_k_system_control.cpp

@@ -346,8 +346,11 @@ namespace ams::kern::board::nintendo::nx {
         }
     }
 
-    KPhysicalAddress KSystemControl::Init::GetInitialProcessBinaryPhysicalAddress() {
+    void KSystemControl::Init::GetInitialProcessBinaryLayout(InitialProcessBinaryLayout *out) {
-        return GetKernelPhysicalBaseAddress(DramPhysicalAddress) + GetIntendedMemorySize() - KTraceBufferSize - InitialProcessBinarySizeMax;
+        *out = {
+            .address = GetInteger(GetKernelPhysicalBaseAddress(DramPhysicalAddress)) + GetIntendedMemorySize() - KTraceBufferSize - InitialProcessBinarySizeMax,
+            ._08     = 0,
+        };
     }
 
     bool KSystemControl::Init::ShouldIncreaseThreadResourceLimit() {
@@ -459,6 +462,7 @@ namespace ams::kern::board::nintendo::nx {
 
             KTargetSystem::EnableDebugMemoryFill(kernel_config.Get<smc::KernelConfiguration::DebugFillMemory>());
             KTargetSystem::EnableUserExceptionHandlers(kernel_config.Get<smc::KernelConfiguration::EnableUserExceptionHandlers>());
+            KTargetSystem::EnableDynamicResourceLimits(!kernel_config.Get<smc::KernelConfiguration::DisableDynamicResourceLimits>());
             KTargetSystem::EnableUserPmuAccess(kernel_config.Get<smc::KernelConfiguration::EnableUserPmuAccess>());
 
             g_call_smc_on_panic = kernel_config.Get<smc::KernelConfiguration::UseSecureMonitorPanicCall>();
@@ -510,8 +514,10 @@ namespace ams::kern::board::nintendo::nx {
             MESOSPHERE_ABORT_UNLESS(Kernel::GetSystemResourceLimit().Reserve(ams::svc::LimitableResource_PhysicalMemoryMax, SecureAppletMemorySize));
 
             constexpr auto SecureAppletAllocateOption = KMemoryManager::EncodeOption(KMemoryManager::Pool_System, KMemoryManager::Direction_FromFront);
-            g_secure_applet_memory_address = Kernel::GetMemoryManager().AllocateAndOpenContinuous(SecureAppletMemorySize / PageSize, 1, SecureAppletAllocateOption);
+            const KPhysicalAddress secure_applet_memory_phys_addr = Kernel::GetMemoryManager().AllocateAndOpenContinuous(SecureAppletMemorySize / PageSize, 1, SecureAppletAllocateOption);
-            MESOSPHERE_ABORT_UNLESS(g_secure_applet_memory_address != Null<KVirtualAddress>);
+            MESOSPHERE_ABORT_UNLESS(secure_applet_memory_phys_addr != Null<KPhysicalAddress>);
+
+            g_secure_applet_memory_address = KMemoryLayout::GetLinearVirtualAddress(secure_applet_memory_phys_addr);
         }
 
         /* Initialize KTrace. */
@@ -689,9 +695,7 @@ namespace ams::kern::board::nintendo::nx {
                     MESOSPHERE_ASSERT(it != page_groups[i].end());
                     MESOSPHERE_ASSERT(it->GetNumPages() == 1);
 
-                    KPhysicalAddress phys_addr = page_table.GetHeapPhysicalAddress(it->GetAddress());
+                    args->r[reg_id] = GetInteger(it->GetAddress()) | (GetInteger(virt_addr) & (PageSize - 1));
-
-                    args->r[reg_id] = GetInteger(phys_addr) | (GetInteger(virt_addr) & (PageSize - 1));
                 } else {
                     /* If we couldn't map, we should clear the address. */
                     args->r[reg_id] = 0;
@@ -728,25 +732,21 @@ namespace ams::kern::board::nintendo::nx {
 
         /* Allocate the memory. */
         const size_t num_pages = size / PageSize;
-        const KVirtualAddress vaddr = Kernel::GetMemoryManager().AllocateAndOpenContinuous(num_pages, alignment / PageSize, KMemoryManager::EncodeOption(static_cast<KMemoryManager::Pool>(pool), KMemoryManager::Direction_FromFront));
+        const KPhysicalAddress paddr = Kernel::GetMemoryManager().AllocateAndOpenContinuous(num_pages, alignment / PageSize, KMemoryManager::EncodeOption(static_cast<KMemoryManager::Pool>(pool), KMemoryManager::Direction_FromFront));
-        R_UNLESS(vaddr != Null<KVirtualAddress>, svc::ResultOutOfMemory());
+        R_UNLESS(paddr != Null<KPhysicalAddress>, svc::ResultOutOfMemory());
 
         /* Ensure we don't leak references to the memory on error. */
-        auto mem_guard = SCOPE_GUARD { Kernel::GetMemoryManager().Close(vaddr, num_pages); };
+        auto mem_guard = SCOPE_GUARD { Kernel::GetMemoryManager().Close(paddr, num_pages); };
 
         /* If the memory isn't already secure, set it as secure. */
         if (pool != KMemoryManager::Pool_System) {
-            /* Get the physical address. */
-            const KPhysicalAddress paddr = KPageTable::GetHeapPhysicalAddress(vaddr);
-            MESOSPHERE_ABORT_UNLESS(paddr != Null<KPhysicalAddress>);
-
             /* Set the secure region. */
             R_UNLESS(SetSecureRegion(paddr, size), svc::ResultOutOfMemory());
         }
 
         /* We succeeded. */
         mem_guard.Cancel();
-        *out = vaddr;
+        *out = KPageTable::GetHeapVirtualAddress(paddr);
         return ResultSuccess();
     }
 
@@ -778,7 +778,7 @@ namespace ams::kern::board::nintendo::nx {
         }
 
         /* Close the secure region's pages. */
-        Kernel::GetMemoryManager().Close(address, size / PageSize);
+        Kernel::GetMemoryManager().Close(KPageTable::GetHeapPhysicalAddress(address), size / PageSize);
     }
 
 }

libraries/libmesosphere/source/board/nintendo/nx/kern_secure_monitor.hpp

@@ -80,14 +80,15 @@ namespace ams::kern::board::nintendo::nx::smc {
     };
 
     struct KernelConfiguration {
-        using DebugFillMemory             = util::BitPack32::Field<0,                                 1, bool>;
+        using DebugFillMemory              = util::BitPack32::Field<0,                                  1, bool>;
-        using EnableUserExceptionHandlers = util::BitPack32::Field<DebugFillMemory::Next,             1, bool>;
+        using EnableUserExceptionHandlers  = util::BitPack32::Field<DebugFillMemory::Next,              1, bool>;
-        using EnableUserPmuAccess         = util::BitPack32::Field<EnableUserExceptionHandlers::Next, 1, bool>;
+        using EnableUserPmuAccess          = util::BitPack32::Field<EnableUserExceptionHandlers::Next,  1, bool>;
-        using IncreaseThreadResourceLimit = util::BitPack32::Field<EnableUserPmuAccess::Next,         1, bool>;
+        using IncreaseThreadResourceLimit  = util::BitPack32::Field<EnableUserPmuAccess::Next,          1, bool>;
-        using Reserved4                   = util::BitPack32::Field<IncreaseThreadResourceLimit::Next, 4, u32>;
+        using DisableDynamicResourceLimits = util::BitPack32::Field<IncreaseThreadResourceLimit::Next,  1, bool>;
-        using UseSecureMonitorPanicCall   = util::BitPack32::Field<Reserved4::Next,                   1, bool>;
+        using Reserved5                    = util::BitPack32::Field<DisableDynamicResourceLimits::Next, 3, u32>;
-        using Reserved9                   = util::BitPack32::Field<UseSecureMonitorPanicCall::Next,   7, u32>;
+        using UseSecureMonitorPanicCall    = util::BitPack32::Field<Reserved5::Next,                    1, bool>;
-        using MemorySize                  = util::BitPack32::Field<Reserved9::Next,                   2, smc::MemorySize>;
+        using Reserved9                    = util::BitPack32::Field<UseSecureMonitorPanicCall::Next,    7, u32>;
+        using MemorySize                   = util::BitPack32::Field<Reserved9::Next,                    2, smc::MemorySize>;
     };
 
     enum UserRebootType {

libraries/libmesosphere/source/init/kern_init_slab_setup.cpp

@@ -39,8 +39,8 @@ namespace ams::kern::init {
         HANDLER(KResourceLimit,      (SLAB_COUNT(KResourceLimit)),                                              ## __VA_ARGS__) \
         HANDLER(KEventInfo,          (SLAB_COUNT(KThread) + SLAB_COUNT(KDebug)),                                ## __VA_ARGS__) \
         HANDLER(KDebug,              (SLAB_COUNT(KDebug)),                                                      ## __VA_ARGS__) \
-        HANDLER(KAlpha,              (SLAB_COUNT(KAlpha)),                                                      ## __VA_ARGS__) \
+        HANDLER(KIoPool,             (SLAB_COUNT(KIoPool)),                                                     ## __VA_ARGS__) \
-        HANDLER(KBeta,               (SLAB_COUNT(KBeta)),                                                       ## __VA_ARGS__)
+        HANDLER(KIoRegion,           (SLAB_COUNT(KIoRegion)),                                                   ## __VA_ARGS__)
 
     namespace {
 
@@ -59,7 +59,7 @@ namespace ams::kern::init {
         constexpr size_t SlabCountKThread               = 800;
         constexpr size_t SlabCountKEvent                = 900;
         constexpr size_t SlabCountKInterruptEvent       = 100;
-        constexpr size_t SlabCountKPort                 = 256 + 0x20 /* Extra 0x20 ports over Nintendo for homebrew. */;
+        constexpr size_t SlabCountKPort                 = 384;
         constexpr size_t SlabCountKSharedMemory         = 80;
         constexpr size_t SlabCountKTransferMemory       = 200;
         constexpr size_t SlabCountKCodeMemory           = 10;
@@ -69,8 +69,8 @@ namespace ams::kern::init {
         constexpr size_t SlabCountKObjectName           = 7;
         constexpr size_t SlabCountKResourceLimit        = 5;
         constexpr size_t SlabCountKDebug                = cpu::NumCores;
-        constexpr size_t SlabCountKAlpha                = 1;
+        constexpr size_t SlabCountKIoPool               = 1;
-        constexpr size_t SlabCountKBeta                 = 6;
+        constexpr size_t SlabCountKIoRegion             = 6;
 
         constexpr size_t SlabCountExtraKThread          = 160;
 
@@ -97,8 +97,8 @@ namespace ams::kern::init {
             .num_KObjectName            = SlabCountKObjectName,
             .num_KResourceLimit         = SlabCountKResourceLimit,
             .num_KDebug                 = SlabCountKDebug,
-            .num_KAlpha                 = SlabCountKAlpha,
+            .num_KIoPool                = SlabCountKIoPool,
-            .num_KBeta                  = SlabCountKBeta,
+            .num_KIoRegion              = SlabCountKIoRegion,
         };
 
         template<typename T>
@@ -165,16 +165,17 @@ namespace ams::kern::init {
 
         /* Allocate memory for the slab. */
         constexpr auto AllocateOption = KMemoryManager::EncodeOption(KMemoryManager::Pool_System, KMemoryManager::Direction_FromFront);
-        const KVirtualAddress slab_address = Kernel::GetMemoryManager().AllocateAndOpenContinuous(num_pages, 1, AllocateOption);
+        const KPhysicalAddress slab_address = Kernel::GetMemoryManager().AllocateAndOpenContinuous(num_pages, 1, AllocateOption);
-        MESOSPHERE_ABORT_UNLESS(slab_address != Null<KVirtualAddress>);
+        MESOSPHERE_ABORT_UNLESS(slab_address != Null<KPhysicalAddress>);
 
         /* Initialize the slabheap. */
-        KPageBuffer::InitializeSlabHeap(GetVoidPointer(slab_address), slab_size);
+        KPageBuffer::InitializeSlabHeap(GetVoidPointer(KMemoryLayout::GetLinearVirtualAddress(slab_address)), slab_size);
     }
 
     void InitializeSlabHeaps() {
-        /* Get the start of the slab region, since that's where we'll be working. */
+        /* Get the slab region, since that's where we'll be working. */
-        KVirtualAddress address = KMemoryLayout::GetSlabRegionAddress();
+        const KMemoryRegion &slab_region = KMemoryLayout::GetSlabRegion();
+        KVirtualAddress address = slab_region.GetAddress();
 
         /* Initialize slab type array to be in sorted order. */
         KSlabType slab_types[KSlabType_Count];
@@ -202,13 +203,21 @@ namespace ams::kern::init {
             }
         }
 
+        /* Track the gaps, so that we can free them to the unused slab tree. */
+        KVirtualAddress gap_start = address;
+        size_t gap_size = 0;
+
         for (size_t i = 0; i < util::size(slab_types); i++) {
             /* Add the random gap to the address. */
-            address += (i == 0) ? slab_gaps[0] : slab_gaps[i] - slab_gaps[i - 1];
+            const auto cur_gap = (i == 0) ? slab_gaps[0] : slab_gaps[i] - slab_gaps[i - 1];
+            address  += cur_gap;
+            gap_size += cur_gap;
 
-            #define INITIALIZE_SLAB_HEAP(NAME, COUNT, ...)              \
+            #define INITIALIZE_SLAB_HEAP(NAME, COUNT, ...)                  \
-                case KSlabType_##NAME:                                  \
+                case KSlabType_##NAME:                                      \
-                    address = InitializeSlabHeap<NAME>(address, COUNT); \
+                    if (COUNT > 0) {                                        \
+                        address = InitializeSlabHeap<NAME>(address, COUNT); \
+                    }                                                       \
                     break;
 
             /* Initialize the slabheap. */
@@ -218,7 +227,17 @@ namespace ams::kern::init {
                 /* If we somehow get an invalid type, abort. */
                 MESOSPHERE_UNREACHABLE_DEFAULT_CASE();
             }
+
+            /* If we've hit the end of a gap, free it. */
+            if (gap_start + gap_size != address) {
+                FreeUnusedSlabMemory(gap_start, gap_size);
+                gap_start = address;
+                gap_size  = 0;
+            }
         }
+
+        /* Free the end of the slab region. */
+        FreeUnusedSlabMemory(gap_start, gap_size + (slab_region.GetEndAddress() - GetInteger(address)));
     }
 
 }

libraries/libmesosphere/source/kern_debug_log.cpp

@@ -20,15 +20,15 @@ namespace ams::kern {
 
     namespace {
 
-        KSpinLock g_debug_log_lock;
+        constinit KSpinLock g_debug_log_lock;
-        bool      g_initialized_impl;
+        constinit bool      g_initialized_impl;
 
         /* NOTE: Nintendo's print buffer is size 0x100. */
-        char g_print_buffer[0x400];
+        constinit char g_print_buffer[0x400];
 
         void PutString(const char *str) {
             /* Only print if the implementation is initialized. */
-            if (!g_initialized_impl) {
+            if (AMS_UNLIKELY(!g_initialized_impl)) {
                 return;
             }
 
@@ -73,6 +73,20 @@ namespace ams::kern {
 
         #endif
 
+        ALWAYS_INLINE void FormatU64(char * const dst, u64 value) {
+            /* Adjust, so that we can print the value backwards. */
+            char *cur = dst + 2 * sizeof(value);
+
+            /* Format the value in (as %016lx) */
+            while (cur > dst) {
+                /* Extract the digit. */
+                const auto digit = value & 0xF;
+                value >>= 4;
+
+                *(--cur) = (digit <= 9) ? ('0' + digit) : ('a' + digit - 10);
+            }
+        }
+
     }
 
     void KDebugLog::Initialize() {
@@ -109,6 +123,33 @@ namespace ams::kern {
         ::ams::util::TVSNPrintf(dst, dst_size, format, vl);
     }
 
+    void KDebugLog::LogException(const char *str) {
+        if (KTargetSystem::IsDebugLoggingEnabled()) {
+            /* Get the current program ID. */
+            /* NOTE: Nintendo does this after printing the string, */
+            /* but it seems wise to avoid holding the lock/disabling interrupts */
+            /* for longer than is strictly necessary. */
+            char suffix[18];
+            if (const auto *cur_process = GetCurrentProcessPointer(); AMS_LIKELY(cur_process != nullptr)) {
+                FormatU64(suffix, cur_process->GetProgramId());
+                suffix[16] = '\n';
+                suffix[17] = '\x00';
+            } else {
+                suffix[0] = '\n';
+                suffix[1] = '\x00';
+            }
+
+            KScopedInterruptDisable di;
+            KScopedSpinLock lk(g_debug_log_lock);
+
+            /* Log the string. */
+            PutString(str);
+
+            /* Log the program id (and newline) suffix. */
+            PutString(suffix);
+        }
+    }
+
     Result KDebugLog::PrintUserString(ams::kern::svc::KUserPointer<const char *> user_str, size_t len) {
         /* If printing is enabled, print the user string. */
         #if defined(MESOSPHERE_ENABLE_DEBUG_PRINT)

libraries/libmesosphere/source/kern_initial_process.cpp

@@ -25,18 +25,18 @@ namespace ams::kern {
             s32 priority;
         };
 
+        constinit KPhysicalAddress g_initial_process_binary_phys_addr = Null<KPhysicalAddress>;
         constinit KVirtualAddress g_initial_process_binary_address = Null<KVirtualAddress>;
         constinit InitialProcessBinaryHeader g_initial_process_binary_header = {};
         constinit size_t g_initial_process_secure_memory_size = 0;
         constinit u64 g_initial_process_id_min = std::numeric_limits<u64>::max();
         constinit u64 g_initial_process_id_max = std::numeric_limits<u64>::min();
 
-        void LoadInitialProcessBinaryHeader(KVirtualAddress virt_addr = Null<KVirtualAddress>) {
+        void LoadInitialProcessBinaryHeader() {
             if (g_initial_process_binary_header.magic != InitialProcessBinaryMagic) {
-                /* Get the virtual address, if it's not overridden. */
+                /* Get the virtual address. */
-                if (virt_addr == Null<KVirtualAddress>) {
+                MESOSPHERE_INIT_ABORT_UNLESS(g_initial_process_binary_phys_addr != Null<KPhysicalAddress>);
-                    virt_addr = GetInitialProcessBinaryAddress();
+                const KVirtualAddress virt_addr = KMemoryLayout::GetLinearVirtualAddress(g_initial_process_binary_phys_addr);
-                }
 
                 /* Copy and validate the header. */
                 g_initial_process_binary_header = *GetPointer<InitialProcessBinaryHeader>(virt_addr);
@@ -101,7 +101,7 @@ namespace ams::kern {
                 /* If we crossed a page boundary, free the pages we're done using. */
                 if (KVirtualAddress aligned_current = util::AlignDown(GetInteger(current), PageSize); aligned_current != data) {
                     const size_t freed_size = data - aligned_current;
-                    Kernel::GetMemoryManager().Close(aligned_current, freed_size / PageSize);
+                    Kernel::GetMemoryManager().Close(KMemoryLayout::GetLinearPhysicalAddress(aligned_current), freed_size / PageSize);
                     Kernel::GetSystemResourceLimit().Release(ams::svc::LimitableResource_PhysicalMemoryMax, freed_size);
                 }
 
@@ -114,7 +114,7 @@ namespace ams::kern {
                 const size_t binary_pages = binary_size / PageSize;
 
                 /* Get the pool for both the current (compressed) image, and the decompressed process. */
-                const auto src_pool = Kernel::GetMemoryManager().GetPool(data);
+                const auto src_pool = Kernel::GetMemoryManager().GetPool(KMemoryLayout::GetLinearPhysicalAddress(data));
                 const auto dst_pool = reader.UsesSecureMemory() ? secure_pool : unsafe_pool;
 
                 /* Determine the process size, and how much memory isn't already reserved. */
@@ -128,19 +128,19 @@ namespace ams::kern {
                 KProcess *new_process = nullptr;
                 {
                     /* Make page groups to represent the data. */
-                    KPageGroup pg(std::addressof(Kernel::GetBlockInfoManager()));
+                    KPageGroup pg(std::addressof(Kernel::GetSystemBlockInfoManager()));
-                    KPageGroup workaround_pg(std::addressof(Kernel::GetBlockInfoManager()));
+                    KPageGroup workaround_pg(std::addressof(Kernel::GetSystemBlockInfoManager()));
 
                     /* Populate the page group to represent the data. */
                     {
                         /* Allocate the previously unreserved pages. */
-                        KPageGroup unreserve_pg(std::addressof(Kernel::GetBlockInfoManager()));
+                        KPageGroup unreserve_pg(std::addressof(Kernel::GetSystemBlockInfoManager()));
                         MESOSPHERE_R_ABORT_UNLESS(Kernel::GetMemoryManager().AllocateAndOpen(std::addressof(unreserve_pg), unreserved_size / PageSize, KMemoryManager::EncodeOption(dst_pool, KMemoryManager::Direction_FromFront)));
 
                         /* Add the previously reserved pages. */
                         if (src_pool == dst_pool && binary_pages != 0) {
                             /* NOTE: Nintendo does not check the result of this operation. */
-                            pg.AddBlock(data, binary_pages);
+                            pg.AddBlock(KMemoryLayout::GetLinearPhysicalAddress(data), binary_pages);
                         }
 
                         /* Add the previously unreserved pages. */
@@ -176,7 +176,7 @@ namespace ams::kern {
                         } else {
                             if (src_pool != dst_pool) {
                                 std::memcpy(GetVoidPointer(temp_address + process_size - binary_size), GetVoidPointer(data), aligned_size);
-                                Kernel::GetMemoryManager().Close(data, aligned_size / PageSize);
+                                Kernel::GetMemoryManager().Close(KMemoryLayout::GetLinearPhysicalAddress(data), aligned_size / PageSize);
                             }
                         }
 
@@ -218,7 +218,7 @@ namespace ams::kern {
 
                                     const size_t cur_pages = std::min(block_remaining, work_remaining);
                                     const size_t cur_size  = cur_pages * PageSize;
-                                    std::memcpy(GetVoidPointer(block_address), GetVoidPointer(work_address), cur_size);
+                                    std::memcpy(GetVoidPointer(KMemoryLayout::GetLinearVirtualAddress(block_address)), GetVoidPointer(KMemoryLayout::GetLinearVirtualAddress(work_address)), cur_size);
 
                                     block_address += cur_size;
                                     work_address  += cur_size;
@@ -268,15 +268,23 @@ namespace ams::kern {
             {
                 const size_t remaining_size  = util::AlignUp(GetInteger(g_initial_process_binary_address) + g_initial_process_binary_header.size, PageSize) - util::AlignDown(GetInteger(current), PageSize);
                 const size_t remaining_pages = remaining_size / PageSize;
-                Kernel::GetMemoryManager().Close(util::AlignDown(GetInteger(current), PageSize), remaining_pages);
+                Kernel::GetMemoryManager().Close(KMemoryLayout::GetLinearPhysicalAddress(util::AlignDown(GetInteger(current), PageSize)), remaining_pages);
                 Kernel::GetSystemResourceLimit().Release(ams::svc::LimitableResource_PhysicalMemoryMax, remaining_size);
             }
         }
 
-        ALWAYS_INLINE KVirtualAddress GetInitialProcessBinaryAddress(KVirtualAddress pool_end) {
+    }
-            return pool_end - InitialProcessBinarySizeMax;
-        }
 
+    void SetInitialProcessBinaryPhysicalAddress(KPhysicalAddress phys_addr) {
+        MESOSPHERE_INIT_ABORT_UNLESS(g_initial_process_binary_phys_addr == Null<KPhysicalAddress>);
+
+        g_initial_process_binary_phys_addr = phys_addr;
+    }
+
+    KPhysicalAddress GetInitialProcessBinaryPhysicalAddress() {
+        MESOSPHERE_INIT_ABORT_UNLESS(g_initial_process_binary_phys_addr != Null<KPhysicalAddress>);
+
+        return g_initial_process_binary_phys_addr;
     }
 
     u64 GetInitialProcessIdMin() {
@@ -287,15 +295,6 @@ namespace ams::kern {
         return g_initial_process_id_max;
     }
 
-    KVirtualAddress GetInitialProcessBinaryAddress() {
-        /* Get, validate the pool region. */
-        const auto *pool_region = KMemoryLayout::GetVirtualMemoryRegionTree().FindLastDerived(KMemoryRegionType_VirtualDramUserPool);
-        MESOSPHERE_INIT_ABORT_UNLESS(pool_region != nullptr);
-        MESOSPHERE_INIT_ABORT_UNLESS(pool_region->GetEndAddress() != 0);
-        MESOSPHERE_ABORT_UNLESS(pool_region->GetSize() >= InitialProcessBinarySizeMax);
-        return GetInitialProcessBinaryAddress(pool_region->GetEndAddress());
-    }
-
     size_t GetInitialProcessesSecureMemorySize() {
         LoadInitialProcessBinaryHeader();
 
@@ -312,7 +311,7 @@ namespace ams::kern {
 
             /* The initial process binary is potentially over-allocated, so free any extra pages. */
             if (total_size < InitialProcessBinarySizeMax) {
-                Kernel::GetMemoryManager().Close(g_initial_process_binary_address + total_size, (InitialProcessBinarySizeMax - total_size) / PageSize);
+                Kernel::GetMemoryManager().Close(KMemoryLayout::GetLinearPhysicalAddress(g_initial_process_binary_address + total_size), (InitialProcessBinarySizeMax - total_size) / PageSize);
             }
 
             return total_size;
@@ -321,10 +320,6 @@ namespace ams::kern {
         }
     }
 
-    void LoadInitialProcessBinaryHeaderDeprecated(KPhysicalAddress pool_end) {
-        LoadInitialProcessBinaryHeader(GetInitialProcessBinaryAddress(KMemoryLayout::GetLinearVirtualAddress(pool_end)));
-    }
-
     void CreateAndRunInitialProcesses() {
         /* Allocate space for the processes. */
         InitialProcessInfo *infos = static_cast<InitialProcessInfo *>(__builtin_alloca(sizeof(InitialProcessInfo) * g_initial_process_binary_header.num_processes));

libraries/libmesosphere/source/kern_k_address_arbiter.cpp

@@ -43,6 +43,24 @@ namespace ams::kern {
             return UserspaceAccess::UpdateIfEqualAtomic(out, GetPointer<s32>(address), value, new_value);
         }
 
+        class ThreadQueueImplForKAddressArbiter final : public KThreadQueue {
+            private:
+                KAddressArbiter::ThreadTree *m_tree;
+            public:
+                constexpr ThreadQueueImplForKAddressArbiter(KAddressArbiter::ThreadTree *t) : KThreadQueue(), m_tree(t) { /* ... */ }
+
+                virtual void CancelWait(KThread *waiting_thread, Result wait_result, bool cancel_timer_task) override {
+                    /* If the thread is waiting on an address arbiter, remove it from the tree. */
+                    if (waiting_thread->IsWaitingForAddressArbiter()) {
+                        m_tree->erase(m_tree->iterator_to(*waiting_thread));
+                        waiting_thread->ClearAddressArbiter();
+                    }
+
+                    /* Invoke the base cancel wait handler. */
+                    KThreadQueue::CancelWait(waiting_thread, wait_result, cancel_timer_task);
+                }
+        };
+
     }
 
     Result KAddressArbiter::Signal(uintptr_t addr, s32 count) {
@@ -53,14 +71,14 @@ namespace ams::kern {
 
             auto it = m_tree.nfind_key({ addr, -1 });
             while ((it != m_tree.end()) && (count <= 0 || num_waiters < count) && (it->GetAddressArbiterKey() == addr)) {
+                /* End the thread's wait. */
                 KThread *target_thread = std::addressof(*it);
-                target_thread->SetSyncedObject(nullptr, ResultSuccess());
+                target_thread->EndWait(ResultSuccess());
 
-                AMS_ASSERT(target_thread->IsWaitingForAddressArbiter());
+                MESOSPHERE_ASSERT(target_thread->IsWaitingForAddressArbiter());
-                target_thread->Wakeup();
+                target_thread->ClearAddressArbiter();
 
                 it = m_tree.erase(it);
-                target_thread->ClearAddressArbiter();
                 ++num_waiters;
             }
         }
@@ -80,14 +98,14 @@ namespace ams::kern {
 
             auto it = m_tree.nfind_key({ addr, -1 });
             while ((it != m_tree.end()) && (count <= 0 || num_waiters < count) && (it->GetAddressArbiterKey() == addr)) {
+                /* End the thread's wait. */
                 KThread *target_thread = std::addressof(*it);
-                target_thread->SetSyncedObject(nullptr, ResultSuccess());
+                target_thread->EndWait(ResultSuccess());
 
-                AMS_ASSERT(target_thread->IsWaitingForAddressArbiter());
+                MESOSPHERE_ASSERT(target_thread->IsWaitingForAddressArbiter());
-                target_thread->Wakeup();
+                target_thread->ClearAddressArbiter();
 
                 it = m_tree.erase(it);
-                target_thread->ClearAddressArbiter();
                 ++num_waiters;
             }
         }
@@ -142,14 +160,14 @@ namespace ams::kern {
             R_UNLESS(user_value == value, svc::ResultInvalidState());
 
             while ((it != m_tree.end()) && (count <= 0 || num_waiters < count) && (it->GetAddressArbiterKey() == addr)) {
+                /* End the thread's wait. */
                 KThread *target_thread = std::addressof(*it);
-                target_thread->SetSyncedObject(nullptr, ResultSuccess());
+                target_thread->EndWait(ResultSuccess());
 
-                AMS_ASSERT(target_thread->IsWaitingForAddressArbiter());
+                MESOSPHERE_ASSERT(target_thread->IsWaitingForAddressArbiter());
-                target_thread->Wakeup();
+                target_thread->ClearAddressArbiter();
 
                 it = m_tree.erase(it);
-                target_thread->ClearAddressArbiter();
                 ++num_waiters;
             }
         }
@@ -160,6 +178,7 @@ namespace ams::kern {
         /* Prepare to wait. */
         KThread *cur_thread = GetCurrentThreadPointer();
         KHardwareTimer *timer;
+        ThreadQueueImplForKAddressArbiter wait_queue(std::addressof(m_tree));
 
         {
             KScopedSchedulerLockAndSleep slp(std::addressof(timer), cur_thread, timeout);
@@ -170,9 +189,6 @@ namespace ams::kern {
                 return svc::ResultTerminationRequested();
             }
 
-            /* Set the synced object. */
-            cur_thread->SetSyncedObject(nullptr, ams::svc::ResultTimedOut());
-
             /* Read the value from userspace. */
             s32 user_value;
             bool succeeded;
@@ -202,33 +218,21 @@ namespace ams::kern {
             /* Set the arbiter. */
             cur_thread->SetAddressArbiter(std::addressof(m_tree), addr);
             m_tree.insert(*cur_thread);
-            cur_thread->SetState(KThread::ThreadState_Waiting);
+
+            /* Wait for the thread to finish. */
+            wait_queue.SetHardwareTimer(timer);
+            cur_thread->BeginWait(std::addressof(wait_queue));
         }
 
-        /* Cancel the timer wait. */
+        /* Get the wait result. */
-        if (timer != nullptr) {
+        return cur_thread->GetWaitResult();
-            timer->CancelTask(cur_thread);
-        }
-
-        /* Remove from the address arbiter. */
-        {
-            KScopedSchedulerLock sl;
-
-            if (cur_thread->IsWaitingForAddressArbiter()) {
-                m_tree.erase(m_tree.iterator_to(*cur_thread));
-                cur_thread->ClearAddressArbiter();
-            }
-        }
-
-        /* Get the result. */
-        KSynchronizationObject *dummy;
-        return cur_thread->GetWaitResult(std::addressof(dummy));
     }
 
     Result KAddressArbiter::WaitIfEqual(uintptr_t addr, s32 value, s64 timeout) {
         /* Prepare to wait. */
         KThread *cur_thread = GetCurrentThreadPointer();
         KHardwareTimer *timer;
+        ThreadQueueImplForKAddressArbiter wait_queue(std::addressof(m_tree));
 
         {
             KScopedSchedulerLockAndSleep slp(std::addressof(timer), cur_thread, timeout);
@@ -239,9 +243,6 @@ namespace ams::kern {
                 return svc::ResultTerminationRequested();
             }
 
-            /* Set the synced object. */
-            cur_thread->SetSyncedObject(nullptr, ams::svc::ResultTimedOut());
-
             /* Read the value from userspace. */
             s32 user_value;
             if (!ReadFromUser(std::addressof(user_value), addr)) {
@@ -264,27 +265,14 @@ namespace ams::kern {
             /* Set the arbiter. */
             cur_thread->SetAddressArbiter(std::addressof(m_tree), addr);
             m_tree.insert(*cur_thread);
-            cur_thread->SetState(KThread::ThreadState_Waiting);
+
+            /* Wait for the thread to finish. */
+            wait_queue.SetHardwareTimer(timer);
+            cur_thread->BeginWait(std::addressof(wait_queue));
         }
 
-        /* Cancel the timer wait. */
+        /* Get the wait result. */
-        if (timer != nullptr) {
+        return cur_thread->GetWaitResult();
-            timer->CancelTask(cur_thread);
-        }
-
-        /* Remove from the address arbiter. */
-        {
-            KScopedSchedulerLock sl;
-
-            if (cur_thread->IsWaitingForAddressArbiter()) {
-                m_tree.erase(m_tree.iterator_to(*cur_thread));
-                cur_thread->ClearAddressArbiter();
-            }
-        }
-
-        /* Get the result. */
-        KSynchronizationObject *dummy;
-        return cur_thread->GetWaitResult(std::addressof(dummy));
     }
 
 }

libraries/libmesosphere/source/kern_k_class_token.cpp

@@ -37,13 +37,12 @@ namespace ams::kern {
     static_assert(ClassToken<KSession>                 == 0b00011001'00000000);
     static_assert(ClassToken<KSharedMemory>            == 0b00101001'00000000);
     static_assert(ClassToken<KEvent>                   == 0b01001001'00000000);
-    static_assert(ClassToken<KWritableEvent>           == 0b10001001'00000000);
+    static_assert(ClassToken<KLightClientSession>      == 0b10001001'00000000);
-    static_assert(ClassToken<KLightClientSession>      == 0b00110001'00000000);
+    static_assert(ClassToken<KLightServerSession>      == 0b00110001'00000000);
-    static_assert(ClassToken<KLightServerSession>      == 0b01010001'00000000);
+    static_assert(ClassToken<KTransferMemory>          == 0b01010001'00000000);
-    static_assert(ClassToken<KTransferMemory>          == 0b10010001'00000000);
+    static_assert(ClassToken<KDeviceAddressSpace>      == 0b10010001'00000000);
-    static_assert(ClassToken<KDeviceAddressSpace>      == 0b01100001'00000000);
+    static_assert(ClassToken<KSessionRequest>          == 0b01100001'00000000);
-    static_assert(ClassToken<KSessionRequest>          == 0b10100001'00000000);
+    static_assert(ClassToken<KCodeMemory>              == 0b10100001'00000000);
-    static_assert(ClassToken<KCodeMemory>              == 0b11000001'00000000);
 
     /* Ensure that the token hierarchy is correct. */
 
@@ -67,13 +66,12 @@ namespace ams::kern {
     static_assert(ClassToken<KSession>                 == ((0b00011001 << 8) | ClassToken<KAutoObject>));
     static_assert(ClassToken<KSharedMemory>            == ((0b00101001 << 8) | ClassToken<KAutoObject>));
     static_assert(ClassToken<KEvent>                   == ((0b01001001 << 8) | ClassToken<KAutoObject>));
-    static_assert(ClassToken<KWritableEvent>           == ((0b10001001 << 8) | ClassToken<KAutoObject>));
+    static_assert(ClassToken<KLightClientSession>      == ((0b10001001 << 8) | ClassToken<KAutoObject>));
-    static_assert(ClassToken<KLightClientSession>      == ((0b00110001 << 8) | ClassToken<KAutoObject>));
+    static_assert(ClassToken<KLightServerSession>      == ((0b00110001 << 8) | ClassToken<KAutoObject>));
-    static_assert(ClassToken<KLightServerSession>      == ((0b01010001 << 8) | ClassToken<KAutoObject>));
+    static_assert(ClassToken<KTransferMemory>          == ((0b01010001 << 8) | ClassToken<KAutoObject>));
-    static_assert(ClassToken<KTransferMemory>          == ((0b10010001 << 8) | ClassToken<KAutoObject>));
+    static_assert(ClassToken<KDeviceAddressSpace>      == ((0b10010001 << 8) | ClassToken<KAutoObject>));
-    static_assert(ClassToken<KDeviceAddressSpace>      == ((0b01100001 << 8) | ClassToken<KAutoObject>));
+    static_assert(ClassToken<KSessionRequest>          == ((0b01100001 << 8) | ClassToken<KAutoObject>));
-    static_assert(ClassToken<KSessionRequest>          == ((0b10100001 << 8) | ClassToken<KAutoObject>));
+    static_assert(ClassToken<KCodeMemory>              == ((0b10100001 << 8) | ClassToken<KAutoObject>));
-    static_assert(ClassToken<KCodeMemory>              == ((0b11000001 << 8) | ClassToken<KAutoObject>));
 
     /* Ensure that the token hierarchy reflects the class hierarchy. */
 
@@ -96,7 +94,6 @@ namespace ams::kern {
     static_assert(std::is_final<KSession>::value                  && std::is_base_of<KAutoObject, KSession>::value);
     static_assert(std::is_final<KSharedMemory>::value             && std::is_base_of<KAutoObject, KSharedMemory>::value);
     static_assert(std::is_final<KEvent>::value                    && std::is_base_of<KAutoObject, KEvent>::value);
-    static_assert(std::is_final<KWritableEvent>::value            && std::is_base_of<KAutoObject, KWritableEvent>::value);
     static_assert(std::is_final<KLightClientSession>::value       && std::is_base_of<KAutoObject, KLightClientSession>::value);
     static_assert(std::is_final<KLightServerSession>::value       && std::is_base_of<KAutoObject, KLightServerSession>::value);
     static_assert(std::is_final<KTransferMemory>::value           && std::is_base_of<KAutoObject, KTransferMemory>::value);

libraries/libmesosphere/source/kern_k_client_port.cpp

@@ -62,11 +62,46 @@ namespace ams::kern {
     Result KClientPort::CreateSession(KClientSession **out) {
         MESOSPHERE_ASSERT_THIS();
 
+        /* Declare the session we're going to allocate. */
+        KSession *session;
+
         /* Reserve a new session from the resource limit. */
         KScopedResourceReservation session_reservation(GetCurrentProcessPointer(), ams::svc::LimitableResource_SessionCountMax);
-        R_UNLESS(session_reservation.Succeeded(), svc::ResultLimitReached());
+        if (session_reservation.Succeeded()) {
+            /* Allocate a session normally. */
+            session = KSession::Create();
+        } else {
+            /* We couldn't reserve a session. Check that we support dynamically expanding the resource limit. */
+            R_UNLESS(GetCurrentProcess().GetResourceLimit() == std::addressof(Kernel::GetSystemResourceLimit()), svc::ResultLimitReached());
+            R_UNLESS(KTargetSystem::IsDynamicResourceLimitsEnabled(),                                            svc::ResultLimitReached());
+
+            /* Try to allocate a session from unused slab memory. */
+            session = KSession::CreateFromUnusedSlabMemory();
+            R_UNLESS(session != nullptr, svc::ResultLimitReached());
+
+            /* Ensure that if we fail to allocate our session requests, we close the session we created. */
+            auto session_guard = SCOPE_GUARD { session->Close(); };
+            {
+                /* We want to add two KSessionRequests to the heap, to prevent request exhaustion. */
+                for (size_t i = 0; i < 2; ++i) {
+                    KSessionRequest *request = KSessionRequest::CreateFromUnusedSlabMemory();
+                    R_UNLESS(request != nullptr, svc::ResultLimitReached());
+
+                    request->Close();
+                }
+            }
+            session_guard.Cancel();
+
+            /* We successfully allocated a session, so add the object we allocated to the resource limit. */
+            Kernel::GetSystemResourceLimit().Add(ams::svc::LimitableResource_SessionCountMax, 1);
+        }
+
+        /* Check that we successfully created a session. */
+        R_UNLESS(session != nullptr, svc::ResultOutOfResource());
+
 
         /* Update the session counts. */
+        auto count_guard = SCOPE_GUARD { session->Close(); };
         {
             /* Atomically increment the number of sessions. */
             s32 new_sessions;
@@ -90,18 +125,7 @@ namespace ams::kern {
                 } while (!m_peak_sessions.compare_exchange_weak(peak, new_sessions, std::memory_order_relaxed));
             }
         }
-
+        count_guard.Cancel();
-        /* Create a new session. */
-        KSession *session = KSession::Create();
-        if (session == nullptr) {
-            /* Decrement the session count. */
-            const auto prev = m_num_sessions--;
-            if (prev == m_max_sessions) {
-                this->NotifyAvailable();
-            }
-
-            return svc::ResultOutOfResource();
-        }
 
         /* Initialize the session. */
         session->Initialize(this, m_parent->GetName());
@@ -128,11 +152,32 @@ namespace ams::kern {
     Result KClientPort::CreateLightSession(KLightClientSession **out) {
         MESOSPHERE_ASSERT_THIS();
 
+        /* Declare the session we're going to allocate. */
+        KLightSession *session;
+
         /* Reserve a new session from the resource limit. */
         KScopedResourceReservation session_reservation(GetCurrentProcessPointer(), ams::svc::LimitableResource_SessionCountMax);
-        R_UNLESS(session_reservation.Succeeded(), svc::ResultLimitReached());
+        if (session_reservation.Succeeded()) {
+            /* Allocate a session normally. */
+            session = KLightSession::Create();
+        } else {
+            /* We couldn't reserve a session. Check that we support dynamically expanding the resource limit. */
+            R_UNLESS(GetCurrentProcess().GetResourceLimit() == std::addressof(Kernel::GetSystemResourceLimit()), svc::ResultLimitReached());
+            R_UNLESS(KTargetSystem::IsDynamicResourceLimitsEnabled(),                                            svc::ResultLimitReached());
+
+            /* Try to allocate a session from unused slab memory. */
+            session = KLightSession::CreateFromUnusedSlabMemory();
+            R_UNLESS(session != nullptr, svc::ResultLimitReached());
+
+            /* We successfully allocated a session, so add the object we allocated to the resource limit. */
+            Kernel::GetSystemResourceLimit().Add(ams::svc::LimitableResource_SessionCountMax, 1);
+        }
+
+        /* Check that we successfully created a session. */
+        R_UNLESS(session != nullptr, svc::ResultOutOfResource());
 
         /* Update the session counts. */
+        auto count_guard = SCOPE_GUARD { session->Close(); };
         {
             /* Atomically increment the number of sessions. */
             s32 new_sessions;
@@ -156,18 +201,7 @@ namespace ams::kern {
                 } while (!m_peak_sessions.compare_exchange_weak(peak, new_sessions, std::memory_order_relaxed));
             }
         }
-
+        count_guard.Cancel();
-        /* Create a new session. */
-        KLightSession *session = KLightSession::Create();
-        if (session == nullptr) {
-            /* Decrement the session count. */
-            const auto prev = m_num_sessions--;
-            if (prev == m_max_sessions) {
-                this->NotifyAvailable();
-            }
-
-            return svc::ResultOutOfResource();
-        }
 
         /* Initialize the session. */
         session->Initialize(this, m_parent->GetName());