kernel patches: fuck armconverter.com

Makefile

@@ -141,15 +141,16 @@ dist: dist-no-debug
 	cp stratosphere/creport/creport.elf atmosphere-$(AMSVER)-debug/creport.elf
 	cp stratosphere/dmnt/dmnt.elf atmosphere-$(AMSVER)-debug/dmnt.elf
 	cp stratosphere/eclct.stub/eclct.stub.elf atmosphere-$(AMSVER)-debug/eclct.stub.elf
+	cp stratosphere/erpt/erpt.elf atmosphere-$(AMSVER)-debug/erpt.elf
 	cp stratosphere/fatal/fatal.elf atmosphere-$(AMSVER)-debug/fatal.elf
+	cp stratosphere/jpegdec/jpegdec.elf atmosphere-$(AMSVER)-debug/jpegdec.elf
 	cp stratosphere/loader/loader.elf atmosphere-$(AMSVER)-debug/loader.elf
+	cp stratosphere/ncm/ncm.elf atmosphere-$(AMSVER)-debug/ncm.elf
+	cp stratosphere/pgl/pgl.elf atmosphere-$(AMSVER)-debug/pgl.elf
 	cp stratosphere/pm/pm.elf atmosphere-$(AMSVER)-debug/pm.elf
 	cp stratosphere/ro/ro.elf atmosphere-$(AMSVER)-debug/ro.elf
 	cp stratosphere/sm/sm.elf atmosphere-$(AMSVER)-debug/sm.elf
 	cp stratosphere/spl/spl.elf atmosphere-$(AMSVER)-debug/spl.elf
-	cp stratosphere/erpt/erpt.elf atmosphere-$(AMSVER)-debug/erpt.elf
-	cp stratosphere/jpegdec/jpegdec.elf atmosphere-$(AMSVER)-debug/jpegdec.elf
-	cp stratosphere/pgl/pgl.elf atmosphere-$(AMSVER)-debug/pgl.elf
 	cp troposphere/daybreak/daybreak.elf atmosphere-$(AMSVER)-debug/daybreak.elf
 	cd atmosphere-$(AMSVER)-debug; zip -r ../atmosphere-$(AMSVER)-debug.zip ./*; cd ../;
 	rm -r atmosphere-$(AMSVER)-debug

emummc/source/FS/FS_offsets.c

@@ -49,6 +49,8 @@
 #include "offsets/1000_exfat.h"
 #include "offsets/1020.h"
 #include "offsets/1020_exfat.h"
+#include "offsets/1100.h"
+#include "offsets/1100_exfat.h"
 #include "../utils/fatal.h"
 
 #define GET_OFFSET_STRUCT_NAME(vers) g_offsets##vers
@@ -108,6 +110,8 @@ DEFINE_OFFSET_STRUCT(_1000);
 DEFINE_OFFSET_STRUCT(_1000_EXFAT);
 DEFINE_OFFSET_STRUCT(_1020);
 DEFINE_OFFSET_STRUCT(_1020_EXFAT);
+DEFINE_OFFSET_STRUCT(_1100);
+DEFINE_OFFSET_STRUCT(_1100_EXFAT);
 
 const fs_offsets_t *get_fs_offsets(enum FS_VER version) {
     switch (version) {
@@ -177,6 +181,10 @@ const fs_offsets_t *get_fs_offsets(enum FS_VER version) {
             return &(GET_OFFSET_STRUCT_NAME(_1020));
         case FS_VER_10_2_0_EXFAT:
             return &(GET_OFFSET_STRUCT_NAME(_1020_EXFAT));
+        case FS_VER_11_0_0:
+            return &(GET_OFFSET_STRUCT_NAME(_1100));
+        case FS_VER_11_0_0_EXFAT:
+            return &(GET_OFFSET_STRUCT_NAME(_1100_EXFAT));
         default:
             fatal_abort(Fatal_UnknownVersion);
     }

emummc/source/FS/FS_versions.h

@@ -71,6 +71,9 @@ enum FS_VER
     FS_VER_10_2_0,
     FS_VER_10_2_0_EXFAT,
 
+    FS_VER_11_0_0,
+    FS_VER_11_0_0_EXFAT,
+
     FS_VER_MAX,
 };
 

emummc/source/FS/offsets/1100.h

@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2019 m4xw <m4x@m4xw.net>
+ * Copyright (c) 2019 Atmosphere-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef __FS_1100_H__
+#define __FS_1100_H__
+
+// Accessor vtable getters
+#define FS_OFFSET_1100_SDMMC_ACCESSOR_GC   0x156D90
+#define FS_OFFSET_1100_SDMMC_ACCESSOR_SD   0x154F40
+#define FS_OFFSET_1100_SDMMC_ACCESSOR_NAND 0x1500F0
+
+// Hooks
+#define FS_OFFSET_1100_SDMMC_WRAPPER_READ  0x14B990
+#define FS_OFFSET_1100_SDMMC_WRAPPER_WRITE 0x14BA70
+#define FS_OFFSET_1100_RTLD                0x688
+#define FS_OFFSET_1100_RTLD_DESTINATION    ((uintptr_t)(INT64_C(-0x3C)))
+
+#define FS_OFFSET_1100_CLKRST_SET_MIN_V_CLK_RATE 0x14AC40
+
+// Misc funcs
+#define FS_OFFSET_1100_LOCK_MUTEX          0x28FF0
+#define FS_OFFSET_1100_UNLOCK_MUTEX        0x29040
+
+#define FS_OFFSET_1100_SDMMC_WRAPPER_CONTROLLER_CLOSE 0x14B8F0
+
+// Misc Data
+#define FS_OFFSET_1100_SD_MUTEX            0xE323E8
+#define FS_OFFSET_1100_NAND_MUTEX          0xE2D338
+#define FS_OFFSET_1100_ACTIVE_PARTITION    0xE2D378
+#define FS_OFFSET_1100_SDMMC_DAS_HANDLE    0xE15D40
+
+// NOPs
+#define FS_OFFSET_1100_SD_DAS_INIT         0x273B4
+
+// Nintendo Paths
+#define FS_OFFSET_1100_NINTENDO_PATHS \
+{ \
+	{.opcode_reg = 3, .adrp_offset = 0x0006D944, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 3, .adrp_offset = 0x0007A3C0, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 3, .adrp_offset = 0x00080708, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 4, .adrp_offset = 0x00092198, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 0, .adrp_offset = 0, .add_rel_offset = 0}, \
+}
+
+#endif // __FS_1100_H__

emummc/source/FS/offsets/1100_exfat.h

@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2019 m4xw <m4x@m4xw.net>
+ * Copyright (c) 2019 Atmosphere-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef __FS_1100_EXFAT_H__
+#define __FS_1100_EXFAT_H__
+
+// Accessor vtable getters
+#define FS_OFFSET_1100_EXFAT_SDMMC_ACCESSOR_GC   0x156D90
+#define FS_OFFSET_1100_EXFAT_SDMMC_ACCESSOR_SD   0x154F40
+#define FS_OFFSET_1100_EXFAT_SDMMC_ACCESSOR_NAND 0x1500F0
+
+// Hooks
+#define FS_OFFSET_1100_EXFAT_SDMMC_WRAPPER_READ  0x14B990
+#define FS_OFFSET_1100_EXFAT_SDMMC_WRAPPER_WRITE 0x14BA70
+#define FS_OFFSET_1100_EXFAT_RTLD                0x688
+#define FS_OFFSET_1100_EXFAT_RTLD_DESTINATION    ((uintptr_t)(INT64_C(-0x3C)))
+
+#define FS_OFFSET_1100_EXFAT_CLKRST_SET_MIN_V_CLK_RATE 0x14AC40
+
+// Misc funcs
+#define FS_OFFSET_1100_EXFAT_LOCK_MUTEX          0x28FF0
+#define FS_OFFSET_1100_EXFAT_UNLOCK_MUTEX        0x29040
+
+#define FS_OFFSET_1100_EXFAT_SDMMC_WRAPPER_CONTROLLER_CLOSE 0x14B8F0
+
+// Misc Data
+#define FS_OFFSET_1100_EXFAT_SD_MUTEX            0xE403E8
+#define FS_OFFSET_1100_EXFAT_NAND_MUTEX          0xE3B338
+#define FS_OFFSET_1100_EXFAT_ACTIVE_PARTITION    0xE3B378
+#define FS_OFFSET_1100_EXFAT_SDMMC_DAS_HANDLE    0xE23D40
+
+// NOPs
+#define FS_OFFSET_1100_EXFAT_SD_DAS_INIT         0x273B4
+
+// Nintendo Paths
+#define FS_OFFSET_1100_EXFAT_NINTENDO_PATHS \
+{ \
+	{.opcode_reg = 3, .adrp_offset = 0x0006D944, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 3, .adrp_offset = 0x0007A3C0, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 3, .adrp_offset = 0x00080708, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 4, .adrp_offset = 0x00092198, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 0, .adrp_offset = 0, .add_rel_offset = 0}, \
+}
+
+#endif // __FS_1100_EXFAT_H__

fusee/fusee-secondary/src/emummc_cfg.h

@@ -88,6 +88,9 @@ typedef enum {
     FS_VER_10_2_0,
     FS_VER_10_2_0_EXFAT,
 
+    FS_VER_11_0_0,
+    FS_VER_11_0_0_EXFAT,
+
     FS_VER_MAX,
 } emummc_fs_ver_t;
 

fusee/fusee-secondary/src/ips.c

@@ -423,6 +423,9 @@ static const uint8_t g_fs_hashes[FS_VER_MAX][0x8] = {
 
     "\xA9\x52\xB6\x57\xAD\xF9\xC2\xBA", /* FS_VER_10_2_0 */
     "\x16\x0D\x3E\x10\x4E\xAD\x61\x76", /* FS_VER_10_2_0_EXFAT */
+
+    "\xE3\x99\x15\x6E\x84\x4E\xB0\xAA", /* FS_VER_11_0_0 */
+    "\x0B\xA1\x5B\xB3\x04\xB5\x05\x63", /* FS_VER_11_0_0_EXFAT */
 };
 
 kip1_header_t *apply_kip_ips_patches(kip1_header_t *kip, size_t kip_size, emummc_fs_ver_t *out_fs_ver) {

fusee/fusee-secondary/src/kernel_patches.c

@@ -541,6 +541,60 @@ static const instruction_t MAKE_KERNEL_PATCH_NAME(1000, proc_id_send)[] = {0xA9B
 static const uint8_t MAKE_KERNEL_PATTERN_NAME(1000, proc_id_recv)[] = {0x88, 0x03, 0x40, 0xF9, 0x08, 0x1D, 0x40, 0xF9, 0xE0, 0x03, 0x1C, 0xAA, 0x00, 0x01, 0x3F, 0xD6, 0xE8, 0x87, 0x40, 0xF9, 0x08, 0x49, 0x3A, 0x8B, 0x09, 0xFC, 0x60, 0xD3};
 static const instruction_t MAKE_KERNEL_PATCH_NAME(1000, proc_id_recv)[] = {0xA9BF2FEA, 0xF94067EB, 0x2A1A03EA, 0xD37EF54A, 0xF86A696A, 0x92FFFFE9, 0x8A090148, 0xD2FFFFE9, 0x8A09014A, 0xD2FFFFC9, 0xEB09015F, 0x54000100, 0xA9BF27E8, 0xF9400388, 0xF9401D08, 0xAA1C03E0, 0xD63F0100, 0xA8C127E8, 0xAA0003E8, 0xA8C12FEA, 0xAA0803E0};
 
+/*
+    stp x10, x11, [sp, #-0x10]!
+    ldr x11, [sp, #0x80]
+    mov w10, #3
+    lsl x10, x10, #2
+    ldr x10, [x11, x10]
+    mov x9, #0x0000ffffffffffff
+    and x8, x10, x9
+    mov x9, #0xffff000000000000
+    and x10, x10, x9
+    mov x9, #0xfffe000000000000
+    cmp x10, x9
+    beq #0x20
+
+    stp x8, x9, [sp, #-0x10]!
+    ldr x8, [x21]
+    ldr x8, [x8, #0x38]
+    mov x0, x21
+    blr x8
+    ldp x8, x9, [sp],#0x10
+    mov x8, x0
+
+    ldp x10, x11, [sp],#0x10
+    mov x0, x8
+*/
+static const uint8_t MAKE_KERNEL_PATTERN_NAME(1100, proc_id_send)[] = {0xE0, 0x03, 0x15, 0xAA, 0xA8, 0x02, 0x40, 0xF9, 0x08, 0x1D, 0x40, 0xF9, 0x00, 0x01, 0x3F, 0xD6, 0x88, 0x4A, 0x3C, 0x8B, 0x09, 0xFC, 0x60, 0xD3, 0x00, 0x25, 0x00, 0x29};
+static const instruction_t MAKE_KERNEL_PATCH_NAME(1100, proc_id_send)[] = {0xA9BF2FEA, 0xF94043EB, 0x5280006A, 0xD37EF54A, 0xF86A696A, 0x92FFFFE9, 0x8A090148, 0xD2FFFFE9, 0x8A09014A, 0xD2FFFFC9, 0xEB09015F, 0x54000100, 0xA9BF27E8, 0xF94002A8, 0xF9401D08, 0xAA1503E0, 0xD63F0100, 0xA8C127E8, 0xAA0003E8, 0xA8C12FEA, 0xAA0803E0};
+/*
+    stp x10, x11, [sp, #-0x10]!
+    ldr x11, [sp, #0xE0]
+    mov w10, #3
+    lsl x10, x10, #2
+    ldr x10, [x11, x10]
+    mov x9, #0x0000ffffffffffff
+    and x8, x10, x9
+    mov x9, #0xffff000000000000
+    and x10, x10, x9
+    mov x9, #0xfffe000000000000
+    cmp x10, x9
+    beq #0x20
+
+    stp x8, x9, [sp, #-0x10]!
+    ldr x8, [x24]
+    ldr x8, [x8, #0x38]
+    mov x0, x24
+    blr x8
+    ldp x8, x9, [sp],#0x10
+    mov x8, x0
+
+    ldp x10, x11, [sp],#0x10
+    mov x0, x8
+*/
+static const uint8_t MAKE_KERNEL_PATTERN_NAME(1100, proc_id_recv)[] = {0x08, 0x03, 0x40, 0xF9, 0xE0, 0x03, 0x18, 0xAA, 0x08, 0x1D, 0x40, 0xF9, 0x00, 0x01, 0x3F, 0xD6, 0xE8, 0x7F, 0x40, 0xF9, 0x09, 0xFC, 0x60, 0xD3, 0xEA, 0x5B, 0x40, 0xF9};
+static const instruction_t MAKE_KERNEL_PATCH_NAME(1100, proc_id_recv)[] = {0xA9BF2FEA, 0xF94073EB, 0x5280006A, 0xD37EF54A, 0xF86A696A, 0x92FFFFE9, 0x8A090148, 0xD2FFFFE9, 0x8A09014A, 0xD2FFFFC9, 0xEB09015F, 0x54000100, 0xA9BF27E8, 0xF9400308, 0xF9401D08, 0xAA1803E0, 0xD63F0100, 0xA8C127E8, 0xAA0003E8, 0xA8C12FEA, 0xAA0803E0};
 
 /* svcControlCodeMemory Patches */
 /* b.eq -> nop */
@@ -550,6 +604,7 @@ static const instruction_t MAKE_KERNEL_PATCH_NAME(700,  svc_control_codememory)[
 static const instruction_t MAKE_KERNEL_PATCH_NAME(800,  svc_control_codememory)[] = {MAKE_NOP};
 static const instruction_t MAKE_KERNEL_PATCH_NAME(900,  svc_control_codememory)[] = {MAKE_NOP};
 static const instruction_t MAKE_KERNEL_PATCH_NAME(1000, svc_control_codememory)[] = {MAKE_NOP};
+static const instruction_t MAKE_KERNEL_PATCH_NAME(1100, svc_control_codememory)[] = {MAKE_NOP};
 
 static const instruction_t MAKE_KERNEL_PATCH_NAME(500,  system_memory_increase)[] = {0x52A3C008}; /* MOV W8,   #0x1E000000 */
 static const instruction_t MAKE_KERNEL_PATCH_NAME(600,  system_memory_increase)[] = {0x52A3B008}; /* MOV W8,   #0x1D800000 */
@@ -557,6 +612,7 @@ static const instruction_t MAKE_KERNEL_PATCH_NAME(700,  system_memory_increase)[
 static const instruction_t MAKE_KERNEL_PATCH_NAME(800,  system_memory_increase)[] = {0x52A3B013}; /* MOV W19,  #0x1D800000 */
 static const instruction_t MAKE_KERNEL_PATCH_NAME(900,  system_memory_increase)[] = {0x52A3B013}; /* MOV W19,  #0x1D800000 */
 static const instruction_t MAKE_KERNEL_PATCH_NAME(1000, system_memory_increase)[] = {0x52A3B013}; /* MOV W19,  #0x1D800000 */
+static const instruction_t MAKE_KERNEL_PATCH_NAME(1100, system_memory_increase)[] = {0x52A3B015}; /* MOV W21,  #0x1D800000 */
 
 /* Hook Definitions. */
 static const kernel_patch_t g_kernel_patches_100[] = {
@@ -821,6 +877,35 @@ static const kernel_patch_t g_kernel_patches_1000[] = {
     }
 };
 
+static const kernel_patch_t g_kernel_patches_1100[] = {
+    {   /* Send Message Process ID Patch. */
+        .pattern_size = 0x1C,
+        .pattern = MAKE_KERNEL_PATTERN_NAME(1100, proc_id_send),
+        .pattern_hook_offset = 0x0,
+        .payload_num_instructions = sizeof(MAKE_KERNEL_PATCH_NAME(1100, proc_id_send))/sizeof(instruction_t),
+        .branch_back_offset = 0x10,
+        .payload = MAKE_KERNEL_PATCH_NAME(1100, proc_id_send)
+    },
+    {   /* Receive Message Process ID Patch. */
+        .pattern_size = 0x1C,
+        .pattern = MAKE_KERNEL_PATTERN_NAME(1100, proc_id_recv),
+        .pattern_hook_offset = 0x0,
+        .payload_num_instructions = sizeof(MAKE_KERNEL_PATCH_NAME(1100, proc_id_recv))/sizeof(instruction_t),
+        .branch_back_offset = 0x10,
+        .payload = MAKE_KERNEL_PATCH_NAME(1100, proc_id_recv)
+    },
+    {   /* svcControlCodeMemory Patch. */
+        .payload_num_instructions = sizeof(MAKE_KERNEL_PATCH_NAME(1100, svc_control_codememory))/sizeof(instruction_t),
+        .payload = MAKE_KERNEL_PATCH_NAME(1100, svc_control_codememory),
+        .patch_offset = 0x2FCE0,
+    },
+    {   /* System Memory Increase Patch. */
+        .payload_num_instructions = sizeof(MAKE_KERNEL_PATCH_NAME(1100, system_memory_increase))/sizeof(instruction_t),
+        .payload = MAKE_KERNEL_PATCH_NAME(1100, system_memory_increase),
+        .patch_offset = 0x490C4,
+    }
+};
+
 #define KERNEL_PATCHES(vers) .num_patches = sizeof(g_kernel_patches_##vers)/sizeof(kernel_patch_t), .patches = g_kernel_patches_##vers,
 
 /* Kernel Infos. */
@@ -906,6 +991,15 @@ static const kernel_info_t g_kernel_infos[] = {
         .embedded_ini_ptr = 0x178,
         .free_code_space_offset = 0x67790,
         KERNEL_PATCHES(1000)
+    },
+    {   /* 11.0.0. */
+        .hash = {0xC2, 0x0E, 0xB3, 0x1B, 0xBF, 0x0B, 0x82, 0xF3, 0x3D, 0xFD, 0x47, 0x04, 0xB4, 0x44, 0x38, 0x47, 0x64, 0xAB, 0xD8, 0x70, 0x2F, 0x0E, 0x0C, 0x37, 0x82, 0x28, 0x02, 0x24, 0xB8, 0x6E, 0xCE, 0x05, },
+        .hash_offset = 0x1C4,
+        .hash_size = 0x69000 - 0x1C4,
+        .embedded_ini_offset = 0x69000,
+        .embedded_ini_ptr = 0x180,
+        .free_code_space_offset = 0x49EE8,
+        KERNEL_PATCHES(1100)
     }
 };
 
@@ -967,13 +1061,6 @@ void package2_patch_kernel(void *_kernel, size_t *kernel_size, bool is_sd_kernel
     }
 
     if (kernel_info->embedded_ini_offset != 0) {
-        /* Copy in our kernel loader. */
-        const uint32_t kernel_ldr_offset = *((volatile uint64_t *)((uintptr_t)_kernel + kernel_info->embedded_ini_ptr + 8));
-        memcpy((void *)((uintptr_t)_kernel + kernel_ldr_offset), kernel_ldr_bin, kernel_ldr_bin_size);
-
-        /* Update size. */
-        *kernel_size = kernel_ldr_offset + kernel_ldr_bin_size;
-
         /* Set output INI ptr. */
         *out_ini1 = (void *)((uintptr_t)_kernel + kernel_info->embedded_ini_offset);
         *((volatile uint64_t *)((uintptr_t)_kernel + kernel_info->embedded_ini_ptr)) = (uint64_t)*kernel_size;
@@ -1003,7 +1090,7 @@ void package2_patch_kernel(void *_kernel, size_t *kernel_size, bool is_sd_kernel
 
         uint8_t *pattern_loc = search_pattern(kernel, *kernel_size, kernel_info->patches[i].pattern, kernel_info->patches[i].pattern_size);
         if (pattern_loc == NULL) {
-            /* TODO: Should we print an error/abort here? */
+            fatal_error("kernel_patcher: failed to identify patch location!\n");
             continue;
         }
         /* Patch kernel to branch to our hook at the desired place. */

fusee/fusee-secondary/src/nxboot.c

@@ -239,7 +239,9 @@ static bool is_nca_present(const char *nca_name) {
 static uint32_t nxboot_get_specific_target_firmware(uint32_t target_firmware){
     #define CHECK_NCA(NCA_ID, VERSION) do { if (is_nca_present(NCA_ID)) { return ATMOSPHERE_TARGET_FIRMWARE_##VERSION; } } while(0)
 
-    if (target_firmware >= ATMOSPHERE_TARGET_FIRMWARE_10_0_0) {
+    if (target_firmware >= ATMOSPHERE_TARGET_FIRMWARE_11_0_0) {
+        CHECK_NCA("594c90bcdbcccad6b062eadba0cd0e7e", 11_0_0);
+    } else if (target_firmware >= ATMOSPHERE_TARGET_FIRMWARE_10_0_0) {
         CHECK_NCA("26325de4db3909e0ef2379787c7e671d", 10_2_0);
         CHECK_NCA("5077973537f6735b564dd7475b779f87", 10_1_1); /* Exclusive to China. */
         CHECK_NCA("fd1faed0ca750700d254c0915b93d506", 10_1_0);
@@ -337,6 +339,8 @@ static uint32_t nxboot_get_target_firmware(const void *package1loader) {
                 return ATMOSPHERE_TARGET_FIRMWARE_9_1_0;
             } else if (memcmp(package1loader_header->build_timestamp, "20200303", 8) == 0) {
                 return ATMOSPHERE_TARGET_FIRMWARE_10_0_0;
+            } else if (memcmp(package1loader_header->build_timestamp, "20201030", 8) == 0) {
+                return ATMOSPHERE_TARGET_FIRMWARE_11_0_0;
             } else {
                 fatal_error("[NXBOOT] Unable to identify package1!\n");
             }
@@ -521,6 +525,10 @@ static void nxboot_configure_stratosphere(uint32_t target_firmware) {
         if (target_firmware >= ATMOSPHERE_TARGET_FIRMWARE_9_0_0 && !(fuse_get_reserved_odm(7) & ~0x000003FF)) {
             kip_patches_set_enable_nogc();
         }
+        /* Check if the fuses are < 11.0.0, but firmware is >= 11.0.0 */
+        if (target_firmware >= ATMOSPHERE_TARGET_FIRMWARE_11_0_0 && !(fuse_get_reserved_odm(7) & ~0x00001FFF)) {
+            kip_patches_set_enable_nogc();
+        }
     }
 }
 

fusee/fusee-secondary/src/package2.c

@@ -250,7 +250,7 @@ static bool package2_validate_metadata(package2_meta_t *metadata, uint8_t data[]
 
     /* Perform version checks. */
     /* We will be compatible with all package2s released before current, but not newer ones. */
-    if (metadata->version_max >= PACKAGE2_MINVER_THEORETICAL && metadata->version_min < PACKAGE2_MAXVER_1000_CURRENT) {
+    if (metadata->version_max >= PACKAGE2_MINVER_THEORETICAL && metadata->version_min < PACKAGE2_MAXVER_1100_CURRENT) {
         return true;
     }
 

fusee/fusee-secondary/src/package2.h

@@ -40,7 +40,8 @@
 #define PACKAGE2_MAXVER_810 0xB
 #define PACKAGE2_MAXVER_900 0xC
 #define PACKAGE2_MAXVER_910_920 0xD
-#define PACKAGE2_MAXVER_1000_CURRENT 0xE
+#define PACKAGE2_MAXVER_1000 0xE
+#define PACKAGE2_MAXVER_1100_CURRENT 0xF
 
 #define PACKAGE2_MINVER_100 0x3
 #define PACKAGE2_MINVER_200 0x4
@@ -54,7 +55,8 @@
 #define PACKAGE2_MINVER_810 0xC
 #define PACKAGE2_MINVER_900 0xD
 #define PACKAGE2_MINVER_910_920 0xE
-#define PACKAGE2_MINVER_1000_CURRENT 0xF
+#define PACKAGE2_MINVER_1000 0xF
+#define PACKAGE2_MINVER_1100_CURRENT 0x10
 
 #define NX_BOOTLOADER_PACKAGE2_LOAD_ADDRESS ((void *)(0xA9800000ull))
 

libraries/config/templates/mesosphere.mk

@@ -7,7 +7,7 @@ include  $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/../common.mk
 # options for code generation
 #---------------------------------------------------------------------------------
 export DEFINES     := $(ATMOSPHERE_DEFINES) -DATMOSPHERE_IS_MESOSPHERE
-export SETTINGS    := $(ATMOSPHERE_SETTINGS) -O2 -mgeneral-regs-only -ffixed-x18 -Wextra -Werror -fno-non-call-exceptions
+export SETTINGS    := $(ATMOSPHERE_SETTINGS) -Os -mgeneral-regs-only -ffixed-x18 -Wextra -Werror -fno-non-call-exceptions
 export CFLAGS      := $(ATMOSPHERE_CFLAGS) $(SETTINGS) $(DEFINES) $(INCLUDE)
 export CXXFLAGS    := $(CFLAGS) $(ATMOSPHERE_CXXFLAGS) -fno-use-cxa-atexit
 export ASFLAGS     := $(ATMOSPHERE_ASFLAGS) $(SETTINGS) $(DEFINES) $(INCLUDE)

libraries/libexosphere/include/exosphere/pkg2.hpp

@@ -24,7 +24,7 @@ namespace ams::pkg2 {
     constexpr inline int PayloadCount = 3;
 
     constexpr inline int MinimumValidDataVersion  = 0;   /* We allow older package2 to load; this value is currently 0x10 in Nintendo's code. */
-    constexpr inline int CurrentBootloaderVersion = 0xD;
+    constexpr inline int CurrentBootloaderVersion = 0xE;
 
     struct Package2Meta {
         using Magic = util::FourCC<'P','K','2','1'>;

libraries/libexosphere/source/fuse/fuse_api.cpp

@@ -165,6 +165,7 @@ namespace ams::fuse {
         }
 
         constexpr const TargetFirmware FuseVersionIncrementFirmwares[] = {
+            TargetFirmware_11_0_0,
             TargetFirmware_10_0_0,
             TargetFirmware_9_1_0,
             TargetFirmware_9_0_0,
@@ -207,7 +208,7 @@ namespace ams::fuse {
             return 0;
         }
 
-        static_assert(GetExpectedFuseVersionImpl(TargetFirmware_10_0_0)          == 13);
+        static_assert(GetExpectedFuseVersionImpl(TargetFirmware_11_0_0)          == 14);
         static_assert(GetExpectedFuseVersionImpl(TargetFirmware_1_0_0)           ==  1);
         static_assert(GetExpectedFuseVersionImpl(static_cast<TargetFirmware>(0)) ==  0);
 

libraries/libmesosphere/Makefile

@@ -11,7 +11,7 @@ include $(CURRENT_DIRECTORY)/../config/common.mk
 PRECOMPILED_HEADERS := include/mesosphere.hpp
 
 DEFINES     := $(ATMOSPHERE_DEFINES) -DATMOSPHERE_IS_MESOSPHERE
-SETTINGS    := $(ATMOSPHERE_SETTINGS) -O2 -mgeneral-regs-only -ffixed-x18 -Wextra -Werror -fno-non-call-exceptions
+SETTINGS    := $(ATMOSPHERE_SETTINGS) -Os -mgeneral-regs-only -ffixed-x18 -Wextra -Werror -fno-non-call-exceptions
 CFLAGS      := $(ATMOSPHERE_CFLAGS) $(SETTINGS) $(DEFINES) $(INCLUDE)
 CXXFLAGS    := $(CFLAGS) $(ATMOSPHERE_CXXFLAGS) -fno-use-cxa-atexit -flto
 ASFLAGS     := $(ATMOSPHERE_ASFLAGS) $(SETTINGS) $(DEFINES) $(INCLUDE)

libraries/libmesosphere/include/mesosphere.hpp

@@ -48,7 +48,6 @@
 #include <mesosphere/kern_k_spin_lock.hpp>
 #include <mesosphere/kern_k_memory_manager.hpp>
 #include <mesosphere/kern_k_interrupt_task_manager.hpp>
-#include <mesosphere/kern_k_core_local_region.hpp>
 #include <mesosphere/kern_k_slab_heap.hpp>
 #include <mesosphere/kern_k_light_lock.hpp>
 #include <mesosphere/kern_k_dpc_manager.hpp>
@@ -78,7 +77,8 @@
 #include <mesosphere/kern_select_debug.hpp>
 #include <mesosphere/kern_k_process.hpp>
 #include <mesosphere/kern_k_resource_limit.hpp>
-#include <mesosphere/kern_k_synchronization.hpp>
+#include <mesosphere/kern_k_alpha.hpp>
+#include <mesosphere/kern_k_beta.hpp>
 
 /* More Miscellaneous objects. */
 #include <mesosphere/kern_k_object_name.hpp>

libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_arguments.hpp

@@ -29,6 +29,7 @@ namespace ams::kern::init {
         u64 entrypoint;
         u64 argument;
         u64 setup_function;
+        u64 exception_stack;
     };
 
 }

libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp

@@ -305,7 +305,7 @@ namespace ams::kern::arch::arm64::init {
 
                     /* Can we make an L1 block? */
                     if (util::IsAligned(GetInteger(virt_addr), L1BlockSize) && util::IsAligned(GetInteger(phys_addr), L1BlockSize) && size >= L1BlockSize) {
-                        *l1_entry = L1PageTableEntry(PageTableEntry::BlockTag{}, phys_addr, attr, false);
+                        *l1_entry = L1PageTableEntry(PageTableEntry::BlockTag{}, phys_addr, attr, PageTableEntry::SoftwareReservedBit_None, false);
                         cpu::DataSynchronizationBarrierInnerShareable();
 
                         virt_addr += L1BlockSize;
@@ -327,7 +327,7 @@ namespace ams::kern::arch::arm64::init {
                     /* Can we make a contiguous L2 block? */
                     if (util::IsAligned(GetInteger(virt_addr), L2ContiguousBlockSize) && util::IsAligned(GetInteger(phys_addr), L2ContiguousBlockSize) && size >= L2ContiguousBlockSize) {
                         for (size_t i = 0; i < L2ContiguousBlockSize / L2BlockSize; i++) {
-                            l2_entry[i] = L2PageTableEntry(PageTableEntry::BlockTag{}, phys_addr, attr, true);
+                            l2_entry[i] = L2PageTableEntry(PageTableEntry::BlockTag{}, phys_addr, attr, PageTableEntry::SoftwareReservedBit_None, true);
                             cpu::DataSynchronizationBarrierInnerShareable();
 
                             virt_addr += L2BlockSize;
@@ -339,7 +339,7 @@ namespace ams::kern::arch::arm64::init {
 
                     /* Can we make an L2 block? */
                     if (util::IsAligned(GetInteger(virt_addr), L2BlockSize) && util::IsAligned(GetInteger(phys_addr), L2BlockSize) && size >= L2BlockSize) {
-                        *l2_entry = L2PageTableEntry(PageTableEntry::BlockTag{}, phys_addr, attr, false);
+                        *l2_entry = L2PageTableEntry(PageTableEntry::BlockTag{}, phys_addr, attr, PageTableEntry::SoftwareReservedBit_None, false);
                         cpu::DataSynchronizationBarrierInnerShareable();
 
                         virt_addr += L2BlockSize;
@@ -361,7 +361,7 @@ namespace ams::kern::arch::arm64::init {
                     /* Can we make a contiguous L3 block? */
                     if (util::IsAligned(GetInteger(virt_addr), L3ContiguousBlockSize) && util::IsAligned(GetInteger(phys_addr), L3ContiguousBlockSize) && size >= L3ContiguousBlockSize) {
                         for (size_t i = 0; i < L3ContiguousBlockSize / L3BlockSize; i++) {
-                            l3_entry[i] = L3PageTableEntry(PageTableEntry::BlockTag{}, phys_addr, attr, true);
+                            l3_entry[i] = L3PageTableEntry(PageTableEntry::BlockTag{}, phys_addr, attr, PageTableEntry::SoftwareReservedBit_None, true);
                             cpu::DataSynchronizationBarrierInnerShareable();
 
                             virt_addr += L3BlockSize;
@@ -372,7 +372,7 @@ namespace ams::kern::arch::arm64::init {
                     }
 
                     /* Make an L3 block. */
-                    *l3_entry = L3PageTableEntry(PageTableEntry::BlockTag{}, phys_addr, attr, false);
+                    *l3_entry = L3PageTableEntry(PageTableEntry::BlockTag{}, phys_addr, attr, PageTableEntry::SoftwareReservedBit_None, false);
                     cpu::DataSynchronizationBarrierInnerShareable();
                     virt_addr += L3BlockSize;
                     phys_addr += L3BlockSize;
@@ -542,7 +542,7 @@ namespace ams::kern::arch::arm64::init {
                         const KPhysicalAddress block = l1_entry->GetBlock();
                         MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), L1BlockSize));
                         MESOSPHERE_INIT_ABORT_UNLESS(size >= L1BlockSize);
-                        MESOSPHERE_INIT_ABORT_UNLESS(l1_entry->IsCompatibleWithAttribute(attr_before, false));
+                        MESOSPHERE_INIT_ABORT_UNLESS(l1_entry->IsCompatibleWithAttribute(attr_before, PageTableEntry::SoftwareReservedBit_None, false));
 
                         /* Invalidate the existing L1 block. */
                         *static_cast<PageTableEntry *>(l1_entry) = InvalidPageTableEntry;
@@ -550,7 +550,7 @@ namespace ams::kern::arch::arm64::init {
                         cpu::InvalidateEntireTlb();
 
                         /* Create new L1 block. */
-                        *l1_entry = L1PageTableEntry(PageTableEntry::BlockTag{}, block, attr_after, false);
+                        *l1_entry = L1PageTableEntry(PageTableEntry::BlockTag{}, block, attr_after, PageTableEntry::SoftwareReservedBit_None, false);
 
                         virt_addr += L1BlockSize;
                         size      -= L1BlockSize;
@@ -573,7 +573,7 @@ namespace ams::kern::arch::arm64::init {
                             /* Invalidate the existing contiguous L2 block. */
                             for (size_t i = 0; i < L2ContiguousBlockSize / L2BlockSize; i++) {
                                 /* Ensure that the entry is valid. */
-                                MESOSPHERE_INIT_ABORT_UNLESS(l2_entry[i].IsCompatibleWithAttribute(attr_before, true));
+                                MESOSPHERE_INIT_ABORT_UNLESS(l2_entry[i].IsCompatibleWithAttribute(attr_before, PageTableEntry::SoftwareReservedBit_None, true));
                                 static_cast<PageTableEntry *>(l2_entry)[i] = InvalidPageTableEntry;
                             }
                             cpu::DataSynchronizationBarrierInnerShareable();
@@ -581,7 +581,7 @@ namespace ams::kern::arch::arm64::init {
 
                             /* Create a new contiguous L2 block. */
                             for (size_t i = 0; i < L2ContiguousBlockSize / L2BlockSize; i++) {
-                                l2_entry[i] = L2PageTableEntry(PageTableEntry::BlockTag{}, block + L2BlockSize * i, attr_after, true);
+                                l2_entry[i] = L2PageTableEntry(PageTableEntry::BlockTag{}, block + L2BlockSize * i, attr_after, PageTableEntry::SoftwareReservedBit_None, true);
                             }
 
                             virt_addr += L2ContiguousBlockSize;
@@ -591,7 +591,7 @@ namespace ams::kern::arch::arm64::init {
                             MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), L2BlockSize));
                             MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(block),     L2BlockSize));
                             MESOSPHERE_INIT_ABORT_UNLESS(size >= L2BlockSize);
-                            MESOSPHERE_INIT_ABORT_UNLESS(l2_entry->IsCompatibleWithAttribute(attr_before, false));
+                            MESOSPHERE_INIT_ABORT_UNLESS(l2_entry->IsCompatibleWithAttribute(attr_before, PageTableEntry::SoftwareReservedBit_None, false));
 
                             /* Invalidate the existing L2 block. */
                             *static_cast<PageTableEntry *>(l2_entry) = InvalidPageTableEntry;
@@ -599,7 +599,7 @@ namespace ams::kern::arch::arm64::init {
                             cpu::InvalidateEntireTlb();
 
                             /* Create new L2 block. */
-                            *l2_entry = L2PageTableEntry(PageTableEntry::BlockTag{}, block, attr_after, false);
+                            *l2_entry = L2PageTableEntry(PageTableEntry::BlockTag{}, block, attr_after, PageTableEntry::SoftwareReservedBit_None, false);
 
                             virt_addr += L2BlockSize;
                             size      -= L2BlockSize;
@@ -625,7 +625,7 @@ namespace ams::kern::arch::arm64::init {
                         /* Invalidate the existing contiguous L3 block. */
                         for (size_t i = 0; i < L3ContiguousBlockSize / L3BlockSize; i++) {
                             /* Ensure that the entry is valid. */
-                            MESOSPHERE_INIT_ABORT_UNLESS(l3_entry[i].IsCompatibleWithAttribute(attr_before, true));
+                            MESOSPHERE_INIT_ABORT_UNLESS(l3_entry[i].IsCompatibleWithAttribute(attr_before, PageTableEntry::SoftwareReservedBit_None, true));
                             static_cast<PageTableEntry *>(l3_entry)[i] = InvalidPageTableEntry;
                         }
                         cpu::DataSynchronizationBarrierInnerShareable();
@@ -633,7 +633,7 @@ namespace ams::kern::arch::arm64::init {
 
                         /* Create a new contiguous L3 block. */
                         for (size_t i = 0; i < L3ContiguousBlockSize / L3BlockSize; i++) {
-                            l3_entry[i] = L3PageTableEntry(PageTableEntry::BlockTag{}, block + L3BlockSize * i, attr_after, true);
+                            l3_entry[i] = L3PageTableEntry(PageTableEntry::BlockTag{}, block + L3BlockSize * i, attr_after, PageTableEntry::SoftwareReservedBit_None, true);
                         }
 
                         virt_addr += L3ContiguousBlockSize;
@@ -643,7 +643,7 @@ namespace ams::kern::arch::arm64::init {
                         MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), L3BlockSize));
                         MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(block),     L3BlockSize));
                         MESOSPHERE_INIT_ABORT_UNLESS(size >= L3BlockSize);
-                        MESOSPHERE_INIT_ABORT_UNLESS(l3_entry->IsCompatibleWithAttribute(attr_before, false));
+                        MESOSPHERE_INIT_ABORT_UNLESS(l3_entry->IsCompatibleWithAttribute(attr_before, PageTableEntry::SoftwareReservedBit_None, false));
 
                         /* Invalidate the existing L3 block. */
                         *static_cast<PageTableEntry *>(l3_entry) = InvalidPageTableEntry;
@@ -651,7 +651,7 @@ namespace ams::kern::arch::arm64::init {
                         cpu::InvalidateEntireTlb();
 
                         /* Create new L3 block. */
-                        *l3_entry = L3PageTableEntry(PageTableEntry::BlockTag{}, block, attr_after, false);
+                        *l3_entry = L3PageTableEntry(PageTableEntry::BlockTag{}, block, attr_after, PageTableEntry::SoftwareReservedBit_None, false);
 
                         virt_addr += L3BlockSize;
                         size      -= L3BlockSize;
@@ -668,6 +668,7 @@ namespace ams::kern::arch::arm64::init {
                 this->PhysicallyRandomize(virt_addr, size, L2BlockSize, do_copy);
                 this->PhysicallyRandomize(virt_addr, size, L3ContiguousBlockSize, do_copy);
                 this->PhysicallyRandomize(virt_addr, size, L3BlockSize, do_copy);
+                cpu::StoreEntireCacheForInit();
             }
 
     };

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu.hpp

@@ -220,16 +220,19 @@ namespace ams::kern::arch::arm64::cpu {
         DataSynchronizationBarrier();
     }
 
-    ALWAYS_INLINE uintptr_t GetCoreLocalRegionAddress() {
+    ALWAYS_INLINE uintptr_t GetCurrentThreadPointerValue() {
         register uintptr_t x18 asm("x18");
         __asm__ __volatile__("" : [x18]"=r"(x18));
         return x18;
     }
 
-    ALWAYS_INLINE void SetCoreLocalRegionAddress(uintptr_t value) {
+    ALWAYS_INLINE void SetCurrentThreadPointerValue(uintptr_t value) {
         register uintptr_t x18 asm("x18") = value;
         __asm__ __volatile__("":: [x18]"r"(x18));
-        SetTpidrEl1(value);
+    }
+
+    ALWAYS_INLINE void SetExceptionThreadStackTop(uintptr_t top) {
+        SetTpidrEl1(top);
     }
 
     ALWAYS_INLINE void SwitchThreadLocalRegion(uintptr_t tlr) {

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_hardware_timer.hpp

@@ -19,18 +19,14 @@
 
 namespace ams::kern::arch::arm64 {
 
-    namespace impl {
+    class KHardwareTimer : public KInterruptTask, public KHardwareTimerBase {
-
+        private:
-        class KHardwareTimerInterruptTask;
+            s64 maximum_time;
-
-    }
-
-    class KHardwareTimer : public KHardwareTimerBase {
         public:
-            constexpr KHardwareTimer() : KHardwareTimerBase() { /* ... */ }
+            constexpr KHardwareTimer() : KInterruptTask(), KHardwareTimerBase(), maximum_time(std::numeric_limits<s64>::max()) { /* ... */ }
         public:
             /* Public API. */
-            NOINLINE void Initialize(s32 core_id);
+            NOINLINE void Initialize();
             NOINLINE void Finalize();
 
             static s64 GetTick() {
@@ -42,13 +38,12 @@ namespace ams::kern::arch::arm64 {
                 KScopedSpinLock lk(this->GetLock());
 
                 if (this->RegisterAbsoluteTaskImpl(task, task_time)) {
-                    SetCompareValue(task_time);
+                    if (task_time <= this->maximum_time) {
-                    EnableInterrupt();
+                        SetCompareValue(task_time);
+                        EnableInterrupt();
+                    }
                 }
             }
-        private:
-            friend class impl::KHardwareTimerInterruptTask;
-            NOINLINE void DoInterruptTask();
         private:
             /* Hardware register accessors. */
             static ALWAYS_INLINE void InitializeGlobalTimer() {
@@ -88,7 +83,13 @@ namespace ams::kern::arch::arm64 {
             static ALWAYS_INLINE void SetCompareValue(s64 value) {
                 cpu::CounterTimerPhysicalTimerCompareValueRegisterAccessor(0).SetCompareValue(static_cast<u64>(value)).Store();
             }
+        public:
+            virtual KInterruptTask *OnInterrupt(s32 interrupt_id) override {
+                MESOSPHERE_UNUSED(interrupt_id);
+                return this;
+            }
 
+            virtual void DoTask() override;
     };
 
 }

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_interrupt_manager.hpp

@@ -47,23 +47,23 @@ namespace ams::kern::arch::arm64 {
                 constexpr KGlobalInterruptEntry() : handler(nullptr), manually_cleared(false), needs_clear(false) { /* ... */ }
             };
         private:
-            static KSpinLock s_lock;
+            KCoreLocalInterruptEntry core_local_interrupts[cpu::NumCores][KInterruptController::NumLocalInterrupts]{};
-            static std::array<KGlobalInterruptEntry, KInterruptController::NumGlobalInterrupts> s_global_interrupts;
+            KInterruptController interrupt_controller{};
-            static KInterruptController::GlobalState s_global_state;
+            KInterruptController::LocalState local_states[cpu::NumCores]{};
-            static bool s_global_state_saved;
+            bool local_state_saved[cpu::NumCores]{};
+            mutable KSpinLock global_interrupt_lock{};
+            KGlobalInterruptEntry global_interrupts[KInterruptController::NumGlobalInterrupts]{};
+            KInterruptController::GlobalState global_state{};
+            bool global_state_saved{};
         private:
-            KCoreLocalInterruptEntry core_local_interrupts[KInterruptController::NumLocalInterrupts];
+            ALWAYS_INLINE KSpinLock &GetGlobalInterruptLock() const { return this->global_interrupt_lock; }
-            KInterruptController interrupt_controller;
+            ALWAYS_INLINE KGlobalInterruptEntry &GetGlobalInterruptEntry(s32 irq) { return this->global_interrupts[KInterruptController::GetGlobalInterruptIndex(irq)]; }
-            KInterruptController::LocalState local_state;
+            ALWAYS_INLINE KCoreLocalInterruptEntry &GetLocalInterruptEntry(s32 irq) { return this->core_local_interrupts[GetCurrentCoreId()][KInterruptController::GetLocalInterruptIndex(irq)]; }
-            bool local_state_saved;
-        private:
-            static ALWAYS_INLINE KSpinLock &GetLock() { return s_lock; }
-            static ALWAYS_INLINE KGlobalInterruptEntry &GetGlobalInterruptEntry(s32 irq) { return s_global_interrupts[KInterruptController::GetGlobalInterruptIndex(irq)]; }
-            ALWAYS_INLINE KCoreLocalInterruptEntry &GetLocalInterruptEntry(s32 irq) { return this->core_local_interrupts[KInterruptController::GetLocalInterruptIndex(irq)]; }
 
             bool OnHandleInterrupt();
         public:
-            constexpr KInterruptManager() : core_local_interrupts(), interrupt_controller(), local_state(), local_state_saved(false) { /* ... */ }
+            constexpr KInterruptManager() = default;
+
             NOINLINE void Initialize(s32 core_id);
             NOINLINE void Finalize(s32 core_id);
 

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table.hpp

@@ -134,9 +134,6 @@ namespace ams::kern::arch::arm64 {
                     entry.SetUserExecuteNever(true);
                 }
 
-                /* Set can be contiguous. */
-                entry.SetContiguousAllowed(!properties.non_contiguous);
-
                 /* Set AP[1] based on perm. */
                 switch (properties.perm & KMemoryPermission_UserReadWrite) {
                     case KMemoryPermission_UserReadWrite:
@@ -179,19 +176,19 @@ namespace ams::kern::arch::arm64 {
             }
 
             NOINLINE Result InitializeForKernel(void *table, KVirtualAddress start, KVirtualAddress end);
-            NOINLINE Result InitializeForProcess(u32 id, ams::svc::CreateProcessFlag as_type, bool enable_aslr, bool from_back, KMemoryManager::Pool pool, KProcessAddress code_address, size_t code_size, KMemoryBlockSlabManager *mem_block_slab_manager, KBlockInfoManager *block_info_manager, KPageTableManager *pt_manager);
+            NOINLINE Result InitializeForProcess(u32 id, ams::svc::CreateProcessFlag as_type, bool enable_aslr, bool enable_das_merge, bool from_back, KMemoryManager::Pool pool, KProcessAddress code_address, size_t code_size, KMemoryBlockSlabManager *mem_block_slab_manager, KBlockInfoManager *block_info_manager, KPageTableManager *pt_manager);
             Result Finalize();
         private:
-            Result MapL1Blocks(KProcessAddress virt_addr, KPhysicalAddress phys_addr, size_t num_pages, PageTableEntry entry_template, PageLinkedList *page_list, bool reuse_ll);
+            Result MapL1Blocks(KProcessAddress virt_addr, KPhysicalAddress phys_addr, size_t num_pages, PageTableEntry entry_template, bool disable_head_merge, PageLinkedList *page_list, bool reuse_ll);
-            Result MapL2Blocks(KProcessAddress virt_addr, KPhysicalAddress phys_addr, size_t num_pages, PageTableEntry entry_template, PageLinkedList *page_list, bool reuse_ll);
+            Result MapL2Blocks(KProcessAddress virt_addr, KPhysicalAddress phys_addr, size_t num_pages, PageTableEntry entry_template, bool disable_head_merge, PageLinkedList *page_list, bool reuse_ll);
-            Result MapL3Blocks(KProcessAddress virt_addr, KPhysicalAddress phys_addr, size_t num_pages, PageTableEntry entry_template, PageLinkedList *page_list, bool reuse_ll);
+            Result MapL3Blocks(KProcessAddress virt_addr, KPhysicalAddress phys_addr, size_t num_pages, PageTableEntry entry_template, bool disable_head_merge, PageLinkedList *page_list, bool reuse_ll);
 
             Result Unmap(KProcessAddress virt_addr, size_t num_pages, PageLinkedList *page_list, bool force, bool reuse_ll);
 
-            Result Map(KProcessAddress virt_addr, KPhysicalAddress phys_addr, size_t num_pages, PageTableEntry entry_template, size_t page_size, PageLinkedList *page_list, bool reuse_ll) {
+            Result Map(KProcessAddress virt_addr, KPhysicalAddress phys_addr, size_t num_pages, PageTableEntry entry_template, bool disable_head_merge, size_t page_size, PageLinkedList *page_list, bool reuse_ll) {
                 switch (page_size) {
                     case L1BlockSize:
-                        return this->MapL1Blocks(virt_addr, phys_addr, num_pages, entry_template, page_list, reuse_ll);
+                        return this->MapL1Blocks(virt_addr, phys_addr, num_pages, entry_template, disable_head_merge, page_list, reuse_ll);
                     case L2ContiguousBlockSize:
                         entry_template.SetContiguous(true);
                         [[fallthrough]];
@@ -199,25 +196,25 @@ namespace ams::kern::arch::arm64 {
                     case L2TegraSmmuBlockSize:
 #endif
                     case L2BlockSize:
-                        return this->MapL2Blocks(virt_addr, phys_addr, num_pages, entry_template, page_list, reuse_ll);
+                        return this->MapL2Blocks(virt_addr, phys_addr, num_pages, entry_template, disable_head_merge, page_list, reuse_ll);
                     case L3ContiguousBlockSize:
                         entry_template.SetContiguous(true);
                         [[fallthrough]];
                     case L3BlockSize:
-                        return this->MapL3Blocks(virt_addr, phys_addr, num_pages, entry_template, page_list, reuse_ll);
+                        return this->MapL3Blocks(virt_addr, phys_addr, num_pages, entry_template, disable_head_merge, page_list, reuse_ll);
                     MESOSPHERE_UNREACHABLE_DEFAULT_CASE();
                 }
             }
 
-            Result MapContiguous(KProcessAddress virt_addr, KPhysicalAddress phys_addr, size_t num_pages, PageTableEntry entry_template, PageLinkedList *page_list, bool reuse_ll);
+            Result MapContiguous(KProcessAddress virt_addr, KPhysicalAddress phys_addr, size_t num_pages, PageTableEntry entry_template, bool disable_head_merge, PageLinkedList *page_list, bool reuse_ll);
-            Result MapGroup(KProcessAddress virt_addr, const KPageGroup &pg, size_t num_pages, PageTableEntry entry_template, PageLinkedList *page_list, bool reuse_ll);
+            Result MapGroup(KProcessAddress virt_addr, const KPageGroup &pg, size_t num_pages, PageTableEntry entry_template, bool disable_head_merge, PageLinkedList *page_list, bool reuse_ll);
 
             bool MergePages(KProcessAddress virt_addr, PageLinkedList *page_list);
 
             ALWAYS_INLINE Result SeparatePagesImpl(KProcessAddress virt_addr, size_t block_size, PageLinkedList *page_list, bool reuse_ll);
             Result SeparatePages(KProcessAddress virt_addr, size_t block_size, PageLinkedList *page_list, bool reuse_ll);
 
-            Result ChangePermissions(KProcessAddress virt_addr, size_t num_pages, PageTableEntry entry_template, bool refresh_mapping, PageLinkedList *page_list, bool reuse_ll);
+            Result ChangePermissions(KProcessAddress virt_addr, size_t num_pages, PageTableEntry entry_template, DisableMergeAttribute disable_merge_attr, bool refresh_mapping, PageLinkedList *page_list, bool reuse_ll);
 
             static void PteDataSynchronizationBarrier() {
                 cpu::DataSynchronizationBarrierInnerShareable();

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table_entry.hpp

@@ -69,9 +69,23 @@ namespace ams::kern::arch::arm64 {
                 MappingFlag_Mapped    = (1 << 0),
             };
 
+            enum SoftwareReservedBit : u8 {
+                SoftwareReservedBit_None                    = 0,
+                SoftwareReservedBit_DisableMergeHead        = (1u << 0),
+                SoftwareReservedBit_DisableMergeHeadAndBody = (1u << 1),
+                SoftwareReservedBit_DisableMergeHeadTail    = (1u << 2),
+                SoftwareReservedBit_Valid                   = (1u << 3),
+            };
+
+            static constexpr ALWAYS_INLINE std::underlying_type<SoftwareReservedBit>::type EncodeSoftwareReservedBits(bool head, bool head_body, bool tail) {
+                return (head ? SoftwareReservedBit_DisableMergeHead : SoftwareReservedBit_None) | (head_body ? SoftwareReservedBit_DisableMergeHeadAndBody : SoftwareReservedBit_None) | (tail ? SoftwareReservedBit_DisableMergeHeadTail : SoftwareReservedBit_None);
+            }
+
             enum ExtensionFlag : u64 {
-                ExtensionFlag_NotContiguous  = (1ul << 55),
+                ExtensionFlag_DisableMergeHead        = (static_cast<u64>(SoftwareReservedBit_DisableMergeHead)        << 55),
-                ExtensionFlag_Valid          = (1ul << 56),
+                ExtensionFlag_DisableMergeHeadAndBody = (static_cast<u64>(SoftwareReservedBit_DisableMergeHeadAndBody) << 55),
+                ExtensionFlag_DisableMergeTail        = (static_cast<u64>(SoftwareReservedBit_DisableMergeHeadTail)    << 55),
+                ExtensionFlag_Valid                   = (static_cast<u64>(SoftwareReservedBit_Valid)                   << 55),
 
                 ExtensionFlag_ValidAndMapped = (ExtensionFlag_Valid | MappingFlag_Mapped),
                 ExtensionFlag_TestTableMask  = (ExtensionFlag_Valid | (1ul << 1)),
@@ -138,23 +152,26 @@ namespace ams::kern::arch::arm64 {
                 }
             }
         public:
-            constexpr ALWAYS_INLINE bool IsContiguousAllowed()       const { return this->GetBits(55, 1) == 0; }
+            constexpr ALWAYS_INLINE u8 GetSoftwareReservedBits()            const { return this->GetBits(55, 3); }
-            constexpr ALWAYS_INLINE bool IsUserExecuteNever()        const { return this->GetBits(54, 1) != 0; }
+            constexpr ALWAYS_INLINE bool IsHeadMergeDisabled()              const { return (this->GetSoftwareReservedBits() & SoftwareReservedBit_DisableMergeHead) != 0; }
-            constexpr ALWAYS_INLINE bool IsPrivilegedExecuteNever()  const { return this->GetBits(53, 1) != 0; }
+            constexpr ALWAYS_INLINE bool IsHeadAndBodyMergeDisabled()       const { return (this->GetSoftwareReservedBits() & SoftwareReservedBit_DisableMergeHeadAndBody) != 0; }
-            constexpr ALWAYS_INLINE bool IsContiguous()              const { return this->GetBits(52, 1) != 0; }
+            constexpr ALWAYS_INLINE bool IsTailMergeDisabled()              const { return (this->GetSoftwareReservedBits() & SoftwareReservedBit_DisableMergeHeadTail) != 0; }
-            constexpr ALWAYS_INLINE bool IsGlobal()                  const { return this->GetBits(11, 1) == 0; }
+            constexpr ALWAYS_INLINE bool IsHeadOrHeadAndBodyMergeDisabled() const { return (this->GetSoftwareReservedBits() & (SoftwareReservedBit_DisableMergeHead | SoftwareReservedBit_DisableMergeHeadAndBody)) != 0; }
-            constexpr ALWAYS_INLINE AccessFlag GetAccessFlag()       const { return static_cast<AccessFlag>(this->GetBits(10, 1)); }
+            constexpr ALWAYS_INLINE bool IsUserExecuteNever()               const { return this->GetBits(54, 1) != 0; }
-            constexpr ALWAYS_INLINE Shareable GetShareable()         const { return static_cast<Shareable>(this->GetBits(8, 2)); }
+            constexpr ALWAYS_INLINE bool IsPrivilegedExecuteNever()         const { return this->GetBits(53, 1) != 0; }
-            constexpr ALWAYS_INLINE PageAttribute GetPageAttribute() const { return static_cast<PageAttribute>(this->GetBits(2, 3)); }
+            constexpr ALWAYS_INLINE bool IsContiguous()                     const { return this->GetBits(52, 1) != 0; }
-            constexpr ALWAYS_INLINE bool IsReadOnly()                const { return this->GetBits(7, 1) != 0; }
+            constexpr ALWAYS_INLINE bool IsGlobal()                         const { return this->GetBits(11, 1) == 0; }
-            constexpr ALWAYS_INLINE bool IsUserAccessible()          const { return this->GetBits(6, 1) != 0; }
+            constexpr ALWAYS_INLINE AccessFlag GetAccessFlag()              const { return static_cast<AccessFlag>(this->GetBits(10, 1)); }
-            constexpr ALWAYS_INLINE bool IsNonSecure()               const { return this->GetBits(5, 1) != 0; }
+            constexpr ALWAYS_INLINE Shareable GetShareable()                const { return static_cast<Shareable>(this->GetBits(8, 2)); }
-            constexpr ALWAYS_INLINE bool IsBlock()                   const { return (this->attributes & ExtensionFlag_TestTableMask) == ExtensionFlag_Valid; }
+            constexpr ALWAYS_INLINE PageAttribute GetPageAttribute()        const { return static_cast<PageAttribute>(this->GetBits(2, 3)); }
-            constexpr ALWAYS_INLINE bool IsTable()                   const { return (this->attributes & ExtensionFlag_TestTableMask) == 2; }
+            constexpr ALWAYS_INLINE bool IsReadOnly()                       const { return this->GetBits(7, 1) != 0; }
-            constexpr ALWAYS_INLINE bool IsEmpty()                   const { return (this->attributes & ExtensionFlag_TestTableMask) == 0; }
+            constexpr ALWAYS_INLINE bool IsUserAccessible()                 const { return this->GetBits(6, 1) != 0; }
-            constexpr ALWAYS_INLINE bool IsMapped()                  const { return this->GetBits(0, 1) != 0; }
+            constexpr ALWAYS_INLINE bool IsNonSecure()                      const { return this->GetBits(5, 1) != 0; }
+            constexpr ALWAYS_INLINE bool IsBlock()                          const { return (this->attributes & ExtensionFlag_TestTableMask) == ExtensionFlag_Valid; }
+            constexpr ALWAYS_INLINE bool IsTable()                          const { return (this->attributes & ExtensionFlag_TestTableMask) == 2; }
+            constexpr ALWAYS_INLINE bool IsEmpty()                          const { return (this->attributes & ExtensionFlag_TestTableMask) == 0; }
+            constexpr ALWAYS_INLINE bool IsMapped()                         const { return this->GetBits(0, 1) != 0; }
 
-            constexpr ALWAYS_INLINE decltype(auto) SetContiguousAllowed(bool en)      { this->SetBit(55, !en); return *this; }
             constexpr ALWAYS_INLINE decltype(auto) SetUserExecuteNever(bool en)       { this->SetBit(54, en); return *this; }
             constexpr ALWAYS_INLINE decltype(auto) SetPrivilegedExecuteNever(bool en) { this->SetBit(53, en); return *this; }
             constexpr ALWAYS_INLINE decltype(auto) SetContiguous(bool en)             { this->SetBit(52, en); return *this; }
@@ -166,13 +183,14 @@ namespace ams::kern::arch::arm64 {
             constexpr ALWAYS_INLINE decltype(auto) SetPageAttribute(PageAttribute a)  { this->SetBitsDirect(2, 3, a); return *this; }
             constexpr ALWAYS_INLINE decltype(auto) SetMapped(bool m)                  { static_assert(static_cast<u64>(MappingFlag_Mapped == (1 << 0))); this->SetBit(0, m); return *this; }
 
-            constexpr ALWAYS_INLINE u64 GetEntryTemplate() const {
+            constexpr ALWAYS_INLINE u64 GetEntryTemplateForMerge() const {
-                constexpr u64 Mask = (0xFFF0000000000FFFul & ~u64((0x1ul << 52) | ExtensionFlag_TestTableMask));
+                constexpr u64 BaseMask = (0xFFF0000000000FFFul & ~static_cast<u64>((0x1ul << 52) | ExtensionFlag_TestTableMask | ExtensionFlag_DisableMergeHead | ExtensionFlag_DisableMergeHeadAndBody | ExtensionFlag_DisableMergeTail));
-                return this->attributes & Mask;
+                return this->attributes & BaseMask;
             }
 
-            constexpr ALWAYS_INLINE bool Is(u64 attr) const {
+            constexpr ALWAYS_INLINE bool IsForMerge(u64 attr) const {
-                return this->attributes == attr;
+                constexpr u64 BaseMaskForMerge = ~static_cast<u64>(ExtensionFlag_DisableMergeHead | ExtensionFlag_DisableMergeHeadAndBody | ExtensionFlag_DisableMergeTail);
+                return (this->attributes & BaseMaskForMerge) == attr;
             }
 
             constexpr ALWAYS_INLINE u64 GetRawAttributesUnsafeForSwap() const {
@@ -207,8 +225,8 @@ namespace ams::kern::arch::arm64 {
                 /* ... */
             }
 
-            constexpr explicit ALWAYS_INLINE L1PageTableEntry(BlockTag, KPhysicalAddress phys_addr, const PageTableEntry &attr, bool contig)
+            constexpr explicit ALWAYS_INLINE L1PageTableEntry(BlockTag, KPhysicalAddress phys_addr, const PageTableEntry &attr, u8 sw_reserved_bits, bool contig)
-                : PageTableEntry(attr, (static_cast<u64>(contig) << 52) | GetInteger(phys_addr) | PageTableEntry::ExtensionFlag_Valid)
+                : PageTableEntry(attr, (static_cast<u64>(sw_reserved_bits) << 55) | (static_cast<u64>(contig) << 52) | GetInteger(phys_addr) | PageTableEntry::ExtensionFlag_Valid)
             {
                 /* ... */
             }
@@ -230,9 +248,26 @@ namespace ams::kern::arch::arm64 {
                 }
             }
 
-            constexpr ALWAYS_INLINE bool IsCompatibleWithAttribute(const PageTableEntry &rhs, bool contig) const {
+            static constexpr ALWAYS_INLINE u64 GetEntryTemplateForL2BlockMask(size_t idx) {
+                constexpr u64 BaseMask = (0xFFF0000000000FFFul & ~static_cast<u64>((0x1ul << 52) | ExtensionFlag_TestTableMask | ExtensionFlag_DisableMergeHead | ExtensionFlag_DisableMergeHeadAndBody | ExtensionFlag_DisableMergeTail));
+                if (idx == 0) {
+                    return BaseMask | ExtensionFlag_DisableMergeHead | ExtensionFlag_DisableMergeHeadAndBody;
+                } else if (idx < L2ContiguousBlockSize / L2BlockSize) {
+                    return BaseMask | ExtensionFlag_DisableMergeHeadAndBody;
+                } else if (idx < (L1BlockSize - L2ContiguousBlockSize) / L2BlockSize) {
+                    return BaseMask;
+                } else {
+                    return BaseMask | ExtensionFlag_DisableMergeTail;
+                }
+            }
+
+            constexpr ALWAYS_INLINE u64 GetEntryTemplateForL2Block(size_t idx) const {
+                return this->attributes & GetEntryTemplateForL2BlockMask(idx);
+            }
+
+            constexpr ALWAYS_INLINE bool IsCompatibleWithAttribute(const PageTableEntry &rhs, u8 sw_reserved_bits, bool contig) const {
                 /* Check whether this has the same permission/etc as the desired attributes. */
-                return L1PageTableEntry(BlockTag{}, this->GetBlock(), rhs, contig).GetRawAttributes() == this->GetRawAttributes();
+                return L1PageTableEntry(BlockTag{}, this->GetBlock(), rhs, sw_reserved_bits, contig).GetRawAttributes() == this->GetRawAttributes();
             }
     };
 
@@ -252,8 +287,8 @@ namespace ams::kern::arch::arm64 {
                 /* ... */
             }
 
-            constexpr explicit ALWAYS_INLINE L2PageTableEntry(BlockTag, KPhysicalAddress phys_addr, const PageTableEntry &attr, bool contig)
+            constexpr explicit ALWAYS_INLINE L2PageTableEntry(BlockTag, KPhysicalAddress phys_addr, const PageTableEntry &attr, u8 sw_reserved_bits, bool contig)
-                : PageTableEntry(attr, (static_cast<u64>(contig) << 52) | GetInteger(phys_addr) | PageTableEntry::ExtensionFlag_Valid)
+                : PageTableEntry(attr, (static_cast<u64>(sw_reserved_bits) << 55) | (static_cast<u64>(contig) << 52) | GetInteger(phys_addr) | PageTableEntry::ExtensionFlag_Valid)
             {
                 /* ... */
             }
@@ -275,9 +310,41 @@ namespace ams::kern::arch::arm64 {
                 }
             }
 
-            constexpr ALWAYS_INLINE bool IsCompatibleWithAttribute(const PageTableEntry &rhs, bool contig) const {
+            static constexpr ALWAYS_INLINE u64 GetEntryTemplateForL2BlockMask(size_t idx) {
+                constexpr u64 BaseMask = (0xFFF0000000000FFFul & ~static_cast<u64>((0x1ul << 52) | ExtensionFlag_TestTableMask | ExtensionFlag_DisableMergeHead | ExtensionFlag_DisableMergeHeadAndBody | ExtensionFlag_DisableMergeTail));
+                if (idx == 0) {
+                    return BaseMask | ExtensionFlag_DisableMergeHead | ExtensionFlag_DisableMergeHeadAndBody;
+                } else if (idx < (L2ContiguousBlockSize / L2BlockSize) - 1) {
+                    return BaseMask;
+                } else {
+                    return BaseMask | ExtensionFlag_DisableMergeTail;
+                }
+            }
+
+            constexpr ALWAYS_INLINE u64 GetEntryTemplateForL2Block(size_t idx) const {
+                return this->attributes & GetEntryTemplateForL2BlockMask(idx);
+            }
+
+            static constexpr ALWAYS_INLINE u64 GetEntryTemplateForL3BlockMask(size_t idx) {
+                constexpr u64 BaseMask = (0xFFF0000000000FFFul & ~static_cast<u64>((0x1ul << 52) | ExtensionFlag_TestTableMask | ExtensionFlag_DisableMergeHead | ExtensionFlag_DisableMergeHeadAndBody | ExtensionFlag_DisableMergeTail));
+                if (idx == 0) {
+                    return BaseMask | ExtensionFlag_DisableMergeHead | ExtensionFlag_DisableMergeHeadAndBody;
+                } else if (idx < L3ContiguousBlockSize / L3BlockSize) {
+                    return BaseMask | ExtensionFlag_DisableMergeHeadAndBody;
+                } else if (idx < (L2BlockSize - L3ContiguousBlockSize) / L3BlockSize) {
+                    return BaseMask;
+                } else {
+                    return BaseMask | ExtensionFlag_DisableMergeTail;
+                }
+            }
+
+            constexpr ALWAYS_INLINE u64 GetEntryTemplateForL3Block(size_t idx) const {
+                return this->attributes & GetEntryTemplateForL3BlockMask(idx);
+            }
+
+            constexpr ALWAYS_INLINE bool IsCompatibleWithAttribute(const PageTableEntry &rhs, u8 sw_reserved_bits, bool contig) const {
                 /* Check whether this has the same permission/etc as the desired attributes. */
-                return L2PageTableEntry(BlockTag{}, this->GetBlock(), rhs, contig).GetRawAttributes() == this->GetRawAttributes();
+                return L2PageTableEntry(BlockTag{}, this->GetBlock(), rhs, sw_reserved_bits, contig).GetRawAttributes() == this->GetRawAttributes();
             }
     };
 
@@ -285,8 +352,8 @@ namespace ams::kern::arch::arm64 {
         public:
             constexpr explicit ALWAYS_INLINE L3PageTableEntry(InvalidTag) : PageTableEntry(InvalidTag{}) { /* ... */ }
 
-            constexpr explicit ALWAYS_INLINE L3PageTableEntry(BlockTag, KPhysicalAddress phys_addr, const PageTableEntry &attr, bool contig)
+            constexpr explicit ALWAYS_INLINE L3PageTableEntry(BlockTag, KPhysicalAddress phys_addr, const PageTableEntry &attr, u8 sw_reserved_bits, bool contig)
-                : PageTableEntry(attr, (static_cast<u64>(contig) << 52) | GetInteger(phys_addr) | static_cast<u64>(ExtensionFlag_TestTableMask))
+                : PageTableEntry(attr, (static_cast<u64>(sw_reserved_bits) << 55) | (static_cast<u64>(contig) << 52) | GetInteger(phys_addr) | static_cast<u64>(ExtensionFlag_TestTableMask))
             {
                 /* ... */
             }
@@ -297,9 +364,24 @@ namespace ams::kern::arch::arm64 {
                 return this->SelectBits(12, 36);
             }
 
-            constexpr ALWAYS_INLINE bool IsCompatibleWithAttribute(const PageTableEntry &rhs, bool contig) const {
+            static constexpr ALWAYS_INLINE u64 GetEntryTemplateForL3BlockMask(size_t idx) {
+                constexpr u64 BaseMask = (0xFFF0000000000FFFul & ~static_cast<u64>((0x1ul << 52) | ExtensionFlag_TestTableMask | ExtensionFlag_DisableMergeHead | ExtensionFlag_DisableMergeHeadAndBody | ExtensionFlag_DisableMergeTail));
+                if (idx == 0) {
+                    return BaseMask | ExtensionFlag_DisableMergeHead | ExtensionFlag_DisableMergeHeadAndBody;
+                } else if (idx < (L3ContiguousBlockSize / L3BlockSize) - 1) {
+                    return BaseMask;
+                } else {
+                    return BaseMask | ExtensionFlag_DisableMergeTail;
+                }
+            }
+
+            constexpr ALWAYS_INLINE u64 GetEntryTemplateForL3Block(size_t idx) const {
+                return this->attributes & GetEntryTemplateForL3BlockMask(idx);
+            }
+
+            constexpr ALWAYS_INLINE bool IsCompatibleWithAttribute(const PageTableEntry &rhs, u8 sw_reserved_bits, bool contig) const {
                 /* Check whether this has the same permission/etc as the desired attributes. */
-                return L3PageTableEntry(BlockTag{}, this->GetBlock(), rhs, contig).GetRawAttributes() == this->GetRawAttributes();
+                return L3PageTableEntry(BlockTag{}, this->GetBlock(), rhs, sw_reserved_bits, contig).GetRawAttributes() == this->GetRawAttributes();
             }
     };
 

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table_impl.hpp

@@ -29,6 +29,11 @@ namespace ams::kern::arch::arm64 {
             struct TraversalEntry {
                 KPhysicalAddress phys_addr;
                 size_t block_size;
+                u8 sw_reserved_bits;
+
+                constexpr bool IsHeadMergeDisabled() const { return (this->sw_reserved_bits & PageTableEntry::SoftwareReservedBit_DisableMergeHead) != 0; }
+                constexpr bool IsHeadAndBodyMergeDisabled() const { return (this->sw_reserved_bits & PageTableEntry::SoftwareReservedBit_DisableMergeHeadAndBody) != 0; }
+                constexpr bool IsTailMergeDisabled() const { return (this->sw_reserved_bits & PageTableEntry::SoftwareReservedBit_DisableMergeHeadTail) != 0; }
             };
 
             struct TraversalContext {

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_process_page_table.hpp

@@ -30,8 +30,8 @@ namespace ams::kern::arch::arm64 {
                 this->page_table.Activate(id);
             }
 
-            Result Initialize(u32 id, ams::svc::CreateProcessFlag as_type, bool enable_aslr, bool from_back, KMemoryManager::Pool pool, KProcessAddress code_address, size_t code_size, KMemoryBlockSlabManager *mem_block_slab_manager, KBlockInfoManager *block_info_manager, KPageTableManager *pt_manager) {
+            Result Initialize(u32 id, ams::svc::CreateProcessFlag as_type, bool enable_aslr, bool enable_das_merge, bool from_back, KMemoryManager::Pool pool, KProcessAddress code_address, size_t code_size, KMemoryBlockSlabManager *mem_block_slab_manager, KBlockInfoManager *block_info_manager, KPageTableManager *pt_manager) {
-                return this->page_table.InitializeForProcess(id, as_type, enable_aslr, from_back, pool, code_address, code_size, mem_block_slab_manager, block_info_manager, pt_manager);
+                return this->page_table.InitializeForProcess(id, as_type, enable_aslr, enable_das_merge, from_back, pool, code_address, code_size, mem_block_slab_manager, block_info_manager, pt_manager);
             }
 
             void Finalize() { this->page_table.Finalize(); }
@@ -128,10 +128,6 @@ namespace ams::kern::arch::arm64 {
                 return this->page_table.MakeAndOpenPageGroup(out, address, num_pages, state_mask, state, perm_mask, perm, attr_mask, attr);
             }
 
-            Result MakeAndOpenPageGroupContiguous(KPageGroup *out, KProcessAddress address, size_t num_pages, u32 state_mask, u32 state, u32 perm_mask, u32 perm, u32 attr_mask, u32 attr) {
-                return this->page_table.MakeAndOpenPageGroupContiguous(out, address, num_pages, state_mask, state, perm_mask, perm, attr_mask, attr);
-            }
-
             Result InvalidateProcessDataCache(KProcessAddress address, size_t size) {
                 return this->page_table.InvalidateProcessDataCache(address, size);
             }
@@ -152,6 +148,14 @@ namespace ams::kern::arch::arm64 {
                 return this->page_table.UnlockForDeviceAddressSpace(address, size);
             }
 
+            Result MakePageGroupForUnmapDeviceAddressSpace(KPageGroup *out, KProcessAddress address, size_t size) {
+                return this->page_table.MakePageGroupForUnmapDeviceAddressSpace(out, address, size);
+            }
+
+            Result UnlockForDeviceAddressSpacePartialMap(KProcessAddress address, size_t size, size_t mapped_size) {
+                return this->page_table.UnlockForDeviceAddressSpacePartialMap(address, size, mapped_size);
+            }
+
             Result LockForIpcUserBuffer(KPhysicalAddress *out, KProcessAddress address, size_t size) {
                 return this->page_table.LockForIpcUserBuffer(out, address, size);
             }

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_spin_lock.hpp

@@ -25,7 +25,7 @@ namespace ams::kern::arch::arm64 {
         public:
             constexpr KNotAlignedSpinLock() : packed_tickets(0) { /* ... */ }
 
-            void Lock() {
+            ALWAYS_INLINE void Lock() {
                 u32 tmp0, tmp1, tmp2;
 
                 __asm__ __volatile__(
@@ -52,7 +52,7 @@ namespace ams::kern::arch::arm64 {
                 );
             }
 
-            void Unlock() {
+            ALWAYS_INLINE void Unlock() {
                 const u32 value = this->packed_tickets + 1;
                 __asm__ __volatile__(
                     "    stlrh %w[value], %[packed_tickets]\n"
@@ -71,7 +71,7 @@ namespace ams::kern::arch::arm64 {
         public:
             constexpr KAlignedSpinLock() : current_ticket(0), next_ticket(0) { /* ... */ }
 
-            void Lock() {
+            ALWAYS_INLINE void Lock() {
                 u32 tmp0, tmp1, got_lock;
 
                 __asm__ __volatile__(
@@ -94,7 +94,7 @@ namespace ams::kern::arch::arm64 {
                 );
             }
 
-            void Unlock() {
+            ALWAYS_INLINE void Unlock() {
                 const u32 value = this->current_ticket + 1;
                 __asm__ __volatile__(
                     "    stlrh %w[value], %[current_ticket]\n"

libraries/libmesosphere/include/mesosphere/board/nintendo/nx/kern_cpu_map.hpp

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+
+namespace ams::kern::board::nintendo::nx::impl::cpu {
+
+    /* Virtual to Physical core map. */
+    constexpr inline const s32 VirtualToPhysicalCoreMap[BITSIZEOF(u64)] = {
+        0, 1, 2, 3, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 3,
+    };
+
+}

libraries/libmesosphere/include/mesosphere/board/nintendo/nx/kern_k_device_page_table.hpp

@@ -71,6 +71,10 @@ namespace ams::kern::board::nintendo::nx {
 
             Result Map(size_t *out_mapped_size, const KPageGroup &pg, KDeviceVirtualAddress device_address, ams::svc::MemoryPermission device_perm, bool refresh_mappings);
             Result Unmap(const KPageGroup &pg, KDeviceVirtualAddress device_address);
+
+            void Unmap(KDeviceVirtualAddress device_address, size_t size) {
+                return this->UnmapImpl(device_address, size, false);
+            }
         private:
             Result MapDevicePage(size_t *out_mapped_size, s32 &num_pt, s32 max_pt, KPhysicalAddress phys_addr, u64 size, KDeviceVirtualAddress address, ams::svc::MemoryPermission device_perm);
 

libraries/libmesosphere/include/mesosphere/init/kern_init_arguments_select.hpp

@@ -25,6 +25,5 @@
 namespace ams::kern::init {
 
     KPhysicalAddress GetInitArgumentsAddress(s32 core_id);
-    void SetInitArguments(s32 core_id, KPhysicalAddress address, uintptr_t arg);
 
 }

libraries/libmesosphere/include/mesosphere/init/kern_init_slab_setup.hpp

@@ -34,6 +34,8 @@ namespace ams::kern::init {
         size_t num_KObjectName;
         size_t num_KResourceLimit;
         size_t num_KDebug;
+        size_t num_KAlpha;
+        size_t num_KBeta;
     };
 
     NOINLINE void InitializeSlabResourceCounts();

libraries/libmesosphere/include/mesosphere/kern_k_alpha.hpp

@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_k_auto_object.hpp>
+#include <mesosphere/kern_slab_helpers.hpp>
+
+namespace ams::kern {
+
+    class KAlpha final : public KAutoObjectWithSlabHeapAndContainer<KAlpha, KAutoObjectWithList> {
+        MESOSPHERE_AUTOOBJECT_TRAITS(KAlpha, KAutoObject);
+        private:
+            /* NOTE: Official KAlpha has size 0x50, corresponding to 0x20 bytes of fields. */
+            /* TODO: Add these fields, if KAlpha is ever instantiable in the NX kernel. */
+        public:
+            explicit KAlpha() {
+                /* ... */
+            }
+
+            virtual ~KAlpha() { /* ... */ }
+
+            /* virtual void Finalize() override; */
+
+            virtual bool IsInitialized() const override { return false /* TODO */; }
+            static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
+    };
+
+}

libraries/libmesosphere/include/mesosphere/kern_k_auto_object.hpp

@@ -198,7 +198,7 @@ namespace ams::kern {
                 }
             }
 
-            ALWAYS_INLINE ~KScopedAutoObject() {
+            ~KScopedAutoObject() {
                 if (this->obj != nullptr) {
                     this->obj->Close();
                 }
@@ -206,7 +206,7 @@ namespace ams::kern {
             }
 
             template<typename U> requires (std::derived_from<T, U> || std::derived_from<U, T>)
-            constexpr ALWAYS_INLINE KScopedAutoObject(KScopedAutoObject<U> &&rhs) {
+            constexpr KScopedAutoObject(KScopedAutoObject<U> &&rhs) {
                 if constexpr (std::derived_from<U, T>) {
                     /* Upcast. */
                     this->obj = rhs.obj;

libraries/libmesosphere/include/mesosphere/kern_k_auto_object_container.hpp

@@ -56,8 +56,8 @@ namespace ams::kern {
             void Initialize() { MESOSPHERE_ASSERT_THIS(); }
             void Finalize() { MESOSPHERE_ASSERT_THIS(); }
 
-            Result Register(KAutoObjectWithList *obj);
+            void Register(KAutoObjectWithList *obj);
-            Result Unregister(KAutoObjectWithList *obj);
+            void Unregister(KAutoObjectWithList *obj);
             size_t GetOwnedCount(KProcess *owner);
     };
 

libraries/libmesosphere/include/mesosphere/kern_k_beta.hpp

@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_k_auto_object.hpp>
+#include <mesosphere/kern_slab_helpers.hpp>
+
+namespace ams::kern {
+
+    class KProcess;
+
+    class KBeta final : public KAutoObjectWithSlabHeapAndContainer<KBeta, KAutoObjectWithList> {
+        MESOSPHERE_AUTOOBJECT_TRAITS(KBeta, KAutoObject);
+        private:
+            friend class KProcess;
+        private:
+            /* NOTE: Official KBeta has size 0x88, corresponding to 0x58 bytes of fields. */
+            /* TODO: Add these fields, if KBeta is ever instantiable in the NX kernel. */
+            util::IntrusiveListNode process_list_node;
+        public:
+            explicit KBeta()
+                : process_list_node()
+            {
+                /* ... */
+            }
+
+            virtual ~KBeta() { /* ... */ }
+
+            /* virtual void Finalize() override; */
+
+            virtual bool IsInitialized() const override { return false /* TODO */; }
+            static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
+    };
+
+}

libraries/libmesosphere/include/mesosphere/kern_k_class_token.hpp

@@ -112,6 +112,10 @@ namespace ams::kern {
                 KSessionRequest,
                 KCodeMemory,
 
+                /* NOTE: True order for these has not been determined yet. */
+                KAlpha,
+                KBeta,
+
                 FinalClassesEnd = FinalClassesStart + NumFinalClasses,
             };
 

libraries/libmesosphere/include/mesosphere/kern_k_condition_variable.hpp

@@ -20,8 +20,6 @@
 
 namespace ams::kern {
 
-    extern KThread g_cv_arbiter_compare_thread;
-
     class KConditionVariable {
         public:
             using ThreadTree = typename KThread::ConditionVariableThreadTreeType;

libraries/libmesosphere/include/mesosphere/kern_k_core_local_region.hpp

@@ -1,61 +0,0 @@
-/*
- * Copyright (c) 2018-2020 Atmosphère-NX
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-#pragma once
-#include <mesosphere/kern_common.hpp>
-#include <mesosphere/kern_select_cpu.hpp>
-#include <mesosphere/kern_k_current_context.hpp>
-#include <mesosphere/kern_k_scheduler.hpp>
-#include <mesosphere/kern_k_interrupt_task_manager.hpp>
-#include <mesosphere/kern_select_interrupt_manager.hpp>
-#include <mesosphere/kern_select_hardware_timer.hpp>
-#include <mesosphere/kern_k_memory_manager.hpp>
-
-namespace ams::kern {
-
-    struct KCoreLocalContext {
-        KCurrentContext current;
-        KScheduler scheduler;
-        KInterruptTaskManager interrupt_task_manager;
-        KInterruptManager interrupt_manager;
-        KHardwareTimer hardware_timer;
-        /* Everything after this point is for debugging. */
-        /* Retail kernel doesn't even consistently update these fields. */
-        u64 num_sw_interrupts;
-        u64 num_hw_interrupts;
-        std::atomic<u64> num_svc;
-        u64 num_process_switches;
-        u64 num_thread_switches;
-        u64 num_fpu_switches;
-        u64 num_scheduler_updates;
-        u64 num_invoked_scheduler_updates;
-        std::atomic<u64> num_specific_svc[0x80];
-        u32 perf_counters[6];
-    };
-    static_assert(sizeof(KCoreLocalContext) < PageSize);
-
-    struct KCoreLocalPage {
-        KCoreLocalContext context;
-        u8 padding[PageSize - sizeof(KCoreLocalContext)];
-    };
-    static_assert(sizeof(KCoreLocalPage) == PageSize);
-
-    struct KCoreLocalRegion {
-        KCoreLocalPage current;
-        KCoreLocalPage absolute[cpu::NumCores];
-    };
-    static_assert(sizeof(KCoreLocalRegion) == PageSize * (1 + cpu::NumCores));
-
-}

libraries/libmesosphere/include/mesosphere/kern_k_current_context.hpp

@@ -21,80 +21,24 @@ namespace ams::kern {
     class KThread;
     class KProcess;
     class KScheduler;
-    class KInterruptTaskManager;
-
-    struct KCurrentContext {
-        std::atomic<KThread *> current_thread;
-        std::atomic<KProcess *> current_process;
-        KScheduler *scheduler;
-        KInterruptTaskManager *interrupt_task_manager;
-        s32 core_id;
-        void *exception_stack_top;
-        ams::svc::ThreadLocalRegion *tlr;
-    };
-    static_assert(std::is_standard_layout<KCurrentContext>::value && std::is_trivially_destructible<KCurrentContext>::value);
-    static_assert(sizeof(KCurrentContext) <= cpu::DataCacheLineSize);
-    static_assert(sizeof(std::atomic<KThread *>) == sizeof(KThread *));
-    static_assert(sizeof(std::atomic<KProcess *>) == sizeof(KProcess *));
-
-    namespace impl {
-
-        ALWAYS_INLINE KCurrentContext &GetCurrentContext() {
-            return *reinterpret_cast<KCurrentContext *>(cpu::GetCoreLocalRegionAddress());
-        }
-
-    }
 
     ALWAYS_INLINE KThread *GetCurrentThreadPointer() {
-        return impl::GetCurrentContext().current_thread.load(std::memory_order_relaxed);
+        return reinterpret_cast<KThread *>(cpu::GetCurrentThreadPointerValue());
     }
 
     ALWAYS_INLINE KThread &GetCurrentThread() {
         return *GetCurrentThreadPointer();
     }
 
-    ALWAYS_INLINE KProcess *GetCurrentProcessPointer() {
-        return impl::GetCurrentContext().current_process.load(std::memory_order_relaxed);
-    }
-
-    ALWAYS_INLINE KProcess &GetCurrentProcess() {
-        return *GetCurrentProcessPointer();
-    }
-
-    ALWAYS_INLINE KScheduler *GetCurrentSchedulerPointer() {
-        return impl::GetCurrentContext().scheduler;
-    }
-
-    ALWAYS_INLINE KScheduler &GetCurrentScheduler() {
-        return *GetCurrentSchedulerPointer();
-    }
-
-    ALWAYS_INLINE KInterruptTaskManager *GetCurrentInterruptTaskManagerPointer() {
-        return impl::GetCurrentContext().interrupt_task_manager;
-    }
-
-    ALWAYS_INLINE KInterruptTaskManager &GetCurrentInterruptTaskManager() {
-        return *GetCurrentInterruptTaskManagerPointer();
-    }
-
-    ALWAYS_INLINE s32 GetCurrentCoreId() {
-        return impl::GetCurrentContext().core_id;
-    }
-
-    ALWAYS_INLINE ams::svc::ThreadLocalRegion *GetCurrentThreadLocalRegion() {
-        return impl::GetCurrentContext().tlr;
-    }
-
     ALWAYS_INLINE void SetCurrentThread(KThread *new_thread) {
-        impl::GetCurrentContext().current_thread = new_thread;
+        cpu::SetCurrentThreadPointerValue(reinterpret_cast<uintptr_t>(new_thread));
     }
 
-    ALWAYS_INLINE void SetCurrentProcess(KProcess *new_process) {
+    ALWAYS_INLINE KProcess *GetCurrentProcessPointer();
-        impl::GetCurrentContext().current_process = new_process;
+    ALWAYS_INLINE KProcess &GetCurrentProcess();
-    }
 
-    ALWAYS_INLINE void SetCurrentThreadLocalRegion(void *address) {
+    ALWAYS_INLINE s32 GetCurrentCoreId();
-        impl::GetCurrentContext().tlr = static_cast<ams::svc::ThreadLocalRegion *>(address);
+
-    }
+    ALWAYS_INLINE KScheduler &GetCurrentScheduler();
 
 }

libraries/libmesosphere/include/mesosphere/kern_k_event_info.hpp

@@ -24,7 +24,6 @@ namespace ams::kern {
             struct InfoCreateThread {
                 u32 thread_id;
                 uintptr_t tls_address;
-                uintptr_t entrypoint;
             };
 
             struct InfoExitProcess {

libraries/libmesosphere/include/mesosphere/kern_k_handle_table.hpp

@@ -160,14 +160,16 @@ namespace ams::kern {
                 return this->template GetObjectWithoutPseudoHandle<T>(handle);
             }
 
-            ALWAYS_INLINE KScopedAutoObject<KAutoObject> GetObjectForIpcWithoutPseudoHandle(ams::svc::Handle handle) const {
+            KScopedAutoObject<KAutoObject> GetObjectForIpcWithoutPseudoHandle(ams::svc::Handle handle) const {
                 /* Lock and look up in table. */
                 KScopedDisableDispatch dd;
                 KScopedSpinLock lk(this->lock);
 
                 KAutoObject *obj = this->GetObjectImpl(handle);
-                if (obj->DynamicCast<KInterruptEvent *>() != nullptr) {
+                if (AMS_LIKELY(obj != nullptr)) {
-                    return nullptr;
+                    if (AMS_UNLIKELY(obj->DynamicCast<KInterruptEvent *>() != nullptr)) {
+                        return nullptr;
+                    }
                 }
 
                 return obj;

libraries/libmesosphere/include/mesosphere/kern_k_interrupt_event.hpp

@@ -27,11 +27,10 @@ namespace ams::kern {
     class KInterruptEvent final : public KAutoObjectWithSlabHeapAndContainer<KInterruptEvent, KReadableEvent> {
         MESOSPHERE_AUTOOBJECT_TRAITS(KInterruptEvent, KReadableEvent);
         private:
-            KInterruptEventTask *task;
             s32 interrupt_id;
             bool is_initialized;
         public:
-            constexpr KInterruptEvent() : task(nullptr), interrupt_id(-1), is_initialized(false) { /* ... */ }
+            constexpr KInterruptEvent() : interrupt_id(-1), is_initialized(false) { /* ... */ }
             virtual ~KInterruptEvent() { /* ... */ }
 
             Result Initialize(int32_t interrupt_name, ams::svc::InterruptType type);
@@ -49,17 +48,19 @@ namespace ams::kern {
     class KInterruptEventTask : public KSlabAllocated<KInterruptEventTask>, public KInterruptTask {
         private:
             KInterruptEvent *event;
-            s32 interrupt_id;
+            KLightLock lock;
         public:
-            constexpr KInterruptEventTask() : event(nullptr), interrupt_id(-1) { /* ... */ }
+            constexpr KInterruptEventTask() : event(nullptr), lock() { /* ... */ }
             ~KInterruptEventTask() { /* ... */ }
 
+            KLightLock &GetLock() { return this->lock; }
+
             virtual KInterruptTask *OnInterrupt(s32 interrupt_id) override;
             virtual void DoTask() override;
 
-            void Unregister();
+            void Unregister(s32 interrupt_id);
         public:
-            static Result Register(KInterruptEventTask **out, s32 interrupt_id, bool level, KInterruptEvent *event);
+            static Result Register(s32 interrupt_id, bool level, KInterruptEvent *event);
     };
 
 }

libraries/libmesosphere/include/mesosphere/kern_k_memory_block.hpp

@@ -165,28 +165,42 @@ namespace ams::kern {
 
     enum KMemoryAttribute : u8 {
         KMemoryAttribute_None           = 0x00,
-        KMemoryAttribute_UserMask       = 0x7F,
         KMemoryAttribute_All            = 0xFF,
+        KMemoryAttribute_UserMask       = KMemoryAttribute_All,
 
         KMemoryAttribute_Locked         = ams::svc::MemoryAttribute_Locked,
         KMemoryAttribute_IpcLocked      = ams::svc::MemoryAttribute_IpcLocked,
         KMemoryAttribute_DeviceShared   = ams::svc::MemoryAttribute_DeviceShared,
         KMemoryAttribute_Uncached       = ams::svc::MemoryAttribute_Uncached,
 
-        KMemoryAttribute_AnyLocked      = 0x80,
-
         KMemoryAttribute_SetMask        = KMemoryAttribute_Uncached,
     };
 
+    enum KMemoryBlockDisableMergeAttribute : u8 {
+        KMemoryBlockDisableMergeAttribute_None        = 0,
+        KMemoryBlockDisableMergeAttribute_Normal      = (1u << 0),
+        KMemoryBlockDisableMergeAttribute_DeviceLeft  = (1u << 1),
+        KMemoryBlockDisableMergeAttribute_IpcLeft     = (1u << 2),
+        KMemoryBlockDisableMergeAttribute_Locked      = (1u << 3),
+        KMemoryBlockDisableMergeAttribute_DeviceRight = (1u << 4),
+
+        KMemoryBlockDisableMergeAttribute_AllLeft  = KMemoryBlockDisableMergeAttribute_Normal | KMemoryBlockDisableMergeAttribute_DeviceLeft | KMemoryBlockDisableMergeAttribute_IpcLeft | KMemoryBlockDisableMergeAttribute_Locked,
+        KMemoryBlockDisableMergeAttribute_AllRight = KMemoryBlockDisableMergeAttribute_DeviceRight,
+    };
+
     struct KMemoryInfo {
         uintptr_t address;
         size_t size;
         KMemoryState state;
+        u16 device_disable_merge_left_count;
+        u16 device_disable_merge_right_count;
+        u16 ipc_lock_count;
+        u16 device_use_count;
+        u16 ipc_disable_merge_count;
         KMemoryPermission perm;
         KMemoryAttribute  attribute;
         KMemoryPermission original_perm;
-        u16 ipc_lock_count;
+        KMemoryBlockDisableMergeAttribute disable_merge_attribute;
-        u16 device_use_count;
 
         constexpr ams::svc::MemoryInfo GetSvcMemoryInfo() const {
             return {
@@ -225,6 +239,10 @@ namespace ams::kern {
             return this->ipc_lock_count;
         }
 
+        constexpr u16 GetIpcDisableMergeCount() const {
+            return this->ipc_disable_merge_count;
+        }
+
         constexpr KMemoryState GetState() const {
             return this->state;
         }
@@ -240,18 +258,26 @@ namespace ams::kern {
         constexpr KMemoryAttribute GetAttribute() const {
             return this->attribute;
         }
+
+        constexpr KMemoryBlockDisableMergeAttribute GetDisableMergeAttribute() const {
+            return this->disable_merge_attribute;
+        }
     };
 
     class KMemoryBlock : public util::IntrusiveRedBlackTreeBaseNode<KMemoryBlock> {
         private:
+            u16 device_disable_merge_left_count;
+            u16 device_disable_merge_right_count;
             KProcessAddress address;
             size_t num_pages;
             KMemoryState memory_state;
             u16 ipc_lock_count;
             u16 device_use_count;
+            u16 ipc_disable_merge_count;
             KMemoryPermission perm;
             KMemoryPermission original_perm;
             KMemoryAttribute attribute;
+            KMemoryBlockDisableMergeAttribute disable_merge_attribute;
         public:
             static constexpr ALWAYS_INLINE int Compare(const KMemoryBlock &lhs, const KMemoryBlock &rhs) {
                 if (lhs.GetAddress() < rhs.GetAddress()) {
@@ -287,6 +313,10 @@ namespace ams::kern {
                 return this->ipc_lock_count;
             }
 
+            constexpr u16 GetIpcDisableMergeCount() const {
+                return this->ipc_disable_merge_count;
+            }
+
             constexpr KMemoryPermission GetPermission() const {
                 return this->perm;
             }
@@ -301,25 +331,29 @@ namespace ams::kern {
 
             constexpr KMemoryInfo GetMemoryInfo() const {
                 return {
-                    .address          = GetInteger(this->GetAddress()),
+                    .address                          = GetInteger(this->GetAddress()),
-                    .size             = this->GetSize(),
+                    .size                             = this->GetSize(),
-                    .state            = this->memory_state,
+                    .state                            = this->memory_state,
-                    .perm             = this->perm,
+                    .device_disable_merge_left_count  = this->device_disable_merge_left_count,
-                    .attribute        = this->attribute,
+                    .device_disable_merge_right_count = this->device_disable_merge_right_count,
-                    .original_perm    = this->original_perm,
+                    .ipc_lock_count                   = this->ipc_lock_count,
-                    .ipc_lock_count   = this->ipc_lock_count,
+                    .device_use_count                 = this->device_use_count,
-                    .device_use_count = this->device_use_count,
+                    .ipc_disable_merge_count          = this->ipc_disable_merge_count,
+                    .perm                             = this->perm,
+                    .attribute                        = this->attribute,
+                    .original_perm                    = this->original_perm,
+                    .disable_merge_attribute          = this->disable_merge_attribute,
                 };
             }
         public:
             constexpr KMemoryBlock()
-                : address(), num_pages(), memory_state(KMemoryState_None), ipc_lock_count(), device_use_count(), perm(), original_perm(), attribute()
+                : device_disable_merge_left_count(), device_disable_merge_right_count(), address(), num_pages(), memory_state(KMemoryState_None), ipc_lock_count(), device_use_count(), ipc_disable_merge_count(), perm(), original_perm(), attribute(), disable_merge_attribute()
             {
                 /* ... */
             }
 
             constexpr KMemoryBlock(KProcessAddress addr, size_t np, KMemoryState ms, KMemoryPermission p, KMemoryAttribute attr)
-                : address(addr), num_pages(np), memory_state(ms), ipc_lock_count(0), device_use_count(0), perm(p), original_perm(KMemoryPermission_None), attribute(attr)
+                : device_disable_merge_left_count(), device_disable_merge_right_count(), address(addr), num_pages(np), memory_state(ms), ipc_lock_count(0), device_use_count(0), ipc_disable_merge_count(), perm(p), original_perm(KMemoryPermission_None), attribute(attr), disable_merge_attribute()
             {
                 /* ... */
             }
@@ -353,21 +387,29 @@ namespace ams::kern {
                        this->device_use_count == rhs.device_use_count;
             }
 
+            constexpr bool CanMergeWith(const KMemoryBlock &rhs) const {
+                return this->HasSameProperties(rhs) &&
+                      (this->disable_merge_attribute & KMemoryBlockDisableMergeAttribute_AllRight) == 0 &&
+                      (rhs.disable_merge_attribute & KMemoryBlockDisableMergeAttribute_AllLeft) == 0;
+            }
+
             constexpr bool Contains(KProcessAddress addr) const {
                 MESOSPHERE_ASSERT_THIS();
 
                 return this->GetAddress() <= addr && addr <= this->GetEndAddress();
             }
 
-            constexpr void Add(size_t np) {
+            constexpr void Add(const KMemoryBlock &added_block) {
                 MESOSPHERE_ASSERT_THIS();
-                MESOSPHERE_ASSERT(np > 0);
+                MESOSPHERE_ASSERT(added_block.GetNumPages() > 0);
-                MESOSPHERE_ASSERT(this->GetAddress() + np * PageSize - 1 < this->GetEndAddress() + np * PageSize - 1);
+                MESOSPHERE_ASSERT(this->GetAddress() + added_block.GetSize() - 1 < this->GetEndAddress() + added_block.GetSize() - 1);
 
-                this->num_pages += np;
+                this->num_pages += added_block.GetNumPages();
+                this->disable_merge_attribute = static_cast<KMemoryBlockDisableMergeAttribute>(this->disable_merge_attribute | added_block.disable_merge_attribute);
+                this->device_disable_merge_right_count = added_block.device_disable_merge_right_count;
             }
 
-            constexpr void Update(KMemoryState s, KMemoryPermission p, KMemoryAttribute a) {
+            constexpr void Update(KMemoryState s, KMemoryPermission p, KMemoryAttribute a, bool set_disable_merge_attr, u8 set_mask, u8 clear_mask) {
                 MESOSPHERE_ASSERT_THIS();
                 MESOSPHERE_ASSERT(this->original_perm == KMemoryPermission_None);
                 MESOSPHERE_ASSERT((this->attribute & KMemoryAttribute_IpcLocked) == 0);
@@ -375,6 +417,13 @@ namespace ams::kern {
                 this->memory_state = s;
                 this->perm         = p;
                 this->attribute    = static_cast<KMemoryAttribute>(a | (this->attribute & (KMemoryAttribute_IpcLocked | KMemoryAttribute_DeviceShared)));
+
+                if (set_disable_merge_attr && set_mask != 0) {
+                    this->disable_merge_attribute = static_cast<KMemoryBlockDisableMergeAttribute>(this->disable_merge_attribute | set_mask);
+                }
+                if (clear_mask != 0) {
+                    this->disable_merge_attribute = static_cast<KMemoryBlockDisableMergeAttribute>(this->disable_merge_attribute & ~clear_mask);
+                }
             }
 
             constexpr void Split(KMemoryBlock *block, KProcessAddress addr) {
@@ -383,20 +432,55 @@ namespace ams::kern {
                 MESOSPHERE_ASSERT(this->Contains(addr));
                 MESOSPHERE_ASSERT(util::IsAligned(GetInteger(addr), PageSize));
 
-                block->address          = this->address;
+                block->address                          = this->address;
-                block->num_pages        = (addr - this->GetAddress()) / PageSize;
+                block->num_pages                        = (addr - this->GetAddress()) / PageSize;
-                block->memory_state     = this->memory_state;
+                block->memory_state                     = this->memory_state;
-                block->ipc_lock_count   = this->ipc_lock_count;
+                block->ipc_lock_count                   = this->ipc_lock_count;
-                block->device_use_count = this->device_use_count;
+                block->device_use_count                 = this->device_use_count;
-                block->perm             = this->perm;
+                block->perm                             = this->perm;
-                block->original_perm    = this->original_perm;
+                block->original_perm                    = this->original_perm;
-                block->attribute        = this->attribute;
+                block->attribute                        = this->attribute;
+                block->disable_merge_attribute          = static_cast<KMemoryBlockDisableMergeAttribute>(this->disable_merge_attribute & KMemoryBlockDisableMergeAttribute_AllLeft);
+                block->ipc_disable_merge_count          = this->ipc_disable_merge_count;
+                block->device_disable_merge_left_count  = this->device_disable_merge_left_count;
+                block->device_disable_merge_right_count = 0;
 
                 this->address = addr;
                 this->num_pages -= block->num_pages;
+
+                this->ipc_disable_merge_count         = 0;
+                this->device_disable_merge_left_count = 0;
+                this->disable_merge_attribute         = static_cast<KMemoryBlockDisableMergeAttribute>(this->disable_merge_attribute & KMemoryBlockDisableMergeAttribute_AllRight);
             }
 
-            constexpr void ShareToDevice(KMemoryPermission new_perm) {
+            constexpr void UpdateDeviceDisableMergeStateForShareLeft(KMemoryPermission new_perm, bool left, bool right) {
+                /* New permission/right aren't used. */
+                MESOSPHERE_UNUSED(new_perm, right);
+
+                if (left) {
+                    this->disable_merge_attribute = static_cast<KMemoryBlockDisableMergeAttribute>(this->disable_merge_attribute | KMemoryBlockDisableMergeAttribute_DeviceLeft);
+                    const u16 new_device_disable_merge_left_count = ++this->device_disable_merge_left_count;
+                    MESOSPHERE_ABORT_UNLESS(new_device_disable_merge_left_count > 0);
+                }
+            }
+
+            constexpr void UpdateDeviceDisableMergeStateForShareRight(KMemoryPermission new_perm, bool left, bool right) {
+                /* New permission/left aren't used. */
+                MESOSPHERE_UNUSED(new_perm, left);
+
+                if (right) {
+                    this->disable_merge_attribute = static_cast<KMemoryBlockDisableMergeAttribute>(this->disable_merge_attribute | KMemoryBlockDisableMergeAttribute_DeviceRight);
+                    const u16 new_device_disable_merge_right_count = ++this->device_disable_merge_right_count;
+                    MESOSPHERE_ABORT_UNLESS(new_device_disable_merge_right_count > 0);
+                }
+            }
+
+            constexpr void UpdateDeviceDisableMergeStateForShare(KMemoryPermission new_perm, bool left, bool right) {
+                this->UpdateDeviceDisableMergeStateForShareLeft(new_perm, left, right);
+                this->UpdateDeviceDisableMergeStateForShareRight(new_perm, left, right);
+            }
+
+            constexpr void ShareToDevice(KMemoryPermission new_perm, bool left, bool right) {
                 /* New permission isn't used. */
                 MESOSPHERE_UNUSED(new_perm);
 
@@ -408,9 +492,47 @@ namespace ams::kern {
                 MESOSPHERE_ABORT_UNLESS(new_count > 0);
 
                 this->attribute = static_cast<KMemoryAttribute>(this->attribute | KMemoryAttribute_DeviceShared);
+
+                this->UpdateDeviceDisableMergeStateForShare(new_perm, left, right);
             }
 
-            constexpr void UnshareToDevice(KMemoryPermission new_perm) {
+            constexpr void UpdateDeviceDisableMergeStateForUnshareLeft(KMemoryPermission new_perm, bool left, bool right) {
+                /* New permission/right aren't used. */
+                MESOSPHERE_UNUSED(new_perm, right);
+
+                if (left) {
+                    if (!this->device_disable_merge_left_count) {
+                        return;
+                    }
+                    --this->device_disable_merge_left_count;
+                }
+
+                this->device_disable_merge_left_count = std::min(this->device_disable_merge_left_count, this->device_use_count);
+
+                if (this->device_disable_merge_left_count == 0) {
+                    this->disable_merge_attribute = static_cast<KMemoryBlockDisableMergeAttribute>(this->disable_merge_attribute & ~KMemoryBlockDisableMergeAttribute_DeviceLeft);
+                }
+            }
+
+            constexpr void UpdateDeviceDisableMergeStateForUnshareRight(KMemoryPermission new_perm, bool left, bool right) {
+                /* New permission/left aren't used. */
+                MESOSPHERE_UNUSED(new_perm, left);
+
+                if (right) {
+                    const u16 old_device_disable_merge_right_count = this->device_disable_merge_right_count--;
+                    MESOSPHERE_ASSERT(old_device_disable_merge_right_count > 0);
+                    if (old_device_disable_merge_right_count == 1) {
+                        this->disable_merge_attribute = static_cast<KMemoryBlockDisableMergeAttribute>(this->disable_merge_attribute & ~KMemoryBlockDisableMergeAttribute_DeviceRight);
+                    }
+                }
+            }
+
+            constexpr void UpdateDeviceDisableMergeStateForUnshare(KMemoryPermission new_perm, bool left, bool right) {
+                this->UpdateDeviceDisableMergeStateForUnshareLeft(new_perm, left, right);
+                this->UpdateDeviceDisableMergeStateForUnshareRight(new_perm, left, right);
+            }
+
+            constexpr void UnshareToDevice(KMemoryPermission new_perm, bool left, bool right) {
                 /* New permission isn't used. */
                 MESOSPHERE_UNUSED(new_perm);
 
@@ -424,9 +546,29 @@ namespace ams::kern {
                 if (old_count == 1) {
                     this->attribute = static_cast<KMemoryAttribute>(this->attribute & ~KMemoryAttribute_DeviceShared);
                 }
+
+                this->UpdateDeviceDisableMergeStateForUnshare(new_perm, left, right);
             }
 
-            constexpr void LockForIpc(KMemoryPermission new_perm) {
+            constexpr void UnshareToDeviceRight(KMemoryPermission new_perm, bool left, bool right) {
+                /* New permission isn't used. */
+                MESOSPHERE_UNUSED(new_perm);
+
+                /* We must be shared. */
+                MESOSPHERE_ASSERT((this->attribute & KMemoryAttribute_DeviceShared) == KMemoryAttribute_DeviceShared);
+
+                /* Unhare. */
+                const u16 old_count = this->device_use_count--;
+                MESOSPHERE_ABORT_UNLESS(old_count > 0);
+
+                if (old_count == 1) {
+                    this->attribute = static_cast<KMemoryAttribute>(this->attribute & ~KMemoryAttribute_DeviceShared);
+                }
+
+                this->UpdateDeviceDisableMergeStateForUnshareRight(new_perm, left, right);
+            }
+
+            constexpr void LockForIpc(KMemoryPermission new_perm, bool left, bool right) {
                 /* We must either be locked or have a zero lock count. */
                 MESOSPHERE_ASSERT((this->attribute & KMemoryAttribute_IpcLocked) == KMemoryAttribute_IpcLocked || this->ipc_lock_count == 0);
 
@@ -443,9 +585,16 @@ namespace ams::kern {
                     this->perm          = static_cast<KMemoryPermission>((new_perm & KMemoryPermission_IpcLockChangeMask) | (this->original_perm & ~KMemoryPermission_IpcLockChangeMask));
                 }
                 this->attribute = static_cast<KMemoryAttribute>(this->attribute | KMemoryAttribute_IpcLocked);
+
+                if (left) {
+                    this->disable_merge_attribute = static_cast<KMemoryBlockDisableMergeAttribute>(this->disable_merge_attribute | KMemoryBlockDisableMergeAttribute_IpcLeft);
+                    const u16 new_ipc_disable_merge_count = ++this->ipc_disable_merge_count;
+                    MESOSPHERE_ABORT_UNLESS(new_ipc_disable_merge_count > 0);
+                }
+                MESOSPHERE_UNUSED(right);
             }
 
-            constexpr void UnlockForIpc(KMemoryPermission new_perm) {
+            constexpr void UnlockForIpc(KMemoryPermission new_perm, bool left, bool right) {
                 /* New permission isn't used. */
                 MESOSPHERE_UNUSED(new_perm);
 
@@ -463,6 +612,19 @@ namespace ams::kern {
                     this->original_perm = KMemoryPermission_None;
                     this->attribute = static_cast<KMemoryAttribute>(this->attribute & ~KMemoryAttribute_IpcLocked);
                 }
+
+                if (left) {
+                    const u16 old_ipc_disable_merge_count = this->ipc_disable_merge_count--;
+                    MESOSPHERE_ASSERT(old_ipc_disable_merge_count > 0);
+                    if (old_ipc_disable_merge_count == 1) {
+                        this->disable_merge_attribute = static_cast<KMemoryBlockDisableMergeAttribute>(this->disable_merge_attribute & ~KMemoryBlockDisableMergeAttribute_IpcLeft);
+                    }
+                }
+                MESOSPHERE_UNUSED(right);
+            }
+
+            constexpr KMemoryBlockDisableMergeAttribute GetDisableMergeAttribute() const {
+                return this->disable_merge_attribute;
             }
     };
     static_assert(std::is_trivially_destructible<KMemoryBlock>::value);

libraries/libmesosphere/include/mesosphere/kern_k_memory_block_manager.hpp

@@ -22,39 +22,40 @@ namespace ams::kern {
 
     class KMemoryBlockManagerUpdateAllocator {
         public:
-            static constexpr size_t NumBlocks = 2;
+            static constexpr size_t MaxBlocks = 2;
         private:
-            KMemoryBlock *blocks[NumBlocks];
+            KMemoryBlock *blocks[MaxBlocks];
             size_t index;
             KMemoryBlockSlabManager *slab_manager;
-            Result result;
         public:
-            explicit KMemoryBlockManagerUpdateAllocator(KMemoryBlockSlabManager *sm) : blocks(), index(), slab_manager(sm), result(svc::ResultOutOfResource()) {
+            constexpr explicit KMemoryBlockManagerUpdateAllocator(KMemoryBlockSlabManager *sm) : blocks(), index(MaxBlocks), slab_manager(sm) { /* ... */ }
-                for (size_t i = 0; i < NumBlocks; i++) {
-                    this->blocks[i] = this->slab_manager->Allocate();
-                    if (this->blocks[i] == nullptr) {
-                        this->result = svc::ResultOutOfResource();
-                        return;
-                    }
-                }
-
-                this->result = ResultSuccess();
-            }
 
             ~KMemoryBlockManagerUpdateAllocator() {
-                for (size_t i = 0; i < NumBlocks; i++) {
+                for (const auto &block : this->blocks) {
-                    if (this->blocks[i] != nullptr) {
+                    if (block != nullptr) {
-                        this->slab_manager->Free(this->blocks[i]);
+                        this->slab_manager->Free(block);
                     }
                 }
             }
 
-            Result GetResult() const {
+            Result Initialize(size_t num_blocks) {
-                return this->result;
+                /* Check num blocks. */
+                MESOSPHERE_ASSERT(num_blocks <= MaxBlocks);
+
+                /* Set index. */
+                this->index = MaxBlocks - num_blocks;
+
+                /* Allocate the blocks. */
+                for (size_t i = 0; i < num_blocks && i < MaxBlocks; ++i) {
+                    this->blocks[this->index + i] = this->slab_manager->Allocate();
+                    R_UNLESS(this->blocks[this->index + i] != nullptr, svc::ResultOutOfResource());
+                }
+
+                return ResultSuccess();
             }
 
             KMemoryBlock *Allocate() {
-                MESOSPHERE_ABORT_UNLESS(this->index < NumBlocks);
+                MESOSPHERE_ABORT_UNLESS(this->index < MaxBlocks);
                 MESOSPHERE_ABORT_UNLESS(this->blocks[this->index] != nullptr);
                 KMemoryBlock *block = nullptr;
                 std::swap(block, this->blocks[this->index++]);
@@ -62,7 +63,7 @@ namespace ams::kern {
             }
 
             void Free(KMemoryBlock *block) {
-                MESOSPHERE_ABORT_UNLESS(this->index <= NumBlocks);
+                MESOSPHERE_ABORT_UNLESS(this->index <= MaxBlocks);
                 MESOSPHERE_ABORT_UNLESS(block != nullptr);
                 if (this->index == 0) {
                     this->slab_manager->Free(block);
@@ -75,12 +76,15 @@ namespace ams::kern {
     class KMemoryBlockManager {
         public:
             using MemoryBlockTree = util::IntrusiveRedBlackTreeBaseTraits<KMemoryBlock>::TreeType<KMemoryBlock>;
+            using MemoryBlockLockFunction = void (KMemoryBlock::*)(KMemoryPermission new_perm, bool left, bool right);
             using iterator = MemoryBlockTree::iterator;
             using const_iterator = MemoryBlockTree::const_iterator;
         private:
             MemoryBlockTree memory_block_tree;
             KProcessAddress start_address;
             KProcessAddress end_address;
+        private:
+            void CoalesceForUpdate(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages);
         public:
             constexpr KMemoryBlockManager() : memory_block_tree(), start_address(), end_address() { /* ... */ }
 
@@ -93,8 +97,8 @@ namespace ams::kern {
 
             KProcessAddress FindFreeArea(KProcessAddress region_start, size_t region_num_pages, size_t num_pages, size_t alignment, size_t offset, size_t guard_pages) const;
 
-            void Update(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages, KMemoryState state, KMemoryPermission perm, KMemoryAttribute attr);
+            void Update(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages, KMemoryState state, KMemoryPermission perm, KMemoryAttribute attr, KMemoryBlockDisableMergeAttribute set_disable_attr, KMemoryBlockDisableMergeAttribute clear_disable_attr);
-            void UpdateLock(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages, void (KMemoryBlock::*lock_func)(KMemoryPermission new_perm), KMemoryPermission perm);
+            void UpdateLock(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages, MemoryBlockLockFunction lock_func, KMemoryPermission perm);
 
             void UpdateIfMatch(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages, KMemoryState test_state, KMemoryPermission test_perm, KMemoryAttribute test_attr, KMemoryState state, KMemoryPermission perm, KMemoryAttribute attr);
 

libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp

@@ -112,7 +112,9 @@ namespace ams::kern {
             }
 
             static ALWAYS_INLINE KVirtualAddress GetStackTopAddress(s32 core_id, KMemoryRegionType type) {
-                return Dereference(GetVirtualMemoryRegionTree().FindByTypeAndAttribute(type, static_cast<u32>(core_id))).GetEndAddress();
+                const auto &region = Dereference(GetVirtualMemoryRegionTree().FindByTypeAndAttribute(type, static_cast<u32>(core_id)));
+                MESOSPHERE_INIT_ABORT_UNLESS(region.GetEndAddress() != 0);
+                return region.GetEndAddress();
             }
         public:
             static ALWAYS_INLINE KMemoryRegionTree &GetVirtualMemoryRegionTree()        { return s_virtual_tree; }
@@ -134,7 +136,6 @@ namespace ams::kern {
             static NOINLINE KVirtualAddress GetExceptionStackTopAddress(s32 core_id) { return GetStackTopAddress(core_id, KMemoryRegionType_KernelMiscExceptionStack); }
 
             static NOINLINE KVirtualAddress GetSlabRegionAddress()      { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_KernelSlab)).GetAddress(); }
-            static NOINLINE KVirtualAddress GetCoreLocalRegionAddress() { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_CoreLocalRegion)).GetAddress(); }
 
             static NOINLINE const KMemoryRegion &GetDeviceRegion(KMemoryRegionType type) { return Dereference(GetPhysicalMemoryRegionTree().FindFirstDerived(type)); }
             static KPhysicalAddress GetDevicePhysicalAddress(KMemoryRegionType type) { return GetDeviceRegion(type).GetAddress(); }
@@ -144,7 +145,6 @@ namespace ams::kern {
             static NOINLINE const KMemoryRegion &GetPageTableHeapRegion()  { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_VirtualDramKernelPtHeap)); }
             static NOINLINE const KMemoryRegion &GetKernelStackRegion()    { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_KernelStack)); }
             static NOINLINE const KMemoryRegion &GetTempRegion()           { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_KernelTemp)); }
-            static NOINLINE const KMemoryRegion &GetCoreLocalRegion()      { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_CoreLocalRegion)); }
 
             static NOINLINE const KMemoryRegion &GetKernelTraceBufferRegion() { return Dereference(GetVirtualLinearMemoryRegionTree().FindByType(KMemoryRegionType_VirtualDramKernelTraceBuffer)); }
 
@@ -216,7 +216,6 @@ namespace ams::kern {
     namespace init {
 
         /* These should be generic, regardless of board. */
-        void SetupCoreLocalRegionMemoryRegions(KInitialPageTable &page_table, KInitialPageAllocator &page_allocator);
         void SetupPoolPartitionMemoryRegions();
 
         /* These may be implemented in a board-specific manner. */

libraries/libmesosphere/include/mesosphere/kern_k_memory_manager.hpp

@@ -70,11 +70,13 @@ namespace ams::kern {
                 public:
                     Impl() : heap(), page_reference_counts(), management_region(), pool(), next(), prev() { /* ... */ }
 
-                    size_t Initialize(const KMemoryRegion *region, Pool pool, KVirtualAddress management_region, KVirtualAddress management_region_end);
+                    size_t Initialize(uintptr_t address, size_t size, KVirtualAddress management, KVirtualAddress management_end, Pool p);
 
                     KVirtualAddress AllocateBlock(s32 index, bool random) { return this->heap.AllocateBlock(index, random); }
                     void Free(KVirtualAddress addr, size_t num_pages) { this->heap.Free(addr, num_pages); }
 
+                    void UpdateUsedHeapSize() { this->heap.UpdateUsedSize(); }
+
                     void InitializeOptimizedMemory() { std::memset(GetVoidPointer(this->management_region), 0, CalculateOptimizedProcessOverheadSize(this->heap.GetSize())); }
 
                     void TrackUnoptimizedAllocation(KVirtualAddress block, size_t num_pages);
@@ -96,12 +98,23 @@ namespace ams::kern {
                     constexpr Impl *GetNext() const { return this->next; }
                     constexpr Impl *GetPrev() const { return this->prev; }
 
+                    void OpenFirst(KVirtualAddress address, size_t num_pages) {
+                        size_t index = this->GetPageOffset(address);
+                        const size_t end = index + num_pages;
+                        while (index < end) {
+                            const RefCount ref_count = (++this->page_reference_counts[index]);
+                            MESOSPHERE_ABORT_UNLESS(ref_count == 1);
+
+                            index++;
+                        }
+                    }
+
                     void Open(KVirtualAddress address, size_t num_pages) {
                         size_t index = this->GetPageOffset(address);
                         const size_t end = index + num_pages;
                         while (index < end) {
                             const RefCount ref_count = (++this->page_reference_counts[index]);
-                            MESOSPHERE_ABORT_UNLESS(ref_count > 0);
+                            MESOSPHERE_ABORT_UNLESS(ref_count > 1);
 
                             index++;
                         }
@@ -178,9 +191,9 @@ namespace ams::kern {
             NOINLINE Result InitializeOptimizedMemory(u64 process_id, Pool pool);
             NOINLINE void FinalizeOptimizedMemory(u64 process_id, Pool pool);
 
-            NOINLINE KVirtualAddress AllocateContinuous(size_t num_pages, size_t align_pages, u32 option);
+            NOINLINE KVirtualAddress AllocateAndOpenContinuous(size_t num_pages, size_t align_pages, u32 option);
-            NOINLINE Result Allocate(KPageGroup *out, size_t num_pages, u32 option);
+            NOINLINE Result AllocateAndOpen(KPageGroup *out, size_t num_pages, u32 option);
-            NOINLINE Result AllocateForProcess(KPageGroup *out, size_t num_pages, u32 option, u64 process_id, u8 fill_pattern);
+            NOINLINE Result AllocateAndOpenForProcess(KPageGroup *out, size_t num_pages, u32 option, u64 process_id, u8 fill_pattern);
 
             void Open(KVirtualAddress address, size_t num_pages) {
                 /* Repeatedly open references until we've done so for all pages. */

libraries/libmesosphere/include/mesosphere/kern_k_memory_region.hpp

@@ -29,7 +29,7 @@ namespace ams::kern {
         private:
             uintptr_t address;
             uintptr_t pair_address;
-            size_t region_size;
+            uintptr_t last_address;
             u32 attributes;
             u32 type_id;
         public:
@@ -43,18 +43,18 @@ namespace ams::kern {
                 }
             }
         public:
-            constexpr ALWAYS_INLINE KMemoryRegion() : address(0), pair_address(0), region_size(0), attributes(0), type_id(0) { /* ... */ }
+            constexpr ALWAYS_INLINE KMemoryRegion() : address(0), pair_address(0), last_address(0), attributes(0), type_id(0) { /* ... */ }
-            constexpr ALWAYS_INLINE KMemoryRegion(uintptr_t a, size_t rs, uintptr_t p, u32 r, u32 t) :
+            constexpr ALWAYS_INLINE KMemoryRegion(uintptr_t a, size_t la, uintptr_t p, u32 r, u32 t) :
-                address(a), pair_address(p), region_size(rs), attributes(r), type_id(t)
+                address(a), pair_address(p), last_address(la), attributes(r), type_id(t)
             {
                 /* ... */
             }
-            constexpr ALWAYS_INLINE KMemoryRegion(uintptr_t a, size_t rs, u32 r, u32 t) : KMemoryRegion(a, rs, std::numeric_limits<uintptr_t>::max(), r, t) { /* ... */ }
+            constexpr ALWAYS_INLINE KMemoryRegion(uintptr_t a, size_t la, u32 r, u32 t) : KMemoryRegion(a, la, std::numeric_limits<uintptr_t>::max(), r, t) { /* ... */ }
         private:
-            constexpr ALWAYS_INLINE void Reset(uintptr_t a, uintptr_t rs, uintptr_t p, u32 r, u32 t) {
+            constexpr ALWAYS_INLINE void Reset(uintptr_t a, uintptr_t la, uintptr_t p, u32 r, u32 t) {
                 this->address      = a;
                 this->pair_address = p;
-                this->region_size  = rs;
+                this->last_address  = la;
                 this->attributes   = r;
                 this->type_id      = t;
             }
@@ -67,16 +67,16 @@ namespace ams::kern {
                 return this->pair_address;
             }
 
-            constexpr ALWAYS_INLINE size_t GetSize() const {
+            constexpr ALWAYS_INLINE uintptr_t GetLastAddress() const {
-                return this->region_size;
+                return this->last_address;
             }
 
             constexpr ALWAYS_INLINE uintptr_t GetEndAddress() const {
-                return this->GetAddress() + this->GetSize();
+                return this->GetLastAddress() + 1;
             }
 
-            constexpr ALWAYS_INLINE uintptr_t GetLastAddress() const {
+            constexpr ALWAYS_INLINE size_t GetSize() const {
-                return this->GetEndAddress() - 1;
+                return this->GetEndAddress() - this->GetAddress();
             }
 
             constexpr ALWAYS_INLINE u32 GetAttributes() const {
@@ -93,6 +93,7 @@ namespace ams::kern {
             }
 
             constexpr ALWAYS_INLINE bool Contains(uintptr_t address) const {
+                MESOSPHERE_INIT_ABORT_UNLESS(this->GetEndAddress() != 0);
                 return this->GetAddress() <= address && address <= this->GetLastAddress();
             }
 
@@ -130,17 +131,17 @@ namespace ams::kern {
                     return this->first_region->GetAddress();
                 }
 
+                constexpr ALWAYS_INLINE uintptr_t GetLastAddress() const {
+                    return this->last_region->GetLastAddress();
+                }
+
                 constexpr ALWAYS_INLINE uintptr_t GetEndAddress() const {
-                    return this->last_region->GetEndAddress();
+                    return this->GetLastAddress() + 1;
                 }
 
                 constexpr ALWAYS_INLINE size_t GetSize() const {
                     return this->GetEndAddress() - this->GetAddress();
                 }
-
-                constexpr ALWAYS_INLINE uintptr_t GetLastAddress() const {
-                    return this->GetEndAddress() - 1;
-                }
             };
         private:
             using TreeType = util::IntrusiveRedBlackTreeBaseTraits<KMemoryRegion>::TreeType<KMemoryRegion>;
@@ -160,7 +161,7 @@ namespace ams::kern {
             constexpr ALWAYS_INLINE KMemoryRegionTree() : tree() { /* ... */ }
         public:
             KMemoryRegion *FindModifiable(uintptr_t address) {
-                if (auto it = this->find(KMemoryRegion(address, 1, 0, 0)); it != this->end()) {
+                if (auto it = this->find(KMemoryRegion(address, address, 0, 0)); it != this->end()) {
                     return std::addressof(*it);
                 } else {
                     return nullptr;
@@ -168,7 +169,7 @@ namespace ams::kern {
             }
 
             const KMemoryRegion *Find(uintptr_t address) const {
-                if (auto it = this->find(KMemoryRegion(address, 1, 0, 0)); it != this->cend()) {
+                if (auto it = this->find(KMemoryRegion(address, address, 0, 0)); it != this->cend()) {
                     return std::addressof(*it);
                 } else {
                     return nullptr;
@@ -234,7 +235,7 @@ namespace ams::kern {
                 return extents;
             }
         public:
-            NOINLINE void InsertDirectly(uintptr_t address, size_t size, u32 attr = 0, u32 type_id = 0);
+            NOINLINE void InsertDirectly(uintptr_t address, uintptr_t last_address, u32 attr = 0, u32 type_id = 0);
             NOINLINE bool Insert(uintptr_t address, size_t size, u32 type_id, u32 new_attr = 0, u32 old_attr = 0);
 
             NOINLINE KVirtualAddress GetRandomAlignedRegion(size_t size, size_t alignment, u32 type_id);

libraries/libmesosphere/include/mesosphere/kern_k_memory_region_type.hpp

@@ -156,12 +156,13 @@ namespace ams::kern {
 
     constexpr inline const auto KMemoryRegionType_None = impl::KMemoryRegionTypeValue();
 
-    constexpr inline const auto KMemoryRegionType_Kernel          = KMemoryRegionType_None.DeriveInitial(0, 2);
+    constexpr inline const auto KMemoryRegionType_Kernel = KMemoryRegionType_None.DeriveInitial(0, 2);
-    constexpr inline const auto KMemoryRegionType_Dram            = KMemoryRegionType_None.DeriveInitial(1, 2);
+    constexpr inline const auto KMemoryRegionType_Dram   = KMemoryRegionType_None.DeriveInitial(1, 2);
-    constexpr inline const auto KMemoryRegionType_CoreLocalRegion = KMemoryRegionType_None.DeriveInitial(2).Finalize();
+    static_assert(KMemoryRegionType_Kernel.GetValue() == 0x1);
-    static_assert(KMemoryRegionType_Kernel         .GetValue() == 0x1);
+    static_assert(KMemoryRegionType_Dram  .GetValue() == 0x2);
-    static_assert(KMemoryRegionType_Dram           .GetValue() == 0x2);
+
-    static_assert(KMemoryRegionType_CoreLocalRegion.GetValue() == 0x4);
+    /* constexpr inline const auto KMemoryRegionType_CoreLocalRegion = KMemoryRegionType_None.DeriveInitial(2).Finalize(); */
+    /* static_assert(KMemoryRegionType_CoreLocalRegion.GetValue() == 0x4); */
 
     constexpr inline const auto KMemoryRegionType_DramKernelBase   = KMemoryRegionType_Dram.DeriveSparse(0, 3, 0).SetAttribute(KMemoryRegionAttr_NoUserMap).SetAttribute(KMemoryRegionAttr_CarveoutProtected);
     constexpr inline const auto KMemoryRegionType_DramReservedBase = KMemoryRegionType_Dram.DeriveSparse(0, 3, 1);
@@ -274,15 +275,15 @@ namespace ams::kern {
                                                                                                          /* UNUSED: .Derive(7, 0); */
     constexpr inline const auto KMemoryRegionType_KernelMiscMainStack      = KMemoryRegionType_KernelMiscDerivedBase.Derive(7, 1);
     constexpr inline const auto KMemoryRegionType_KernelMiscMappedDevice   = KMemoryRegionType_KernelMiscDerivedBase.Derive(7, 2);
-    constexpr inline const auto KMemoryRegionType_KernelMiscIdleStack      = KMemoryRegionType_KernelMiscDerivedBase.Derive(7, 3);
+    constexpr inline const auto KMemoryRegionType_KernelMiscExceptionStack = KMemoryRegionType_KernelMiscDerivedBase.Derive(7, 3);
     constexpr inline const auto KMemoryRegionType_KernelMiscUnknownDebug   = KMemoryRegionType_KernelMiscDerivedBase.Derive(7, 4);
                                                                                                          /* UNUSED: .Derive(7, 5); */
-    constexpr inline const auto KMemoryRegionType_KernelMiscExceptionStack = KMemoryRegionType_KernelMiscDerivedBase.Derive(7, 6);
+    constexpr inline const auto KMemoryRegionType_KernelMiscIdleStack = KMemoryRegionType_KernelMiscDerivedBase.Derive(7, 6);
     static_assert(KMemoryRegionType_KernelMiscMainStack     .GetValue() == 0xB49);
     static_assert(KMemoryRegionType_KernelMiscMappedDevice  .GetValue() == 0xD49);
-    static_assert(KMemoryRegionType_KernelMiscIdleStack     .GetValue() == 0x1349);
+    static_assert(KMemoryRegionType_KernelMiscExceptionStack.GetValue() == 0x1349);
     static_assert(KMemoryRegionType_KernelMiscUnknownDebug  .GetValue() == 0x1549);
-    static_assert(KMemoryRegionType_KernelMiscExceptionStack.GetValue() == 0x2349);
+    static_assert(KMemoryRegionType_KernelMiscIdleStack     .GetValue() == 0x2349);
 
     constexpr inline const auto KMemoryRegionType_KernelTemp = KMemoryRegionType_Kernel.Advance(2).Derive(2, 0);
     static_assert(KMemoryRegionType_KernelTemp.GetValue() == 0x31);

libraries/libmesosphere/include/mesosphere/kern_k_page_table_base.hpp

@@ -24,13 +24,28 @@
 
 namespace ams::kern {
 
+    enum DisableMergeAttribute : u8 {
+        DisableMergeAttribute_None                       = (0u << 0),
+
+        DisableMergeAttribute_DisableHead                = (1u << 0),
+        DisableMergeAttribute_DisableHeadAndBody         = (1u << 1),
+        DisableMergeAttribute_EnableHeadAndBody          = (1u << 2),
+        DisableMergeAttribute_DisableTail                = (1u << 3),
+        DisableMergeAttribute_EnableTail                 = (1u << 4),
+        DisableMergeAttribute_EnableAndMergeHeadBodyTail = (1u << 5),
+
+        DisableMergeAttribute_EnableHeadBodyTail         = DisableMergeAttribute_EnableHeadAndBody  | DisableMergeAttribute_EnableTail,
+        DisableMergeAttribute_DisableHeadBodyTail        = DisableMergeAttribute_DisableHeadAndBody | DisableMergeAttribute_DisableTail,
+    };
+
     struct KPageProperties {
         KMemoryPermission perm;
         bool io;
         bool uncached;
-        bool non_contiguous;
+        DisableMergeAttribute disable_merge_attributes;
     };
     static_assert(std::is_trivial<KPageProperties>::value);
+    static_assert(sizeof(KPageProperties) == sizeof(u32));
 
     class KPageTableBase {
         NON_COPYABLE(KPageTableBase);
@@ -143,6 +158,7 @@ namespace ams::kern {
             u32 address_space_width;
             bool is_kernel;
             bool enable_aslr;
+            bool enable_device_address_space_merge;
             KMemoryBlockSlabManager *memory_block_slab_manager;
             KBlockInfoManager *block_info_manager;
             const KMemoryRegion *cached_physical_linear_region;
@@ -157,15 +173,15 @@ namespace ams::kern {
                 alias_region_start(), alias_region_end(), stack_region_start(), stack_region_end(), kernel_map_region_start(),
                 kernel_map_region_end(), alias_code_region_start(), alias_code_region_end(), code_region_start(), code_region_end(),
                 max_heap_size(), mapped_physical_memory_size(), mapped_unsafe_physical_memory(), general_lock(), map_physical_memory_lock(),
-                impl(), memory_block_manager(), allocate_option(), address_space_width(), is_kernel(), enable_aslr(), memory_block_slab_manager(),
+                impl(), memory_block_manager(), allocate_option(), address_space_width(), is_kernel(), enable_aslr(), enable_device_address_space_merge(),
-                block_info_manager(), cached_physical_linear_region(), cached_physical_heap_region(), cached_virtual_heap_region(),
+                memory_block_slab_manager(), block_info_manager(), cached_physical_linear_region(), cached_physical_heap_region(), cached_virtual_heap_region(),
                 heap_fill_value(), ipc_fill_value(), stack_fill_value()
             {
                 /* ... */
             }
 
             NOINLINE Result InitializeForKernel(bool is_64_bit, void *table, KVirtualAddress start, KVirtualAddress end);
-            NOINLINE Result InitializeForProcess(ams::svc::CreateProcessFlag as_type, bool enable_aslr, bool from_back, KMemoryManager::Pool pool, void *table, KProcessAddress start, KProcessAddress end, KProcessAddress code_address, size_t code_size, KMemoryBlockSlabManager *mem_block_slab_manager, KBlockInfoManager *block_info_manager);
+            NOINLINE Result InitializeForProcess(ams::svc::CreateProcessFlag as_type, bool enable_aslr, bool enable_device_address_space_merge, bool from_back, KMemoryManager::Pool pool, void *table, KProcessAddress start, KProcessAddress end, KProcessAddress code_address, size_t code_size, KMemoryBlockSlabManager *mem_block_slab_manager, KBlockInfoManager *block_info_manager);
 
             void Finalize();
 
@@ -251,12 +267,18 @@ namespace ams::kern {
             constexpr size_t GetNumGuardPages() const { return this->IsKernel() ? 1 : 4; }
             ALWAYS_INLINE KProcessAddress FindFreeArea(KProcessAddress region_start, size_t region_num_pages, size_t num_pages, size_t alignment, size_t offset, size_t guard_pages) const;
 
-            Result CheckMemoryStateContiguous(KProcessAddress addr, size_t size, u32 state_mask, u32 state, u32 perm_mask, u32 perm, u32 attr_mask, u32 attr) const;
+            Result CheckMemoryStateContiguous(size_t *out_blocks_needed, KProcessAddress addr, size_t size, u32 state_mask, u32 state, u32 perm_mask, u32 perm, u32 attr_mask, u32 attr) const;
+            Result CheckMemoryStateContiguous(KProcessAddress addr, size_t size, u32 state_mask, u32 state, u32 perm_mask, u32 perm, u32 attr_mask, u32 attr) const {
+                return this->CheckMemoryStateContiguous(nullptr, addr, size, state_mask, state, perm_mask, perm, attr_mask, attr);
+            }
 
             Result CheckMemoryState(const KMemoryInfo &info, u32 state_mask, u32 state, u32 perm_mask, u32 perm, u32 attr_mask, u32 attr) const;
-            Result CheckMemoryState(KMemoryState *out_state, KMemoryPermission *out_perm, KMemoryAttribute *out_attr, KProcessAddress addr, size_t size, u32 state_mask, u32 state, u32 perm_mask, u32 perm, u32 attr_mask, u32 attr, u32 ignore_attr = DefaultMemoryIgnoreAttr) const;
+            Result CheckMemoryState(KMemoryState *out_state, KMemoryPermission *out_perm, KMemoryAttribute *out_attr, size_t *out_blocks_needed, KProcessAddress addr, size_t size, u32 state_mask, u32 state, u32 perm_mask, u32 perm, u32 attr_mask, u32 attr, u32 ignore_attr = DefaultMemoryIgnoreAttr) const;
+            Result CheckMemoryState(size_t *out_blocks_needed, KProcessAddress addr, size_t size, u32 state_mask, u32 state, u32 perm_mask, u32 perm, u32 attr_mask, u32 attr, u32 ignore_attr = DefaultMemoryIgnoreAttr) const {
+                return this->CheckMemoryState(nullptr, nullptr, nullptr, out_blocks_needed, addr, size, state_mask, state, perm_mask, perm, attr_mask, attr, ignore_attr);
+            }
             Result CheckMemoryState(KProcessAddress addr, size_t size, u32 state_mask, u32 state, u32 perm_mask, u32 perm, u32 attr_mask, u32 attr, u32 ignore_attr = DefaultMemoryIgnoreAttr) const {
-                return this->CheckMemoryState(nullptr, nullptr, nullptr, addr, size, state_mask, state, perm_mask, perm, attr_mask, attr, ignore_attr);
+                return this->CheckMemoryState(nullptr, addr, size, state_mask, state, perm_mask, perm, attr_mask, attr, ignore_attr);
             }
 
             Result LockMemoryAndOpen(KPageGroup *out_pg, KPhysicalAddress *out_paddr, KProcessAddress addr, size_t size, u32 state_mask, u32 state, u32 perm_mask, u32 perm, u32 attr_mask, u32 attr, KMemoryPermission new_perm, u32 lock_attr);
@@ -266,17 +288,19 @@ namespace ams::kern {
 
             Result QueryMappingImpl(KProcessAddress *out, KPhysicalAddress address, size_t size, KMemoryState state) const;
 
-            Result AllocateAndMapPagesImpl(PageLinkedList *page_list, KProcessAddress address, size_t num_pages, const KPageProperties properties);
+            Result AllocateAndMapPagesImpl(PageLinkedList *page_list, KProcessAddress address, size_t num_pages, KMemoryPermission perm);
             Result MapPageGroupImpl(PageLinkedList *page_list, KProcessAddress address, const KPageGroup &pg, const KPageProperties properties, bool reuse_ll);
 
+            void RemapPageGroup(PageLinkedList *page_list, KProcessAddress address, size_t size, const KPageGroup &pg);
+
             Result MakePageGroup(KPageGroup &pg, KProcessAddress addr, size_t num_pages);
             bool IsValidPageGroup(const KPageGroup &pg, KProcessAddress addr, size_t num_pages);
 
             NOINLINE Result MapPages(KProcessAddress *out_addr, size_t num_pages, size_t alignment, KPhysicalAddress phys_addr, bool is_pa_valid, KProcessAddress region_start, size_t region_num_pages, KMemoryState state, KMemoryPermission perm);
 
-            Result SetupForIpcClient(PageLinkedList *page_list, KProcessAddress address, size_t size, KMemoryPermission test_perm, KMemoryState dst_state);
+            Result SetupForIpcClient(PageLinkedList *page_list, size_t *out_blocks_needed, KProcessAddress address, size_t size, KMemoryPermission test_perm, KMemoryState dst_state);
             Result SetupForIpcServer(KProcessAddress *out_addr, size_t size, KProcessAddress src_addr, KMemoryPermission test_perm, KMemoryState dst_state, KPageTableBase &src_page_table, bool send);
-            Result CleanupForIpcClientOnServerSetupFailure(PageLinkedList *page_list, KProcessAddress address, size_t size, KMemoryPermission test_perm);
+            void CleanupForIpcClientOnServerSetupFailure(PageLinkedList *page_list, KProcessAddress address, size_t size, KMemoryPermission prot_perm);
         public:
             bool GetPhysicalAddress(KPhysicalAddress *out, KProcessAddress virt_addr) const {
                 return this->GetImpl().GetPhysicalAddress(out, virt_addr);
@@ -321,7 +345,6 @@ namespace ams::kern {
             Result UnmapPageGroup(KProcessAddress address, const KPageGroup &pg, KMemoryState state);
 
             Result MakeAndOpenPageGroup(KPageGroup *out, KProcessAddress address, size_t num_pages, u32 state_mask, u32 state, u32 perm_mask, u32 perm, u32 attr_mask, u32 attr);
-            Result MakeAndOpenPageGroupContiguous(KPageGroup *out, KProcessAddress address, size_t num_pages, u32 state_mask, u32 state, u32 perm_mask, u32 perm, u32 attr_mask, u32 attr);
 
             Result InvalidateProcessDataCache(KProcessAddress address, size_t size);
 
@@ -330,6 +353,10 @@ namespace ams::kern {
 
             Result LockForDeviceAddressSpace(KPageGroup *out, KProcessAddress address, size_t size, KMemoryPermission perm, bool is_aligned);
             Result UnlockForDeviceAddressSpace(KProcessAddress address, size_t size);
+
+            Result MakePageGroupForUnmapDeviceAddressSpace(KPageGroup *out, KProcessAddress address, size_t size);
+            Result UnlockForDeviceAddressSpacePartialMap(KProcessAddress address, size_t size, size_t mapped_size);
+
             Result LockForIpcUserBuffer(KPhysicalAddress *out, KProcessAddress address, size_t size);
             Result UnlockForIpcUserBuffer(KProcessAddress address, size_t size);
 

libraries/libmesosphere/include/mesosphere/kern_k_process.hpp

@@ -22,6 +22,7 @@
 #include <mesosphere/kern_k_thread.hpp>
 #include <mesosphere/kern_k_thread_local_page.hpp>
 #include <mesosphere/kern_k_shared_memory_info.hpp>
+#include <mesosphere/kern_k_beta.hpp>
 #include <mesosphere/kern_k_worker_task.hpp>
 #include <mesosphere/kern_select_page_table.hpp>
 #include <mesosphere/kern_k_condition_variable.hpp>
@@ -52,6 +53,7 @@ namespace ams::kern {
             static constexpr size_t AslrAlignment = KernelAslrAlignment;
         private:
             using SharedMemoryInfoList = util::IntrusiveListBaseTraits<KSharedMemoryInfo>::ListType;
+            using BetaList = util::IntrusiveListMemberTraits<&KBeta::process_list_node>::ListType;
             using TLPTree = util::IntrusiveRedBlackTreeBaseTraits<KThreadLocalPage>::TreeType<KThreadLocalPage>;
             using TLPIterator = TLPTree::iterator;
         private:
@@ -95,6 +97,7 @@ namespace ams::kern {
             KThread                    *exception_thread{};
             ThreadList                  thread_list{};
             SharedMemoryInfoList        shared_memory_list{};
+            BetaList                    beta_list{};
             bool                        is_suspended{};
             bool                        is_jit_debug{};
             ams::svc::DebugEvent        jit_debug_event_type{};
@@ -361,9 +364,6 @@ namespace ams::kern {
             static void Switch(KProcess *cur_process, KProcess *next_process) {
                 MESOSPHERE_UNUSED(cur_process);
 
-                /* Set the current process pointer. */
-                SetCurrentProcess(next_process);
-
                 /* Update the current page table. */
                 if (next_process) {
                     next_process->GetPageTable().Activate(next_process->GetProcessId());

libraries/libmesosphere/include/mesosphere/kern_k_resource_limit.hpp

@@ -28,11 +28,12 @@ namespace ams::kern {
             s64 limit_values[ams::svc::LimitableResource_Count];
             s64 current_values[ams::svc::LimitableResource_Count];
             s64 current_hints[ams::svc::LimitableResource_Count];
+            s64 peak_values[ams::svc::LimitableResource_Count];
             mutable KLightLock lock;
             s32 waiter_count;
             KLightConditionVariable cond_var;
         public:
-            constexpr ALWAYS_INLINE KResourceLimit() : limit_values(), current_values(), current_hints(), lock(), waiter_count(), cond_var() { /* ... */ }
+            constexpr ALWAYS_INLINE KResourceLimit() : limit_values(), current_values(), current_hints(), peak_values(), lock(), waiter_count(), cond_var() { /* ... */ }
             virtual ~KResourceLimit() { /* ... */ }
 
             static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
@@ -42,6 +43,7 @@ namespace ams::kern {
 
             s64 GetLimitValue(ams::svc::LimitableResource which) const;
             s64 GetCurrentValue(ams::svc::LimitableResource which) const;
+            s64 GetPeakValue(ams::svc::LimitableResource which) const;
             s64 GetFreeValue(ams::svc::LimitableResource which) const;
 
             Result SetLimitValue(ams::svc::LimitableResource which, s64 value);

libraries/libmesosphere/include/mesosphere/kern_k_scheduler.hpp

@@ -56,9 +56,10 @@ namespace ams::kern {
             KThread *prev_thread;
             s64 last_context_switch_time;
             KThread *idle_thread;
+            std::atomic<KThread *> current_thread;
         public:
             constexpr KScheduler()
-                : state(), is_active(false), core_id(0), prev_thread(nullptr), last_context_switch_time(0), idle_thread(nullptr)
+                : state(), is_active(false), core_id(0), prev_thread(nullptr), last_context_switch_time(0), idle_thread(nullptr), current_thread(nullptr)
             {
                 this->state.needs_scheduling = true;
                 this->state.interrupt_task_thread_runnable = false;
@@ -96,6 +97,10 @@ namespace ams::kern {
                 return this->prev_thread;
             }
 
+            ALWAYS_INLINE KThread *GetSchedulerCurrentThread() const {
+                return this->current_thread;
+            }
+
             ALWAYS_INLINE s64 GetLastContextSwitchTime() const {
                 return this->last_context_switch_time;
             }

libraries/libmesosphere/include/mesosphere/kern_k_scheduler_lock.hpp

@@ -45,7 +45,7 @@ namespace ams::kern {
                 return this->owner_thread == GetCurrentThreadPointer();
             }
 
-            ALWAYS_INLINE void Lock() {
+            void Lock() {
                 MESOSPHERE_ASSERT_THIS();
 
                 if (this->IsLockedByCurrentThread()) {
@@ -67,7 +67,7 @@ namespace ams::kern {
                 }
             }
 
-            ALWAYS_INLINE void Unlock() {
+            void Unlock() {
                 MESOSPHERE_ASSERT_THIS();
                 MESOSPHERE_ASSERT(this->IsLockedByCurrentThread());
                 MESOSPHERE_ASSERT(this->lock_count > 0);

libraries/libmesosphere/include/mesosphere/kern_k_scoped_scheduler_lock_and_sleep.hpp

@@ -37,7 +37,7 @@ namespace ams::kern {
                 *out_timer = this->timer;
             }
 
-            ALWAYS_INLINE ~KScopedSchedulerLockAndSleep() {
+            ~KScopedSchedulerLockAndSleep() {
                 /* Register the sleep. */
                 if (this->timeout_tick > 0) {
                     this->timer->RegisterAbsoluteTask(this->thread, this->timeout_tick);

libraries/libmesosphere/include/mesosphere/kern_k_session_request.hpp

@@ -45,10 +45,10 @@ namespace ams::kern {
                                 this->state          = st;
                             }
 
-                            constexpr KProcessAddress GetClientAddress() const { return this->client_address; }
+                            constexpr ALWAYS_INLINE KProcessAddress GetClientAddress() const { return this->client_address; }
-                            constexpr KProcessAddress GetServerAddress() const { return this->server_address; }
+                            constexpr ALWAYS_INLINE KProcessAddress GetServerAddress() const { return this->server_address; }
-                            constexpr size_t GetSize() const { return this->size; }
+                            constexpr ALWAYS_INLINE size_t GetSize() const { return this->size; }
-                            constexpr KMemoryState GetMemoryState() const { return this->state; }
+                            constexpr ALWAYS_INLINE KMemoryState GetMemoryState() const { return this->state; }
                     };
                 private:
                     Mapping static_mappings[NumStaticMappings];
@@ -62,32 +62,32 @@ namespace ams::kern {
                     void Initialize() { /* ... */ }
                     void Finalize();
 
-                    constexpr size_t GetSendCount() const { return this->num_send; }
+                    constexpr ALWAYS_INLINE size_t GetSendCount() const { return this->num_send; }
-                    constexpr size_t GetReceiveCount() const { return this->num_recv; }
+                    constexpr ALWAYS_INLINE size_t GetReceiveCount() const { return this->num_recv; }
-                    constexpr size_t GetExchangeCount() const { return this->num_exch; }
+                    constexpr ALWAYS_INLINE size_t GetExchangeCount() const { return this->num_exch; }
 
                     Result PushSend(KProcessAddress client, KProcessAddress server, size_t size, KMemoryState state);
                     Result PushReceive(KProcessAddress client, KProcessAddress server, size_t size, KMemoryState state);
                     Result PushExchange(KProcessAddress client, KProcessAddress server, size_t size, KMemoryState state);
 
-                    constexpr KProcessAddress GetSendClientAddress(size_t i) const { return GetSendMapping(i).GetClientAddress(); }
+                    constexpr ALWAYS_INLINE KProcessAddress GetSendClientAddress(size_t i) const { return GetSendMapping(i).GetClientAddress(); }
-                    constexpr KProcessAddress GetSendServerAddress(size_t i) const { return GetSendMapping(i).GetServerAddress(); }
+                    constexpr ALWAYS_INLINE KProcessAddress GetSendServerAddress(size_t i) const { return GetSendMapping(i).GetServerAddress(); }
-                    constexpr size_t          GetSendSize(size_t i)          const { return GetSendMapping(i).GetSize(); }
+                    constexpr ALWAYS_INLINE size_t          GetSendSize(size_t i)          const { return GetSendMapping(i).GetSize(); }
-                    constexpr KMemoryState    GetSendMemoryState(size_t i)   const { return GetSendMapping(i).GetMemoryState(); }
+                    constexpr ALWAYS_INLINE KMemoryState    GetSendMemoryState(size_t i)   const { return GetSendMapping(i).GetMemoryState(); }
 
-                    constexpr KProcessAddress GetReceiveClientAddress(size_t i) const { return GetReceiveMapping(i).GetClientAddress(); }
+                    constexpr ALWAYS_INLINE KProcessAddress GetReceiveClientAddress(size_t i) const { return GetReceiveMapping(i).GetClientAddress(); }
-                    constexpr KProcessAddress GetReceiveServerAddress(size_t i) const { return GetReceiveMapping(i).GetServerAddress(); }
+                    constexpr ALWAYS_INLINE KProcessAddress GetReceiveServerAddress(size_t i) const { return GetReceiveMapping(i).GetServerAddress(); }
-                    constexpr size_t          GetReceiveSize(size_t i)          const { return GetReceiveMapping(i).GetSize(); }
+                    constexpr ALWAYS_INLINE size_t          GetReceiveSize(size_t i)          const { return GetReceiveMapping(i).GetSize(); }
-                    constexpr KMemoryState    GetReceiveMemoryState(size_t i)   const { return GetReceiveMapping(i).GetMemoryState(); }
+                    constexpr ALWAYS_INLINE KMemoryState    GetReceiveMemoryState(size_t i)   const { return GetReceiveMapping(i).GetMemoryState(); }
 
-                    constexpr KProcessAddress GetExchangeClientAddress(size_t i) const { return GetExchangeMapping(i).GetClientAddress(); }
+                    constexpr ALWAYS_INLINE KProcessAddress GetExchangeClientAddress(size_t i) const { return GetExchangeMapping(i).GetClientAddress(); }
-                    constexpr KProcessAddress GetExchangeServerAddress(size_t i) const { return GetExchangeMapping(i).GetServerAddress(); }
+                    constexpr ALWAYS_INLINE KProcessAddress GetExchangeServerAddress(size_t i) const { return GetExchangeMapping(i).GetServerAddress(); }
-                    constexpr size_t          GetExchangeSize(size_t i)          const { return GetExchangeMapping(i).GetSize(); }
+                    constexpr ALWAYS_INLINE size_t          GetExchangeSize(size_t i)          const { return GetExchangeMapping(i).GetSize(); }
-                    constexpr KMemoryState    GetExchangeMemoryState(size_t i)   const { return GetExchangeMapping(i).GetMemoryState(); }
+                    constexpr ALWAYS_INLINE KMemoryState    GetExchangeMemoryState(size_t i)   const { return GetExchangeMapping(i).GetMemoryState(); }
                 private:
                     Result PushMap(KProcessAddress client, KProcessAddress server, size_t size, KMemoryState state, size_t index);
 
-                    constexpr const Mapping &GetSendMapping(size_t i) const {
+                    constexpr ALWAYS_INLINE const Mapping &GetSendMapping(size_t i) const {
                         MESOSPHERE_ASSERT(i < this->num_send);
 
                         const size_t index = i;
@@ -98,7 +98,7 @@ namespace ams::kern {
                         }
                     }
 
-                    constexpr const Mapping &GetReceiveMapping(size_t i) const {
+                    constexpr ALWAYS_INLINE const Mapping &GetReceiveMapping(size_t i) const {
                         MESOSPHERE_ASSERT(i < this->num_recv);
 
                         const size_t index = this->num_send + i;
@@ -109,7 +109,7 @@ namespace ams::kern {
                         }
                     }
 
-                    constexpr const Mapping &GetExchangeMapping(size_t i) const {
+                    constexpr ALWAYS_INLINE const Mapping &GetExchangeMapping(size_t i) const {
                         MESOSPHERE_ASSERT(i < this->num_exch);
 
                         const size_t index = this->num_send + this->num_recv + i;
@@ -176,50 +176,50 @@ namespace ams::kern {
 
             static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
 
-            constexpr KThread *GetThread() const { return this->thread; }
+            constexpr ALWAYS_INLINE KThread *GetThread() const { return this->thread; }
-            constexpr KWritableEvent *GetEvent() const { return this->event; }
+            constexpr ALWAYS_INLINE KWritableEvent *GetEvent() const { return this->event; }
-            constexpr uintptr_t GetAddress() const { return this->address; }
+            constexpr ALWAYS_INLINE uintptr_t GetAddress() const { return this->address; }
-            constexpr size_t GetSize() const { return this->size; }
+            constexpr ALWAYS_INLINE size_t GetSize() const { return this->size; }
-            constexpr KProcess *GetServerProcess() const { return this->server; }
+            constexpr ALWAYS_INLINE KProcess *GetServerProcess() const { return this->server; }
 
-            void SetServerProcess(KProcess *process) {
+            void ALWAYS_INLINE SetServerProcess(KProcess *process) {
                 this->server = process;
                 this->server->Open();
             }
 
-            constexpr void ClearThread() { this->thread = nullptr; }
+            constexpr ALWAYS_INLINE void ClearThread() { this->thread = nullptr; }
-            constexpr void ClearEvent()  { this->event  = nullptr; }
+            constexpr ALWAYS_INLINE void ClearEvent()  { this->event  = nullptr; }
 
-            constexpr size_t GetSendCount() const { return this->mappings.GetSendCount(); }
+            constexpr ALWAYS_INLINE size_t GetSendCount() const { return this->mappings.GetSendCount(); }
-            constexpr size_t GetReceiveCount() const { return this->mappings.GetReceiveCount(); }
+            constexpr ALWAYS_INLINE size_t GetReceiveCount() const { return this->mappings.GetReceiveCount(); }
-            constexpr size_t GetExchangeCount() const { return this->mappings.GetExchangeCount(); }
+            constexpr ALWAYS_INLINE size_t GetExchangeCount() const { return this->mappings.GetExchangeCount(); }
 
-            Result PushSend(KProcessAddress client, KProcessAddress server, size_t size, KMemoryState state) {
+            ALWAYS_INLINE Result PushSend(KProcessAddress client, KProcessAddress server, size_t size, KMemoryState state) {
                 return this->mappings.PushSend(client, server, size, state);
             }
 
-            Result PushReceive(KProcessAddress client, KProcessAddress server, size_t size, KMemoryState state) {
+            ALWAYS_INLINE Result PushReceive(KProcessAddress client, KProcessAddress server, size_t size, KMemoryState state) {
                 return this->mappings.PushReceive(client, server, size, state);
             }
 
-            Result PushExchange(KProcessAddress client, KProcessAddress server, size_t size, KMemoryState state) {
+            ALWAYS_INLINE Result PushExchange(KProcessAddress client, KProcessAddress server, size_t size, KMemoryState state) {
                 return this->mappings.PushExchange(client, server, size, state);
             }
 
-            constexpr KProcessAddress GetSendClientAddress(size_t i) const { return this->mappings.GetSendClientAddress(i); }
+            constexpr ALWAYS_INLINE KProcessAddress GetSendClientAddress(size_t i) const { return this->mappings.GetSendClientAddress(i); }
-            constexpr KProcessAddress GetSendServerAddress(size_t i) const { return this->mappings.GetSendServerAddress(i); }
+            constexpr ALWAYS_INLINE KProcessAddress GetSendServerAddress(size_t i) const { return this->mappings.GetSendServerAddress(i); }
-            constexpr size_t          GetSendSize(size_t i)          const { return this->mappings.GetSendSize(i);          }
+            constexpr ALWAYS_INLINE size_t          GetSendSize(size_t i)          const { return this->mappings.GetSendSize(i);          }
-            constexpr KMemoryState    GetSendMemoryState(size_t i)   const { return this->mappings.GetSendMemoryState(i);   }
+            constexpr ALWAYS_INLINE KMemoryState    GetSendMemoryState(size_t i)   const { return this->mappings.GetSendMemoryState(i);   }
 
-            constexpr KProcessAddress GetReceiveClientAddress(size_t i) const { return this->mappings.GetReceiveClientAddress(i); }
+            constexpr ALWAYS_INLINE KProcessAddress GetReceiveClientAddress(size_t i) const { return this->mappings.GetReceiveClientAddress(i); }
-            constexpr KProcessAddress GetReceiveServerAddress(size_t i) const { return this->mappings.GetReceiveServerAddress(i); }
+            constexpr ALWAYS_INLINE KProcessAddress GetReceiveServerAddress(size_t i) const { return this->mappings.GetReceiveServerAddress(i); }
-            constexpr size_t          GetReceiveSize(size_t i)          const { return this->mappings.GetReceiveSize(i);          }
+            constexpr ALWAYS_INLINE size_t          GetReceiveSize(size_t i)          const { return this->mappings.GetReceiveSize(i);          }
-            constexpr KMemoryState    GetReceiveMemoryState(size_t i)   const { return this->mappings.GetReceiveMemoryState(i);   }
+            constexpr ALWAYS_INLINE KMemoryState    GetReceiveMemoryState(size_t i)   const { return this->mappings.GetReceiveMemoryState(i);   }
 
-            constexpr KProcessAddress GetExchangeClientAddress(size_t i) const { return this->mappings.GetExchangeClientAddress(i);  }
+            constexpr ALWAYS_INLINE KProcessAddress GetExchangeClientAddress(size_t i) const { return this->mappings.GetExchangeClientAddress(i);  }
-            constexpr KProcessAddress GetExchangeServerAddress(size_t i) const { return this->mappings.GetExchangeServerAddress(i);  }
+            constexpr ALWAYS_INLINE KProcessAddress GetExchangeServerAddress(size_t i) const { return this->mappings.GetExchangeServerAddress(i);  }
-            constexpr size_t          GetExchangeSize(size_t i)          const { return this->mappings.GetExchangeSize(i);           }
+            constexpr ALWAYS_INLINE size_t          GetExchangeSize(size_t i)          const { return this->mappings.GetExchangeSize(i);           }
-            constexpr KMemoryState    GetExchangeMemoryState(size_t i)   const { return this->mappings.GetExchangeMemoryState(i);    }
+            constexpr ALWAYS_INLINE KMemoryState    GetExchangeMemoryState(size_t i)   const { return this->mappings.GetExchangeMemoryState(i);    }
     };
 
 }

libraries/libmesosphere/include/mesosphere/kern_k_synchronization.hpp

@@ -1,54 +0,0 @@
-/*
- * Copyright (c) 2018-2020 Atmosphère-NX
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-#pragma once
-#include <mesosphere/kern_k_auto_object.hpp>
-#include <mesosphere/kern_slab_helpers.hpp>
-#include <mesosphere/kern_k_linked_list.hpp>
-/*
- * Copyright (c) 2018-2020 Atmosphère-NX
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-#pragma once
-#include <mesosphere/kern_k_auto_object.hpp>
-#include <mesosphere/kern_k_synchronization_object.hpp>
-#include <mesosphere/kern_k_thread.hpp>
-
-namespace ams::kern {
-
-    class KSynchronization {
-        private:
-            friend class KSynchronizationObject;
-        public:
-            constexpr KSynchronization() { /* ... */ }
-
-            Result Wait(s32 *out_index, KSynchronizationObject **objects, const s32 num_objects, s64 timeout);
-        private:
-            void OnAvailable(KSynchronizationObject *object);
-            void OnAbort(KSynchronizationObject *object, Result abort_reason);
-    };
-
-}

libraries/libmesosphere/include/mesosphere/kern_k_synchronization_object.hpp

@@ -25,28 +25,28 @@ namespace ams::kern {
     class KSynchronizationObject : public KAutoObjectWithList {
         MESOSPHERE_AUTOOBJECT_TRAITS(KSynchronizationObject, KAutoObject);
         public:
-            using ThreadList = KLinkedList<KThread>;
+            struct ThreadListNode {
-            using iterator = ThreadList::iterator;
+                ThreadListNode *next;
+                KThread *thread;
+            };
         private:
-            ThreadList thread_list;
+            ThreadListNode *thread_list_root;
         protected:
-            constexpr ALWAYS_INLINE explicit KSynchronizationObject() : KAutoObjectWithList(), thread_list() { MESOSPHERE_ASSERT_THIS(); }
+            constexpr ALWAYS_INLINE explicit KSynchronizationObject() : KAutoObjectWithList(), thread_list_root() { MESOSPHERE_ASSERT_THIS(); }
             virtual ~KSynchronizationObject() { MESOSPHERE_ASSERT_THIS(); }
 
             virtual void OnFinalizeSynchronizationObject() { MESOSPHERE_ASSERT_THIS(); }
 
-            void NotifyAvailable();
+            void NotifyAvailable(Result result);
-            void NotifyAbort(Result abort_reason);
+            void NotifyAvailable() {
+                return this->NotifyAvailable(ResultSuccess());
+            }
+        public:
+            static Result Wait(s32 *out_index, KSynchronizationObject **objects, const s32 num_objects, s64 timeout);
         public:
             virtual void Finalize() override;
             virtual bool IsSignaled() const = 0;
             virtual void DebugWaiters();
-
-            iterator RegisterWaitingThread(KThread *thread);
-            iterator UnregisterWaitingThread(iterator it);
-
-            iterator begin();
-            iterator end();
     };
 
 }

libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp

@@ -31,7 +31,7 @@ namespace ams::kern {
 
     using KThreadFunction = void (*)(uintptr_t);
 
-    class KThread final : public KAutoObjectWithSlabHeapAndContainer<KThread, KSynchronizationObject>, public KTimerTask, public KWorkerTask {
+    class KThread final : public KAutoObjectWithSlabHeapAndContainer<KThread, KSynchronizationObject>, public util::IntrusiveListBaseNode<KThread>, public KTimerTask, public KWorkerTask {
         MESOSPHERE_AUTOOBJECT_TRAITS(KThread, KSynchronizationObject);
         private:
             friend class KProcess;
@@ -90,6 +90,7 @@ namespace ams::kern {
                 bool is_pinned;
                 s32 disable_count;
                 KThreadContext *context;
+                KThread *cur_thread;
             };
             static_assert(alignof(StackParameters) == 0x10);
 
@@ -110,6 +111,8 @@ namespace ams::kern {
                     constexpr void SetPrev(KThread *t) { this->prev = t; }
                     constexpr void SetNext(KThread *t) { this->next = t; }
             };
+
+            using WaiterList = util::IntrusiveListBaseTraits<KThread>::ListType;
         private:
             static constexpr size_t PriorityInheritanceCountMax = 10;
             union SyncObjectBuffer {
@@ -121,7 +124,21 @@ namespace ams::kern {
             static_assert(sizeof(SyncObjectBuffer::sync_objects) == sizeof(SyncObjectBuffer::handles));
 
             struct ConditionVariableComparator {
-                static constexpr ALWAYS_INLINE int Compare(const KThread &lhs, const KThread &rhs) {
+                struct LightCompareType {
+                    uintptr_t cv_key;
+                    s32 priority;
+
+                    constexpr ALWAYS_INLINE uintptr_t GetConditionVariableKey() const {
+                        return this->cv_key;
+                    }
+
+                    constexpr ALWAYS_INLINE s32 GetPriority() const {
+                        return this->priority;
+                    }
+                };
+
+                template<typename T> requires (std::same_as<T, KThread> || std::same_as<T, LightCompareType>)
+                static constexpr ALWAYS_INLINE int Compare(const T &lhs, const KThread &rhs) {
                     const uintptr_t l_key = lhs.GetConditionVariableKey();
                     const uintptr_t r_key = rhs.GetConditionVariableKey();
 
@@ -136,17 +153,26 @@ namespace ams::kern {
                     }
                 }
             };
+            static_assert(ams::util::HasLightCompareType<ConditionVariableComparator>);
+            static_assert(std::same_as<ams::util::LightCompareType<ConditionVariableComparator, void>, ConditionVariableComparator::LightCompareType>);
         private:
             static inline std::atomic<u64> s_next_thread_id = 0;
         private:
             alignas(16) KThreadContext      thread_context{};
-            KAffinityMask                   affinity_mask{};
+            util::IntrusiveListNode         process_list_node{};
+            util::IntrusiveRedBlackTreeNode condvar_arbiter_tree_node{};
+            s32                             priority{};
+
+            using ConditionVariableThreadTreeTraits = util::IntrusiveRedBlackTreeMemberTraitsDeferredAssert<&KThread::condvar_arbiter_tree_node>;
+            using ConditionVariableThreadTree       = ConditionVariableThreadTreeTraits::TreeType<ConditionVariableComparator>;
+
+            ConditionVariableThreadTree    *condvar_tree{};
+            uintptr_t                       condvar_key{};
+            u64                             virtual_affinity_mask{};
+            KAffinityMask                   physical_affinity_mask{};
             u64                             thread_id{};
             std::atomic<s64>                cpu_time{};
             KSynchronizationObject         *synced_object{};
-            KLightLock                     *waiting_lock{};
-            uintptr_t                       condvar_key{};
-            uintptr_t                       entrypoint{};
             KProcessAddress                 address_key{};
             KProcess                       *parent{};
             void                           *kernel_stack_top{};
@@ -158,42 +184,32 @@ namespace ams::kern {
             s64                             schedule_count{};
             s64                             last_scheduled_tick{};
             QueueEntry                      per_core_priority_queue_entry[cpu::NumCores]{};
-            QueueEntry                      sleeping_queue_entry{};
+            KLightLock                     *waiting_lock{};
+
             KThreadQueue                   *sleeping_queue{};
-            util::IntrusiveListNode         waiter_list_node{};
-            util::IntrusiveRedBlackTreeNode condvar_arbiter_tree_node{};
-            util::IntrusiveListNode         process_list_node{};
-
-            using WaiterListTraits = util::IntrusiveListMemberTraitsDeferredAssert<&KThread::waiter_list_node>;
-            using WaiterList       = WaiterListTraits::ListType;
-
-            using ConditionVariableThreadTreeTraits = util::IntrusiveRedBlackTreeMemberTraitsDeferredAssert<&KThread::condvar_arbiter_tree_node>;
-            using ConditionVariableThreadTree       = ConditionVariableThreadTreeTraits::TreeType<ConditionVariableComparator>;
 
             WaiterList                      waiter_list{};
             WaiterList                      pinned_waiter_list{};
             KThread                        *lock_owner{};
-            ConditionVariableThreadTree    *condvar_tree{};
             uintptr_t                       debug_params[3]{};
             u32                             address_key_value{};
             u32                             suspend_request_flags{};
             u32                             suspend_allowed_flags{};
             Result                          wait_result;
             Result                          debug_exception_result;
-            s32                             priority{};
-            s32                             core_id{};
             s32                             base_priority{};
-            s32                             ideal_core_id{};
+            s32                             physical_ideal_core_id{};
+            s32                             virtual_ideal_core_id{};
             s32                             num_kernel_waiters{};
-            KAffinityMask                   original_affinity_mask{};
+            s32                             current_core_id{};
-            s32                             original_ideal_core_id{};
+            s32                             core_id{};
+            KAffinityMask                   original_physical_affinity_mask{};
+            s32                             original_physical_ideal_core_id{};
             s32                             num_core_migration_disables{};
             ThreadState                     thread_state{};
             std::atomic<bool>               termination_requested{};
-            bool                            ipc_cancelled{};
             bool                            wait_cancelled{};
             bool                            cancellable{};
-            bool                            registered{};
             bool                            signaled{};
             bool                            initialized{};
             bool                            debug_attached{};
@@ -204,21 +220,21 @@ namespace ams::kern {
 
             virtual ~KThread() { /* ... */ }
 
-            Result Initialize(KThreadFunction func, uintptr_t arg, void *kern_stack_top, KProcessAddress user_stack_top, s32 prio, s32 core, KProcess *owner, ThreadType type);
+            Result Initialize(KThreadFunction func, uintptr_t arg, void *kern_stack_top, KProcessAddress user_stack_top, s32 prio, s32 virt_core, KProcess *owner, ThreadType type);
 
         private:
-            static Result InitializeThread(KThread *thread, KThreadFunction func, uintptr_t arg, KProcessAddress user_stack_top, s32 prio, s32 core, KProcess *owner, ThreadType type);
+            static Result InitializeThread(KThread *thread, KThreadFunction func, uintptr_t arg, KProcessAddress user_stack_top, s32 prio, s32 virt_core, KProcess *owner, ThreadType type);
         public:
-            static Result InitializeKernelThread(KThread *thread, KThreadFunction func, uintptr_t arg, s32 prio, s32 core) {
+            static Result InitializeKernelThread(KThread *thread, KThreadFunction func, uintptr_t arg, s32 prio, s32 virt_core) {
-                return InitializeThread(thread, func, arg, Null<KProcessAddress>, prio, core, nullptr, ThreadType_Kernel);
+                return InitializeThread(thread, func, arg, Null<KProcessAddress>, prio, virt_core, nullptr, ThreadType_Kernel);
             }
 
             static Result InitializeHighPriorityThread(KThread *thread, KThreadFunction func, uintptr_t arg) {
                 return InitializeThread(thread, func, arg, Null<KProcessAddress>, 0, GetCurrentCoreId(), nullptr, ThreadType_HighPriority);
             }
 
-            static Result InitializeUserThread(KThread *thread, KThreadFunction func, uintptr_t arg, KProcessAddress user_stack_top, s32 prio, s32 core, KProcess *owner) {
+            static Result InitializeUserThread(KThread *thread, KThreadFunction func, uintptr_t arg, KProcessAddress user_stack_top, s32 prio, s32 virt_core, KProcess *owner) {
-                return InitializeThread(thread, func, arg, user_stack_top, prio, core, owner, ThreadType_User);
+                return InitializeThread(thread, func, arg, user_stack_top, prio, virt_core, owner, ThreadType_User);
             }
 
             static void ResumeThreadsSuspendedForInit();
@@ -325,10 +341,14 @@ namespace ams::kern {
             constexpr KThreadContext &GetContext() { return this->thread_context; }
             constexpr const KThreadContext &GetContext() const { return this->thread_context; }
 
-            constexpr const KAffinityMask &GetAffinityMask() const { return this->affinity_mask; }
+            constexpr u64 GetVirtualAffinityMask() const { return this->virtual_affinity_mask; }
+            constexpr const KAffinityMask &GetAffinityMask() const { return this->physical_affinity_mask; }
+
             Result GetCoreMask(int32_t *out_ideal_core, u64 *out_affinity_mask);
             Result SetCoreMask(int32_t ideal_core, u64 affinity_mask);
 
+            Result GetPhysicalCoreMask(int32_t *out_ideal_core, u64 *out_affinity_mask);
+
             constexpr ThreadState GetState() const { return static_cast<ThreadState>(this->thread_state & ThreadState_Mask); }
             constexpr ThreadState GetRawState() const { return this->thread_state; }
             NOINLINE void SetState(ThreadState state);
@@ -338,11 +358,6 @@ namespace ams::kern {
             constexpr uintptr_t GetConditionVariableKey() const { return this->condvar_key; }
             constexpr uintptr_t GetAddressArbiterKey() const { return this->condvar_key; }
 
-            constexpr void SetupForConditionVariableCompare(uintptr_t cv_key, int priority) {
-                this->condvar_key = cv_key;
-                this->priority    = priority;
-            }
-
             constexpr void SetConditionVariable(ConditionVariableThreadTree *tree, KProcessAddress address, uintptr_t cv_key, u32 value) {
                 this->condvar_tree      = tree;
                 this->condvar_key       = cv_key;
@@ -358,11 +373,6 @@ namespace ams::kern {
                 return this->condvar_tree != nullptr;
             }
 
-            constexpr void SetupForAddressArbiterCompare(uintptr_t address, int priority) {
-                this->condvar_key = address;
-                this->priority    = priority;
-            }
-
             constexpr void SetAddressArbiter(ConditionVariableThreadTree *tree, uintptr_t address) {
                 this->condvar_tree = tree;
                 this->condvar_key  = address;
@@ -376,10 +386,15 @@ namespace ams::kern {
                 return this->condvar_tree != nullptr;
             }
 
-            constexpr s32 GetIdealCore() const { return this->ideal_core_id; }
+            constexpr s32 GetIdealVirtualCore() const { return this->virtual_ideal_core_id; }
+            constexpr s32 GetIdealPhysicalCore() const { return this->physical_ideal_core_id; }
+
             constexpr s32 GetActiveCore() const { return this->core_id; }
             constexpr void SetActiveCore(s32 core) { this->core_id = core; }
 
+            constexpr ALWAYS_INLINE s32 GetCurrentCore() const { return this->current_core_id; }
+            constexpr void SetCurrentCore(s32 core) { this->current_core_id = core; }
+
             constexpr s32 GetPriority() const { return this->priority; }
             constexpr void SetPriority(s32 prio) { this->priority = prio; }
 
@@ -388,8 +403,6 @@ namespace ams::kern {
             constexpr QueueEntry &GetPriorityQueueEntry(s32 core) { return this->per_core_priority_queue_entry[core]; }
             constexpr const QueueEntry &GetPriorityQueueEntry(s32 core) const { return this->per_core_priority_queue_entry[core]; }
 
-            constexpr QueueEntry &GetSleepingQueueEntry() { return this->sleeping_queue_entry; }
-            constexpr const QueueEntry &GetSleepingQueueEntry() const { return this->sleeping_queue_entry; }
             constexpr void SetSleepingQueue(KThreadQueue *q) { this->sleeping_queue = q; }
 
             constexpr ConditionVariableThreadTree *GetConditionVariableTree() const { return this->condvar_tree; }
@@ -454,8 +467,6 @@ namespace ams::kern {
             constexpr KProcess *GetOwnerProcess() const { return this->parent; }
             constexpr bool IsUserThread() const { return this->parent != nullptr; }
 
-            constexpr uintptr_t GetEntrypoint() const { return this->entrypoint; }
-
             constexpr KProcessAddress GetThreadLocalRegionAddress() const { return this->tls_address; }
             constexpr void           *GetThreadLocalRegionHeapAddress() const { return this->tls_heap_address; }
 
@@ -536,10 +547,6 @@ namespace ams::kern {
             virtual void OnTimer() override;
             virtual void DoWorkerTask() override;
         public:
-            static constexpr bool IsWaiterListValid() {
-                return WaiterListTraits::IsValid();
-            }
-
             static constexpr bool IsConditionVariableThreadTreeValid() {
                 return ConditionVariableThreadTreeTraits::IsValid();
             }
@@ -550,7 +557,6 @@ namespace ams::kern {
             using ConditionVariableThreadTreeType = ConditionVariableThreadTree;
     };
     static_assert(alignof(KThread) == 0x10);
-    static_assert(KThread::IsWaiterListValid());
     static_assert(KThread::IsConditionVariableThreadTreeValid());
 
     class KScopedDisableDispatch {
@@ -570,4 +576,16 @@ namespace ams::kern {
         return reinterpret_cast<const KExceptionContext *>(reinterpret_cast<uintptr_t>(thread->GetKernelStackTop()) - sizeof(KThread::StackParameters) - sizeof(KExceptionContext));
     }
 
+    ALWAYS_INLINE KProcess *GetCurrentProcessPointer() {
+        return GetCurrentThread().GetOwnerProcess();
+    }
+
+    ALWAYS_INLINE KProcess &GetCurrentProcess() {
+        return *GetCurrentProcessPointer();
+    }
+
+    ALWAYS_INLINE s32 GetCurrentCoreId() {
+        return GetCurrentThread().GetCurrentCore();
+    }
+
 }

libraries/libmesosphere/include/mesosphere/kern_k_thread_queue.hpp

@@ -21,92 +21,38 @@ namespace ams::kern {
 
     class KThreadQueue {
         private:
-            using Entry = KThread::QueueEntry;
+            KThread::WaiterList wait_list;
-        private:
-            Entry root;
         public:
-            constexpr ALWAYS_INLINE KThreadQueue() : root() { /* ... */ }
+            constexpr ALWAYS_INLINE KThreadQueue() : wait_list() { /* ... */ }
 
-            constexpr ALWAYS_INLINE bool IsEmpty() const { return this->root.GetNext() == nullptr; }
+            bool IsEmpty() const { return this->wait_list.empty(); }
 
-            constexpr ALWAYS_INLINE KThread *GetFront() const { return this->root.GetNext(); }
+            KThread::WaiterList::iterator begin() { return this->wait_list.begin(); }
-            constexpr ALWAYS_INLINE KThread *GetNext(KThread *t) const { return t->GetSleepingQueueEntry().GetNext(); }
+            KThread::WaiterList::iterator end() { return this->wait_list.end(); }
-        private:
-            constexpr ALWAYS_INLINE KThread *GetBack() const { return this->root.GetPrev(); }
-
-            constexpr ALWAYS_INLINE void Enqueue(KThread *add) {
-                /* Get the entry associated with the added thread. */
-                Entry &add_entry = add->GetSleepingQueueEntry();
-
-                /* Get the entry associated with the end of the queue. */
-                KThread *tail       = this->GetBack();
-                Entry   &tail_entry = (tail != nullptr) ? tail->GetSleepingQueueEntry() : this->root;
-
-                /* Link the entries. */
-                add_entry.SetPrev(tail);
-                add_entry.SetNext(nullptr);
-                tail_entry.SetNext(add);
-                this->root.SetPrev(add);
-            }
-
-            constexpr ALWAYS_INLINE void Remove(KThread *remove) {
-                /* Get the entry associated with the thread. */
-                Entry &remove_entry = remove->GetSleepingQueueEntry();
-
-                /* Get the entries associated with next and prev. */
-                KThread *prev = remove_entry.GetPrev();
-                KThread *next = remove_entry.GetNext();
-                Entry   &prev_entry = (prev != nullptr) ? prev->GetSleepingQueueEntry() : this->root;
-                Entry   &next_entry = (next != nullptr) ? next->GetSleepingQueueEntry() : this->root;
-
-                /* Unlink. */
-                prev_entry.SetNext(next);
-                next_entry.SetPrev(prev);
-            }
-        public:
-            constexpr ALWAYS_INLINE void Dequeue() {
-                /* Get the front of the queue. */
-                KThread *head = this->GetFront();
-                if (head == nullptr) {
-                    return;
-                }
-
-                MESOSPHERE_ASSERT(head->GetState() == KThread::ThreadState_Waiting);
-
-                /* Get the entry for the next head. */
-                KThread *next = GetNext(head);
-                Entry   &next_entry = (next != nullptr) ? next->GetSleepingQueueEntry() : this->root;
-
-                /* Link the entries. */
-                this->root.SetNext(next);
-                next_entry.SetPrev(nullptr);
-
-                /* Clear the head's queue. */
-                head->SetSleepingQueue(nullptr);
-            }
 
             bool SleepThread(KThread *t) {
+                KScopedSchedulerLock sl;
+
+                /* If the thread needs terminating, don't enqueue it. */
+                if (t->IsTerminationRequested()) {
+                    return false;
+                }
+
                 /* Set the thread's queue and mark it as waiting. */
                 t->SetSleepingQueue(this);
                 t->SetState(KThread::ThreadState_Waiting);
 
                 /* Add the thread to the queue. */
-                this->Enqueue(t);
+                this->wait_list.push_back(*t);
-
-                /* If the thread needs terminating, undo our work. */
-                if (t->IsTerminationRequested()) {
-                    this->WakeupThread(t);
-                    return false;
-                }
 
                 return true;
             }
 
             void WakeupThread(KThread *t) {
-                MESOSPHERE_ASSERT(t->GetState() == KThread::ThreadState_Waiting);
+                KScopedSchedulerLock sl;
 
                 /* Remove the thread from the queue. */
-                this->Remove(t);
+                this->wait_list.erase(this->wait_list.iterator_to(*t));
 
                 /* Mark the thread as no longer sleeping. */
                 t->SetState(KThread::ThreadState_Runnable);
@@ -114,18 +60,24 @@ namespace ams::kern {
             }
 
             KThread *WakeupFrontThread() {
-                KThread *front = this->GetFront();
+                KScopedSchedulerLock sl;
-                if (front != nullptr) {
-                    MESOSPHERE_ASSERT(front->GetState() == KThread::ThreadState_Waiting);
 
+                if (this->wait_list.empty()) {
+                    return nullptr;
+                } else {
                     /* Remove the thread from the queue. */
-                    this->Dequeue();
+                    auto it = this->wait_list.begin();
+                    KThread *thread = std::addressof(*it);
+                    this->wait_list.erase(it);
+
+                    MESOSPHERE_ASSERT(thread->GetState() == KThread::ThreadState_Waiting);
 
                     /* Mark the thread as no longer sleeping. */
-                    front->SetState(KThread::ThreadState_Runnable);
+                    thread->SetState(KThread::ThreadState_Runnable);
-                    front->SetSleepingQueue(nullptr);
+                    thread->SetSleepingQueue(nullptr);
+
+                    return thread;
                 }
-                return front;
             }
     };
 

libraries/libmesosphere/include/mesosphere/kern_k_wait_object.hpp

@@ -22,45 +22,13 @@ namespace ams::kern {
 
     class KWaitObject : public KTimerTask {
         private:
-            using Entry = KThread::QueueEntry;
+            KThread::WaiterList wait_list;
-        private:
-            Entry root;
             bool  timer_used;
         public:
-            constexpr KWaitObject() : root(), timer_used() { /* ... */ }
+            constexpr KWaitObject() : wait_list(), timer_used() { /* ... */ }
 
             virtual void OnTimer() override;
             Result Synchronize(s64 timeout);
-        private:
-            constexpr ALWAYS_INLINE void Enqueue(KThread *add) {
-                /* Get the entry associated with the added thread. */
-                Entry &add_entry = add->GetSleepingQueueEntry();
-
-                /* Get the entry associated with the end of the queue. */
-                KThread *tail       = this->root.GetPrev();
-                Entry   &tail_entry = (tail != nullptr) ? tail->GetSleepingQueueEntry() : this->root;
-
-                /* Link the entries. */
-                add_entry.SetPrev(tail);
-                add_entry.SetNext(nullptr);
-                tail_entry.SetNext(add);
-                this->root.SetPrev(add);
-            }
-
-            constexpr ALWAYS_INLINE void Remove(KThread *remove) {
-                /* Get the entry associated with the thread. */
-                Entry &remove_entry = remove->GetSleepingQueueEntry();
-
-                /* Get the entries associated with next and prev. */
-                KThread *prev = remove_entry.GetPrev();
-                KThread *next = remove_entry.GetNext();
-                Entry   &prev_entry = (prev != nullptr) ? prev->GetSleepingQueueEntry() : this->root;
-                Entry   &next_entry = (next != nullptr) ? next->GetSleepingQueueEntry() : this->root;
-
-                /* Unlink. */
-                prev_entry.SetNext(next);
-                next_entry.SetPrev(prev);
-            }
     };
 
 }

libraries/libmesosphere/include/mesosphere/kern_kernel.hpp

@@ -19,7 +19,10 @@
 #include <mesosphere/kern_select_cpu.hpp>
 #include <mesosphere/kern_k_memory_layout.hpp>
 #include <mesosphere/kern_k_memory_manager.hpp>
-#include <mesosphere/kern_k_core_local_region.hpp>
+#include <mesosphere/kern_k_scheduler.hpp>
+#include <mesosphere/kern_k_interrupt_task_manager.hpp>
+#include <mesosphere/kern_select_interrupt_manager.hpp>
+#include <mesosphere/kern_select_hardware_timer.hpp>
 #include <mesosphere/kern_k_worker_task_manager.hpp>
 
 namespace ams::kern {
@@ -34,11 +37,8 @@ namespace ams::kern {
     class KPageTableManager;
     class KMemoryBlockSlabManager;
     class KBlockInfoManager;
-    class KSynchronization;
     class KUnsafeMemory;
 
-
-
 #if defined(ATMOSPHERE_ARCH_ARM64)
 
     namespace arch::arm64 {
@@ -72,16 +72,12 @@ namespace ams::kern {
             static KMemoryBlockSlabManager s_sys_memory_block_manager;
             static KBlockInfoManager s_block_info_manager;
             static KSupervisorPageTable s_supervisor_page_table;
-            static KSynchronization s_synchronization;
             static KUnsafeMemory s_unsafe_memory;
             static KWorkerTaskManager s_worker_task_managers[KWorkerTaskManager::WorkerType_Count];
-        private:
+            static KInterruptManager s_interrupt_manager;
-            static ALWAYS_INLINE KCoreLocalContext &GetCoreLocalContext() {
+            static KScheduler s_schedulers[cpu::NumCores];
-                return reinterpret_cast<KCoreLocalRegion *>(cpu::GetCoreLocalRegionAddress())->current.context;
+            static KInterruptTaskManager s_interrupt_task_managers[cpu::NumCores];
-            }
+            static KHardwareTimer s_hardware_timers[cpu::NumCores];
-            static ALWAYS_INLINE KCoreLocalContext &GetCoreLocalContext(s32 core_id) {
-                return reinterpret_cast<KCoreLocalRegion *>(cpu::GetCoreLocalRegionAddress())->absolute[core_id].context;
-            }
         public:
             static NOINLINE void InitializeCoreLocalRegion(s32 core_id);
             static NOINLINE void InitializeMainAndIdleThreads(s32 core_id);
@@ -94,28 +90,28 @@ namespace ams::kern {
             static KThread &GetMainThread(s32 core_id);
             static KThread &GetIdleThread(s32 core_id);
 
-            static ALWAYS_INLINE KCurrentContext &GetCurrentContext(s32 core_id) {
-                return GetCoreLocalContext(core_id).current;
-            }
-
             static ALWAYS_INLINE KScheduler &GetScheduler() {
-                return GetCoreLocalContext().scheduler;
+                return s_schedulers[GetCurrentCoreId()];
             }
 
             static ALWAYS_INLINE KScheduler &GetScheduler(s32 core_id) {
-                return GetCoreLocalContext(core_id).scheduler;
+                return s_schedulers[core_id];
             }
 
             static ALWAYS_INLINE KInterruptTaskManager &GetInterruptTaskManager() {
-                return GetCoreLocalContext().interrupt_task_manager;
+                return s_interrupt_task_managers[GetCurrentCoreId()];
             }
 
             static ALWAYS_INLINE KInterruptManager &GetInterruptManager() {
-                return GetCoreLocalContext().interrupt_manager;
+                return s_interrupt_manager;
             }
 
             static ALWAYS_INLINE KHardwareTimer &GetHardwareTimer() {
-                return GetCoreLocalContext(GetCurrentCoreId()).hardware_timer;
+                return s_hardware_timers[GetCurrentCoreId()];
+            }
+
+            static ALWAYS_INLINE KHardwareTimer &GetHardwareTimer(s32 core_id) {
+                return s_hardware_timers[core_id];
             }
 
             static ALWAYS_INLINE KResourceLimit &GetSystemResourceLimit() {
@@ -146,10 +142,6 @@ namespace ams::kern {
                 return s_supervisor_page_table;
             }
 
-            static ALWAYS_INLINE KSynchronization &GetSynchronization() {
-                return s_synchronization;
-            }
-
             static ALWAYS_INLINE KUnsafeMemory &GetUnsafeMemory() {
                 return s_unsafe_memory;
             }
@@ -160,4 +152,8 @@ namespace ams::kern {
             }
     };
 
+    ALWAYS_INLINE KScheduler &GetCurrentScheduler() {
+        return Kernel::GetScheduler();
+    }
+
 }

libraries/libmesosphere/include/mesosphere/kern_select_cpu.hpp

@@ -28,3 +28,24 @@
 #else
     #error "Unknown architecture for CPU"
 #endif
+
+#ifdef ATMOSPHERE_BOARD_NINTENDO_NX
+
+    #include <mesosphere/board/nintendo/nx/kern_cpu_map.hpp>
+
+    namespace ams::kern::cpu {
+
+        using namespace ams::kern::board::nintendo::nx::impl::cpu;
+
+    }
+
+#else
+    #error "Unknown board for CPU Map"
+#endif
+
+namespace ams::kern {
+
+    static_assert(cpu::NumCores <= static_cast<s32>(BITSIZEOF(u64)));
+    static_assert(util::size(cpu::VirtualToPhysicalCoreMap) == BITSIZEOF(u64));
+
+}

libraries/libmesosphere/include/mesosphere/kern_select_interrupt_manager.hpp

@@ -42,7 +42,7 @@ namespace ams::kern {
             u32 prev_intr_state;
         public:
             ALWAYS_INLINE KScopedInterruptDisable() : prev_intr_state(KInterruptManager::DisableInterrupts()) { /* ... */ }
-            ~KScopedInterruptDisable() { KInterruptManager::RestoreInterrupts(prev_intr_state); }
+            ALWAYS_INLINE ~KScopedInterruptDisable() { KInterruptManager::RestoreInterrupts(prev_intr_state); }
     };
 
     class KScopedInterruptEnable {
@@ -52,7 +52,7 @@ namespace ams::kern {
             u32 prev_intr_state;
         public:
             ALWAYS_INLINE KScopedInterruptEnable() : prev_intr_state(KInterruptManager::EnableInterrupts()) { /* ... */ }
-            ~KScopedInterruptEnable() { KInterruptManager::RestoreInterrupts(prev_intr_state); }
+            ALWAYS_INLINE ~KScopedInterruptEnable() { KInterruptManager::RestoreInterrupts(prev_intr_state); }
     };
 
 }

libraries/libmesosphere/include/mesosphere/kern_slab_helpers.hpp

@@ -108,7 +108,7 @@ namespace ams::kern {
                 return obj;
             }
 
-            static Result Register(Derived *obj) {
+            static void Register(Derived *obj) {
                 return s_container.Register(obj);
             }
 

libraries/libmesosphere/source/arch/arm/kern_generic_interrupt_controller.inc

@@ -79,9 +79,6 @@ namespace ams::kern::arch::arm {
 
         /* Setup all interrupt lines. */
         SetupInterruptLines(core_id);
-
-        this->gicd = nullptr;
-        this->gicc = nullptr;
     }
 
     void KInterruptController::SaveCoreLocal(LocalState *state) const {

libraries/libmesosphere/source/arch/arm64/kern_k_hardware_timer.cpp

@@ -17,41 +17,15 @@
 
 namespace ams::kern::arch::arm64 {
 
-    namespace impl {
+    void KHardwareTimer::Initialize() {
-
-        class KHardwareTimerInterruptTask : public KInterruptTask {
-            public:
-                constexpr KHardwareTimerInterruptTask() : KInterruptTask() { /* ... */ }
-
-                virtual KInterruptTask *OnInterrupt(s32 interrupt_id) override {
-                    MESOSPHERE_UNUSED(interrupt_id);
-                    return this;
-                }
-
-                virtual void DoTask() override {
-                    Kernel::GetHardwareTimer().DoInterruptTask();
-                }
-        };
-
-    }
-
-    namespace {
-
-        /* One global hardware timer interrupt task per core. */
-        impl::KHardwareTimerInterruptTask g_hardware_timer_interrupt_tasks[cpu::NumCores];
-
-        ALWAYS_INLINE auto *GetHardwareTimerInterruptTask(s32 core_id) {
-            return std::addressof(g_hardware_timer_interrupt_tasks[core_id]);
-        }
-
-    }
-
-    void KHardwareTimer::Initialize(s32 core_id) {
         /* Setup the global timer for the core. */
         InitializeGlobalTimer();
 
+        /* Set maximum time. */
+        this->maximum_time = static_cast<s64>(std::min<u64>(std::numeric_limits<s64>::max(), cpu::CounterTimerPhysicalTimerCompareValueRegisterAccessor().GetCompareValue()));
+
         /* Bind the interrupt task for this core. */
-        Kernel::GetInterruptManager().BindHandler(GetHardwareTimerInterruptTask(core_id), KInterruptName_NonSecurePhysicalTimer, core_id, KInterruptController::PriorityLevel_Timer, true, true);
+        Kernel::GetInterruptManager().BindHandler(this, KInterruptName_NonSecurePhysicalTimer, GetCurrentCoreId(), KInterruptController::PriorityLevel_Timer, true, true);
     }
 
     void KHardwareTimer::Finalize() {
@@ -59,7 +33,7 @@ namespace ams::kern::arch::arm64 {
         StopTimer();
     }
 
-    void KHardwareTimer::DoInterruptTask() {
+    void KHardwareTimer::DoTask() {
         /* Handle the interrupt. */
         {
             KScopedSchedulerLock slk;
@@ -67,7 +41,7 @@ namespace ams::kern::arch::arm64 {
 
             /* Disable the timer interrupt while we handle this. */
             DisableInterrupt();
-            if (const s64 next_time = this->DoInterruptTaskImpl(GetTick()); next_time > 0) {
+            if (const s64 next_time = this->DoInterruptTaskImpl(GetTick()); 0 < next_time && next_time <= this->maximum_time) {
                 /* We have a next time, so we should set the time to interrupt and turn the interrupt on. */
                 SetCompareValue(next_time);
                 EnableInterrupt();

libraries/libmesosphere/source/arch/arm64/kern_k_interrupt_manager.cpp

@@ -17,12 +17,6 @@
 
 namespace ams::kern::arch::arm64 {
 
-    /* Instantiate static members in specific translation unit. */
-    KSpinLock KInterruptManager::s_lock;
-    std::array<KInterruptManager::KGlobalInterruptEntry, KInterruptController::NumGlobalInterrupts> KInterruptManager::s_global_interrupts;
-    KInterruptController::GlobalState KInterruptManager::s_global_state;
-    bool KInterruptManager::s_global_state_saved;
-
     void KInterruptManager::Initialize(s32 core_id) {
         this->interrupt_controller.Initialize(core_id);
     }
@@ -32,23 +26,26 @@ namespace ams::kern::arch::arm64 {
     }
 
     void KInterruptManager::Save(s32 core_id) {
+        /* Verify core id. */
+        MESOSPHERE_ASSERT(core_id == GetCurrentCoreId());
+
         /* Ensure all cores get to this point before continuing. */
         cpu::SynchronizeAllCores();
 
         /* If on core 0, save the global interrupts. */
         if (core_id == 0) {
-            MESOSPHERE_ABORT_UNLESS(!s_global_state_saved);
+            MESOSPHERE_ABORT_UNLESS(!this->global_state_saved);
-            this->interrupt_controller.SaveGlobal(std::addressof(s_global_state));
+            this->interrupt_controller.SaveGlobal(std::addressof(this->global_state));
-            s_global_state_saved = true;
+            this->global_state_saved = true;
         }
 
         /* Ensure all cores get to this point before continuing. */
         cpu::SynchronizeAllCores();
 
         /* Save all local interrupts. */
-        MESOSPHERE_ABORT_UNLESS(!this->local_state_saved);
+        MESOSPHERE_ABORT_UNLESS(!this->local_state_saved[core_id]);
-        this->interrupt_controller.SaveCoreLocal(std::addressof(this->local_state));
+        this->interrupt_controller.SaveCoreLocal(std::addressof(this->local_states[core_id]));
-        this->local_state_saved = true;
+        this->local_state_saved[core_id] = true;
 
         /* Ensure all cores get to this point before continuing. */
         cpu::SynchronizeAllCores();
@@ -68,6 +65,9 @@ namespace ams::kern::arch::arm64 {
     }
 
     void KInterruptManager::Restore(s32 core_id) {
+        /* Verify core id. */
+        MESOSPHERE_ASSERT(core_id == GetCurrentCoreId());
+
         /* Ensure all cores get to this point before continuing. */
         cpu::SynchronizeAllCores();
 
@@ -88,18 +88,18 @@ namespace ams::kern::arch::arm64 {
         cpu::SynchronizeAllCores();
 
         /* Restore all local interrupts. */
-        MESOSPHERE_ASSERT(this->local_state_saved);
+        MESOSPHERE_ASSERT(this->local_state_saved[core_id]);
-        this->interrupt_controller.RestoreCoreLocal(std::addressof(this->local_state));
+        this->interrupt_controller.RestoreCoreLocal(std::addressof(this->local_states[core_id]));
-        this->local_state_saved = false;
+        this->local_state_saved[core_id] = false;
 
         /* Ensure all cores get to this point before continuing. */
         cpu::SynchronizeAllCores();
 
         /* If on core 0, restore the global interrupts. */
         if (core_id == 0) {
-            MESOSPHERE_ASSERT(s_global_state_saved);
+            MESOSPHERE_ASSERT(this->global_state_saved);
-            this->interrupt_controller.RestoreGlobal(std::addressof(s_global_state));
+            this->interrupt_controller.RestoreGlobal(std::addressof(this->global_state));
-            s_global_state_saved = false;
+            this->global_state_saved = false;
         }
 
         /* Ensure all cores get to this point before continuing. */
@@ -136,7 +136,7 @@ namespace ams::kern::arch::arm64 {
                 MESOSPHERE_LOG("Core%d: Unhandled local interrupt %d\n", GetCurrentCoreId(), irq);
             }
         } else if (KInterruptController::IsGlobal(irq)) {
-            KScopedSpinLock lk(GetLock());
+            KScopedSpinLock lk(this->GetGlobalInterruptLock());
 
             /* Get global interrupt entry. */
             auto &entry = GetGlobalInterruptEntry(irq);
@@ -213,7 +213,7 @@ namespace ams::kern::arch::arm64 {
         KScopedInterruptDisable di;
 
         if (KInterruptController::IsGlobal(irq)) {
-            KScopedSpinLock lk(GetLock());
+            KScopedSpinLock lk(this->GetGlobalInterruptLock());
             return this->BindGlobal(handler, irq, core_id, priority, manual_clear, level);
         } else {
             MESOSPHERE_ASSERT(core_id == GetCurrentCoreId());
@@ -227,7 +227,7 @@ namespace ams::kern::arch::arm64 {
         KScopedInterruptDisable di;
 
         if (KInterruptController::IsGlobal(irq)) {
-            KScopedSpinLock lk(GetLock());
+            KScopedSpinLock lk(this->GetGlobalInterruptLock());
             return this->UnbindGlobal(irq);
         } else {
             MESOSPHERE_ASSERT(core_id == GetCurrentCoreId());
@@ -239,7 +239,7 @@ namespace ams::kern::arch::arm64 {
         R_UNLESS(KInterruptController::IsGlobal(irq), svc::ResultOutOfRange());
 
         KScopedInterruptDisable di;
-        KScopedSpinLock lk(GetLock());
+        KScopedSpinLock lk(this->GetGlobalInterruptLock());
         return this->ClearGlobal(irq);
     }
 
@@ -249,7 +249,7 @@ namespace ams::kern::arch::arm64 {
         KScopedInterruptDisable di;
 
         if (KInterruptController::IsGlobal(irq)) {
-            KScopedSpinLock lk(GetLock());
+            KScopedSpinLock lk(this->GetGlobalInterruptLock());
             return this->ClearGlobal(irq);
         } else {
             MESOSPHERE_ASSERT(core_id == GetCurrentCoreId());

libraries/libmesosphere/source/arch/arm64/kern_k_page_table.cpp

@@ -181,7 +181,7 @@ namespace ams::kern::arch::arm64 {
         return ResultSuccess();
     }
 
-    Result KPageTable::InitializeForProcess(u32 id, ams::svc::CreateProcessFlag as_type, bool enable_aslr, bool from_back, KMemoryManager::Pool pool, KProcessAddress code_address, size_t code_size, KMemoryBlockSlabManager *mem_block_slab_manager, KBlockInfoManager *block_info_manager, KPageTableManager *pt_manager) {
+    Result KPageTable::InitializeForProcess(u32 id, ams::svc::CreateProcessFlag as_type, bool enable_aslr, bool enable_das_merge, bool from_back, KMemoryManager::Pool pool, KProcessAddress code_address, size_t code_size, KMemoryBlockSlabManager *mem_block_slab_manager, KBlockInfoManager *block_info_manager, KPageTableManager *pt_manager) {
         /* The input ID isn't actually used. */
         MESOSPHERE_UNUSED(id);
 
@@ -202,7 +202,7 @@ namespace ams::kern::arch::arm64 {
         const size_t as_width = GetAddressSpaceWidth(as_type);
         const KProcessAddress as_start = 0;
         const KProcessAddress as_end   = (1ul << as_width);
-        R_TRY(KPageTableBase::InitializeForProcess(as_type, enable_aslr, from_back, pool, GetVoidPointer(new_table), as_start, as_end, code_address, code_size, mem_block_slab_manager, block_info_manager));
+        R_TRY(KPageTableBase::InitializeForProcess(as_type, enable_aslr, enable_das_merge, from_back, pool, GetVoidPointer(new_table), as_start, as_end, code_address, code_size, mem_block_slab_manager, block_info_manager));
 
         /* We succeeded! */
         table_guard.Cancel();
@@ -334,11 +334,11 @@ namespace ams::kern::arch::arm64 {
 
             switch (operation) {
                 case OperationType_Map:
-                    return this->MapContiguous(virt_addr, phys_addr, num_pages, entry_template, page_list, reuse_ll);
+                    return this->MapContiguous(virt_addr, phys_addr, num_pages, entry_template, properties.disable_merge_attributes == DisableMergeAttribute_DisableHead, page_list, reuse_ll);
                 case OperationType_ChangePermissions:
-                    return this->ChangePermissions(virt_addr, num_pages, entry_template, false, page_list, reuse_ll);
+                    return this->ChangePermissions(virt_addr, num_pages, entry_template, properties.disable_merge_attributes, false, page_list, reuse_ll);
                 case OperationType_ChangePermissionsAndRefresh:
-                    return this->ChangePermissions(virt_addr, num_pages, entry_template, true, page_list, reuse_ll);
+                    return this->ChangePermissions(virt_addr, num_pages, entry_template, properties.disable_merge_attributes, true, page_list, reuse_ll);
                 MESOSPHERE_UNREACHABLE_DEFAULT_CASE();
             }
         }
@@ -355,12 +355,12 @@ namespace ams::kern::arch::arm64 {
         auto entry_template = this->GetEntryTemplate(properties);
         switch (operation) {
             case OperationType_MapGroup:
-                return this->MapGroup(virt_addr, page_group, num_pages, entry_template, page_list, reuse_ll);
+                return this->MapGroup(virt_addr, page_group, num_pages, entry_template, properties.disable_merge_attributes == DisableMergeAttribute_DisableHead, page_list, reuse_ll);
             MESOSPHERE_UNREACHABLE_DEFAULT_CASE();
         }
     }
 
-    Result KPageTable::MapL1Blocks(KProcessAddress virt_addr, KPhysicalAddress phys_addr, size_t num_pages, PageTableEntry entry_template, PageLinkedList *page_list, bool reuse_ll) {
+    Result KPageTable::MapL1Blocks(KProcessAddress virt_addr, KPhysicalAddress phys_addr, size_t num_pages, PageTableEntry entry_template, bool disable_head_merge, PageLinkedList *page_list, bool reuse_ll) {
         MESOSPHERE_ASSERT(this->IsLockedByCurrentThread());
         MESOSPHERE_ASSERT(util::IsAligned(GetInteger(virt_addr), L1BlockSize));
         MESOSPHERE_ASSERT(util::IsAligned(GetInteger(phys_addr), L1BlockSize));
@@ -371,10 +371,13 @@ namespace ams::kern::arch::arm64 {
 
         auto &impl = this->GetImpl();
 
+        u8 sw_reserved_bits = PageTableEntry::EncodeSoftwareReservedBits(disable_head_merge, false, false);
+
         /* Iterate, mapping each block. */
         for (size_t i = 0; i < num_pages; i += L1BlockSize / PageSize) {
             /* Map the block. */
-            *impl.GetL1Entry(virt_addr) = L1PageTableEntry(PageTableEntry::BlockTag{}, phys_addr, PageTableEntry(entry_template), false);
+            *impl.GetL1Entry(virt_addr) = L1PageTableEntry(PageTableEntry::BlockTag{}, phys_addr, PageTableEntry(entry_template), sw_reserved_bits, false);
+            sw_reserved_bits &= ~(PageTableEntry::SoftwareReservedBit_DisableMergeHead);
             virt_addr += L1BlockSize;
             phys_addr += L1BlockSize;
         }
@@ -382,7 +385,7 @@ namespace ams::kern::arch::arm64 {
         return ResultSuccess();
     }
 
-    Result KPageTable::MapL2Blocks(KProcessAddress virt_addr, KPhysicalAddress phys_addr, size_t num_pages, PageTableEntry entry_template, PageLinkedList *page_list, bool reuse_ll) {
+    Result KPageTable::MapL2Blocks(KProcessAddress virt_addr, KPhysicalAddress phys_addr, size_t num_pages, PageTableEntry entry_template, bool disable_head_merge, PageLinkedList *page_list, bool reuse_ll) {
         MESOSPHERE_ASSERT(this->IsLockedByCurrentThread());
         MESOSPHERE_ASSERT(util::IsAligned(GetInteger(virt_addr), L2BlockSize));
         MESOSPHERE_ASSERT(util::IsAligned(GetInteger(phys_addr), L2BlockSize));
@@ -392,6 +395,8 @@ namespace ams::kern::arch::arm64 {
         KVirtualAddress l2_virt = Null<KVirtualAddress>;
         int l2_open_count = 0;
 
+        u8 sw_reserved_bits = PageTableEntry::EncodeSoftwareReservedBits(disable_head_merge, false, false);
+
         /* Iterate, mapping each block. */
         for (size_t i = 0; i < num_pages; i += L2BlockSize / PageSize) {
             KPhysicalAddress l2_phys = Null<KPhysicalAddress>;
@@ -415,7 +420,8 @@ namespace ams::kern::arch::arm64 {
             MESOSPHERE_ASSERT(l2_virt != Null<KVirtualAddress>);
 
             /* Map the block. */
-            *impl.GetL2EntryFromTable(l2_virt, virt_addr) = L2PageTableEntry(PageTableEntry::BlockTag{}, phys_addr, PageTableEntry(entry_template), false);
+            *impl.GetL2EntryFromTable(l2_virt, virt_addr) = L2PageTableEntry(PageTableEntry::BlockTag{}, phys_addr, PageTableEntry(entry_template), sw_reserved_bits, false);
+            sw_reserved_bits &= ~(PageTableEntry::SoftwareReservedBit_DisableMergeHead);
             l2_open_count++;
             virt_addr += L2BlockSize;
             phys_addr += L2BlockSize;
@@ -438,7 +444,7 @@ namespace ams::kern::arch::arm64 {
         return ResultSuccess();
     }
 
-    Result KPageTable::MapL3Blocks(KProcessAddress virt_addr, KPhysicalAddress phys_addr, size_t num_pages, PageTableEntry entry_template, PageLinkedList *page_list, bool reuse_ll) {
+    Result KPageTable::MapL3Blocks(KProcessAddress virt_addr, KPhysicalAddress phys_addr, size_t num_pages, PageTableEntry entry_template, bool disable_head_merge, PageLinkedList *page_list, bool reuse_ll) {
         MESOSPHERE_ASSERT(this->IsLockedByCurrentThread());
         MESOSPHERE_ASSERT(util::IsAligned(GetInteger(virt_addr), PageSize));
         MESOSPHERE_ASSERT(util::IsAligned(GetInteger(phys_addr), PageSize));
@@ -449,6 +455,8 @@ namespace ams::kern::arch::arm64 {
         int l2_open_count = 0;
         int l3_open_count = 0;
 
+        u8 sw_reserved_bits = PageTableEntry::EncodeSoftwareReservedBits(disable_head_merge, false, false);
+
         /* Iterate, mapping each page. */
         for (size_t i = 0; i < num_pages; i++) {
             KPhysicalAddress l3_phys = Null<KPhysicalAddress>;
@@ -505,7 +513,8 @@ namespace ams::kern::arch::arm64 {
             MESOSPHERE_ASSERT(l3_virt != Null<KVirtualAddress>);
 
             /* Map the page. */
-            *impl.GetL3EntryFromTable(l3_virt, virt_addr) = L3PageTableEntry(PageTableEntry::BlockTag{}, phys_addr, PageTableEntry(entry_template), false);
+            *impl.GetL3EntryFromTable(l3_virt, virt_addr) = L3PageTableEntry(PageTableEntry::BlockTag{}, phys_addr, PageTableEntry(entry_template), sw_reserved_bits, false);
+            sw_reserved_bits &= ~(PageTableEntry::SoftwareReservedBit_DisableMergeHead);
             l3_open_count++;
             virt_addr += PageSize;
             phys_addr += PageSize;
@@ -564,7 +573,7 @@ namespace ams::kern::arch::arm64 {
 
         /* Ensure that any pages we track close on exit. */
         KPageGroup pages_to_close(this->GetBlockInfoManager());
-        KScopedPageGroup spg(pages_to_close);
+        ON_SCOPE_EXIT { pages_to_close.Close(); };
 
         /* Begin traversal. */
         TraversalContext context;
@@ -702,7 +711,7 @@ namespace ams::kern::arch::arm64 {
         return ResultSuccess();
     }
 
-    Result KPageTable::MapContiguous(KProcessAddress virt_addr, KPhysicalAddress phys_addr, size_t num_pages, PageTableEntry entry_template, PageLinkedList *page_list, bool reuse_ll) {
+    Result KPageTable::MapContiguous(KProcessAddress virt_addr, KPhysicalAddress phys_addr, size_t num_pages, PageTableEntry entry_template, bool disable_head_merge, PageLinkedList *page_list, bool reuse_ll) {
         MESOSPHERE_ASSERT(this->IsLockedByCurrentThread());
 
         /* Cache initial addresses for use on cleanup. */
@@ -716,7 +725,7 @@ namespace ams::kern::arch::arm64 {
             auto map_guard = SCOPE_GUARD { MESOSPHERE_R_ABORT_UNLESS(this->Unmap(orig_virt_addr, num_pages, page_list, true, true)); };
 
             if (num_pages < ContiguousPageSize / PageSize) {
-                R_TRY(this->Map(virt_addr, phys_addr, num_pages, entry_template, L3BlockSize, page_list, reuse_ll));
+                R_TRY(this->Map(virt_addr, phys_addr, num_pages, entry_template, disable_head_merge && virt_addr == orig_virt_addr, L3BlockSize, page_list, reuse_ll));
                 remaining_pages -= num_pages;
                 virt_addr += num_pages * PageSize;
                 phys_addr += num_pages * PageSize;
@@ -732,7 +741,7 @@ namespace ams::kern::arch::arm64 {
 
                     /* Map pages, if we should. */
                     if (pages_to_map > 0) {
-                        R_TRY(this->Map(virt_addr, phys_addr, pages_to_map, entry_template, GetSmallerAlignment(alignment), page_list, reuse_ll));
+                        R_TRY(this->Map(virt_addr, phys_addr, pages_to_map, entry_template, disable_head_merge && virt_addr == orig_virt_addr, GetSmallerAlignment(alignment), page_list, reuse_ll));
                         remaining_pages -= pages_to_map;
                         virt_addr += pages_to_map * PageSize;
                         phys_addr += pages_to_map * PageSize;
@@ -753,7 +762,7 @@ namespace ams::kern::arch::arm64 {
                     /* Map pages, if we should. */
                     const size_t pages_to_map = util::AlignDown(remaining_pages, alignment / PageSize);
                     if (pages_to_map > 0) {
-                        R_TRY(this->Map(virt_addr, phys_addr, pages_to_map, entry_template, alignment, page_list, reuse_ll));
+                        R_TRY(this->Map(virt_addr, phys_addr, pages_to_map, entry_template, disable_head_merge && virt_addr == orig_virt_addr, alignment, page_list, reuse_ll));
                         remaining_pages -= pages_to_map;
                         virt_addr += pages_to_map * PageSize;
                         phys_addr += pages_to_map * PageSize;
@@ -779,7 +788,7 @@ namespace ams::kern::arch::arm64 {
         return ResultSuccess();
     }
 
-    Result KPageTable::MapGroup(KProcessAddress virt_addr, const KPageGroup &pg, size_t num_pages, PageTableEntry entry_template, PageLinkedList *page_list, bool reuse_ll) {
+    Result KPageTable::MapGroup(KProcessAddress virt_addr, const KPageGroup &pg, size_t num_pages, PageTableEntry entry_template, bool disable_head_merge, PageLinkedList *page_list, bool reuse_ll) {
         MESOSPHERE_ASSERT(this->IsLockedByCurrentThread());
 
         /* We want to maintain a new reference to every page in the group. */
@@ -798,7 +807,7 @@ namespace ams::kern::arch::arm64 {
                 for (const auto &block : pg) {
                     const KPhysicalAddress block_phys_addr = GetLinearMappedPhysicalAddress(block.GetAddress());
                     const size_t cur_pages = block.GetNumPages();
-                    R_TRY(this->Map(virt_addr, block_phys_addr, cur_pages, entry_template, L3BlockSize, page_list, reuse_ll));
+                    R_TRY(this->Map(virt_addr, block_phys_addr, cur_pages, entry_template, disable_head_merge && virt_addr == orig_virt_addr, L3BlockSize, page_list, reuse_ll));
 
                     virt_addr    += cur_pages * PageSize;
                     mapped_pages += cur_pages;
@@ -846,7 +855,7 @@ namespace ams::kern::arch::arm64 {
                             }
 
                             /* Map! */
-                            R_TRY(this->Map(virt_choice, phys_choice, virt_pages, entry_template, virt_block.GetAlignment(), page_list, reuse_ll));
+                            R_TRY(this->Map(virt_choice, phys_choice, virt_pages, entry_template, disable_head_merge && virt_addr == orig_virt_addr, virt_block.GetAlignment(), page_list, reuse_ll));
 
                             /* Advance. */
                             phys_choice  += virt_pages * PageSize;
@@ -893,26 +902,39 @@ namespace ams::kern::arch::arm64 {
         if (l2_entry->IsTable()) {
             /* We have an L3 entry. */
             L3PageTableEntry *l3_entry = impl.GetL3Entry(l2_entry, virt_addr);
-            if (!l3_entry->IsBlock() || !l3_entry->IsContiguousAllowed()) {
+            if (!l3_entry->IsBlock()) {
                 return merged;
             }
 
             /* If it's not contiguous, try to make it so. */
             if (!l3_entry->IsContiguous()) {
                 virt_addr = util::AlignDown(GetInteger(virt_addr), L3ContiguousBlockSize);
-                KPhysicalAddress phys_addr = util::AlignDown(GetInteger(l3_entry->GetBlock()), L3ContiguousBlockSize);
+                const KPhysicalAddress phys_addr = util::AlignDown(GetInteger(l3_entry->GetBlock()), L3ContiguousBlockSize);
-                const u64 entry_template = l3_entry->GetEntryTemplate();
+                const u64 entry_template = l3_entry->GetEntryTemplateForMerge();
 
                 /* Validate that we can merge. */
                 for (size_t i = 0; i < L3ContiguousBlockSize / L3BlockSize; i++) {
-                    if (!impl.GetL3Entry(l2_entry, virt_addr + L3BlockSize * i)->Is(entry_template | GetInteger(phys_addr + L3BlockSize * i) | PageTableEntry::Type_L3Block)) {
+                    const L3PageTableEntry *check_entry = impl.GetL3Entry(l2_entry, virt_addr + L3BlockSize * i);
+                    if (!check_entry->IsForMerge(entry_template | GetInteger(phys_addr + L3BlockSize * i) | PageTableEntry::Type_L3Block)) {
+                        return merged;
+                    }
+                    if (i > 0 && (check_entry->IsHeadOrHeadAndBodyMergeDisabled())) {
+                        return merged;
+                    }
+                    if ((i < (L3ContiguousBlockSize / L3BlockSize) - 1) && check_entry->IsTailMergeDisabled()) {
                         return merged;
                     }
                 }
 
+                /* Determine the new software reserved bits. */
+                const L3PageTableEntry *head_entry = impl.GetL3Entry(l2_entry, virt_addr + L3BlockSize * 0);
+                const L3PageTableEntry *tail_entry = impl.GetL3Entry(l2_entry, virt_addr + L3BlockSize * ((L3ContiguousBlockSize / L3BlockSize) - 1));
+                auto sw_reserved_bits = PageTableEntry::EncodeSoftwareReservedBits(head_entry->IsHeadMergeDisabled(), head_entry->IsHeadAndBodyMergeDisabled(), tail_entry->IsTailMergeDisabled());
+
                 /* Merge! */
                 for (size_t i = 0; i < L3ContiguousBlockSize / L3BlockSize; i++) {
-                    impl.GetL3Entry(l2_entry, virt_addr + L3BlockSize * i)->SetContiguous(true);
+                    *impl.GetL3Entry(l2_entry, virt_addr + L3BlockSize * i) = L3PageTableEntry(PageTableEntry::BlockTag{}, phys_addr + L3BlockSize * i, PageTableEntry(entry_template), sw_reserved_bits, true);
+                    sw_reserved_bits &= ~(PageTableEntry::SoftwareReservedBit_DisableMergeHead);
                 }
 
                 /* Note that we updated. */
@@ -923,18 +945,30 @@ namespace ams::kern::arch::arm64 {
             /* We might be able to upgrade a contiguous set of L3 entries into an L2 block. */
             virt_addr = util::AlignDown(GetInteger(virt_addr), L2BlockSize);
             KPhysicalAddress phys_addr = util::AlignDown(GetInteger(l3_entry->GetBlock()), L2BlockSize);
-            const u64 entry_template = l3_entry->GetEntryTemplate();
+            const u64 entry_template = l3_entry->GetEntryTemplateForMerge();
 
             /* Validate that we can merge. */
             for (size_t i = 0; i < L2BlockSize / L3ContiguousBlockSize; i++) {
-                if (!impl.GetL3Entry(l2_entry, virt_addr + L3ContiguousBlockSize * i)->Is(entry_template | GetInteger(phys_addr + L3ContiguousBlockSize * i) | PageTableEntry::ContigType_Contiguous | PageTableEntry::Type_L3Block)) {
+                const L3PageTableEntry *check_entry = impl.GetL3Entry(l2_entry, virt_addr + L3ContiguousBlockSize * i);
+                if (!check_entry->IsForMerge(entry_template | GetInteger(phys_addr + L3ContiguousBlockSize * i) | PageTableEntry::ContigType_Contiguous | PageTableEntry::Type_L3Block)) {
+                    return merged;
+                }
+                if (i > 0 && (check_entry->IsHeadOrHeadAndBodyMergeDisabled())) {
+                    return merged;
+                }
+                if ((i < (L2BlockSize / L3ContiguousBlockSize) - 1) && check_entry->IsTailMergeDisabled()) {
                     return merged;
                 }
             }
 
+            /* Determine the new software reserved bits. */
+            const L3PageTableEntry *head_entry = impl.GetL3Entry(l2_entry, virt_addr + L3ContiguousBlockSize * 0);
+            const L3PageTableEntry *tail_entry = impl.GetL3Entry(l2_entry, virt_addr + L3ContiguousBlockSize * ((L2BlockSize / L3ContiguousBlockSize) - 1));
+            auto sw_reserved_bits = PageTableEntry::EncodeSoftwareReservedBits(head_entry->IsHeadMergeDisabled(), head_entry->IsHeadAndBodyMergeDisabled(), tail_entry->IsTailMergeDisabled());
+
             /* Merge! */
             PteDataSynchronizationBarrier();
-            *l2_entry = L2PageTableEntry(PageTableEntry::BlockTag{}, phys_addr, PageTableEntry(entry_template), false);
+            *l2_entry = L2PageTableEntry(PageTableEntry::BlockTag{}, phys_addr, PageTableEntry(entry_template), sw_reserved_bits, false);
 
             /* Note that we updated. */
             this->NoteUpdated();
@@ -950,7 +984,7 @@ namespace ams::kern::arch::arm64 {
         }
 
         /* If the l2 entry is not a block or we can't make it contiguous, we're done. */
-        if (!l2_entry->IsBlock() || !l2_entry->IsContiguousAllowed()) {
+        if (!l2_entry->IsBlock()) {
             return merged;
         }
 
@@ -958,18 +992,31 @@ namespace ams::kern::arch::arm64 {
         if (!l2_entry->IsContiguous()) {
             virt_addr = util::AlignDown(GetInteger(virt_addr), L2ContiguousBlockSize);
             KPhysicalAddress phys_addr = util::AlignDown(GetInteger(l2_entry->GetBlock()), L2ContiguousBlockSize);
-            const u64 entry_template = l2_entry->GetEntryTemplate();
+            const u64 entry_template = l2_entry->GetEntryTemplateForMerge();
 
             /* Validate that we can merge. */
             for (size_t i = 0; i < L2ContiguousBlockSize / L2BlockSize; i++) {
-                if (!impl.GetL2Entry(l1_entry, virt_addr + L2BlockSize * i)->Is(entry_template | GetInteger(phys_addr + L2BlockSize * i) | PageTableEntry::Type_L2Block)) {
+                const L2PageTableEntry *check_entry = impl.GetL2Entry(l1_entry, virt_addr + L2BlockSize * i);
+                if (!check_entry->IsForMerge(entry_template | GetInteger(phys_addr + L2BlockSize * i) | PageTableEntry::Type_L2Block)) {
+                    return merged;
+                }
+                if (i > 0 && (check_entry->IsHeadOrHeadAndBodyMergeDisabled())) {
+                    return merged;
+                }
+                if ((i < (L2ContiguousBlockSize / L2BlockSize) - 1) && check_entry->IsTailMergeDisabled()) {
                     return merged;
                 }
             }
 
+            /* Determine the new software reserved bits. */
+            const L2PageTableEntry *head_entry = impl.GetL2Entry(l1_entry, virt_addr + L2BlockSize * 0);
+            const L2PageTableEntry *tail_entry = impl.GetL2Entry(l1_entry, virt_addr + L2BlockSize * ((L2ContiguousBlockSize / L2BlockSize) - 1));
+            auto sw_reserved_bits = PageTableEntry::EncodeSoftwareReservedBits(head_entry->IsHeadMergeDisabled(), head_entry->IsHeadAndBodyMergeDisabled(), tail_entry->IsTailMergeDisabled());
+
             /* Merge! */
             for (size_t i = 0; i < L2ContiguousBlockSize / L2BlockSize; i++) {
-                impl.GetL2Entry(l1_entry, virt_addr + L2BlockSize * i)->SetContiguous(true);
+                *impl.GetL2Entry(l1_entry, virt_addr + L2BlockSize * i) = L2PageTableEntry(PageTableEntry::BlockTag{}, phys_addr + L2BlockSize * i, PageTableEntry(entry_template), sw_reserved_bits, true);
+                sw_reserved_bits &= ~(PageTableEntry::SoftwareReservedBit_DisableMergeHead);
             }
 
             /* Note that we updated. */
@@ -980,18 +1027,30 @@ namespace ams::kern::arch::arm64 {
         /* We might be able to upgrade a contiguous set of L2 entries into an L1 block. */
         virt_addr = util::AlignDown(GetInteger(virt_addr), L1BlockSize);
         KPhysicalAddress phys_addr = util::AlignDown(GetInteger(l2_entry->GetBlock()), L1BlockSize);
-        const u64 entry_template = l2_entry->GetEntryTemplate();
+        const u64 entry_template = l2_entry->GetEntryTemplateForMerge();
 
         /* Validate that we can merge. */
         for (size_t i = 0; i < L1BlockSize / L2ContiguousBlockSize; i++) {
-            if (!impl.GetL2Entry(l1_entry, virt_addr + L2ContiguousBlockSize * i)->Is(entry_template | GetInteger(phys_addr + L2ContiguousBlockSize * i) | PageTableEntry::ContigType_Contiguous | PageTableEntry::Type_L2Block)) {
+            const L2PageTableEntry *check_entry = impl.GetL2Entry(l1_entry, virt_addr + L2ContiguousBlockSize * i);
+            if (!check_entry->IsForMerge(entry_template | GetInteger(phys_addr + L2ContiguousBlockSize * i) | PageTableEntry::ContigType_Contiguous | PageTableEntry::Type_L2Block)) {
+                return merged;
+            }
+            if (i > 0 && (check_entry->IsHeadOrHeadAndBodyMergeDisabled())) {
+                return merged;
+            }
+            if ((i < (L1ContiguousBlockSize / L2ContiguousBlockSize) - 1) && check_entry->IsTailMergeDisabled()) {
                 return merged;
             }
         }
 
+        /* Determine the new software reserved bits. */
+        const L2PageTableEntry *head_entry = impl.GetL2Entry(l1_entry, virt_addr + L2ContiguousBlockSize * 0);
+        const L2PageTableEntry *tail_entry = impl.GetL2Entry(l1_entry, virt_addr + L2ContiguousBlockSize * ((L1BlockSize / L2ContiguousBlockSize) - 1));
+        auto sw_reserved_bits = PageTableEntry::EncodeSoftwareReservedBits(head_entry->IsHeadMergeDisabled(), head_entry->IsHeadAndBodyMergeDisabled(), tail_entry->IsTailMergeDisabled());
+
         /* Merge! */
         PteDataSynchronizationBarrier();
-        *l1_entry = L1PageTableEntry(PageTableEntry::BlockTag{}, phys_addr, PageTableEntry(entry_template), false);
+        *l1_entry = L1PageTableEntry(PageTableEntry::BlockTag{}, phys_addr, PageTableEntry(entry_template), sw_reserved_bits, false);
 
         /* Note that we updated. */
         this->NoteUpdated();
@@ -1029,9 +1088,9 @@ namespace ams::kern::arch::arm64 {
             const KPhysicalAddress l2_phys = GetPageTablePhysicalAddress(l2_table);
 
             /* Set the entries in the L2 table. */
-            const u64 entry_template = l1_entry->GetEntryTemplate();
             for (size_t i = 0; i < L1BlockSize / L2BlockSize; i++) {
-                *(impl.GetL2EntryFromTable(l2_table, block_virt_addr + L2BlockSize * i)) = L2PageTableEntry(PageTableEntry::BlockTag{}, block_phys_addr + L2BlockSize * i, PageTableEntry(entry_template), true);
+                const u64 entry_template = l1_entry->GetEntryTemplateForL2Block(i);
+                *(impl.GetL2EntryFromTable(l2_table, block_virt_addr + L2BlockSize * i)) = L2PageTableEntry(PageTableEntry::BlockTag{}, block_phys_addr + L2BlockSize * i, PageTableEntry(entry_template), PageTableEntry::SoftwareReservedBit_None, true);
             }
 
             /* Open references to the L2 table. */
@@ -1055,10 +1114,13 @@ namespace ams::kern::arch::arm64 {
             /* If we're contiguous, try to separate. */
             if (l2_entry->IsContiguous()) {
                 const KProcessAddress block_virt_addr  = util::AlignDown(GetInteger(virt_addr), L2ContiguousBlockSize);
+                const KPhysicalAddress block_phys_addr = util::AlignDown(GetInteger(l2_entry->GetBlock()), L2ContiguousBlockSize);
 
                 /* Mark the entries as non-contiguous. */
                 for (size_t i = 0; i < L2ContiguousBlockSize / L2BlockSize; i++) {
-                    impl.GetL2Entry(l1_entry, block_virt_addr + L2BlockSize * i)->SetContiguous(false);
+                    L2PageTableEntry *target = impl.GetL2Entry(l1_entry, block_virt_addr + L2BlockSize * i);
+                    const u64 entry_template = target->GetEntryTemplateForL2Block(i);
+                    *target = L2PageTableEntry(PageTableEntry::BlockTag{}, block_phys_addr + L2BlockSize * i, PageTableEntry(entry_template), PageTableEntry::SoftwareReservedBit_None, false);
                 }
                 this->NoteUpdated();
             }
@@ -1076,9 +1138,9 @@ namespace ams::kern::arch::arm64 {
             const KPhysicalAddress l3_phys = GetPageTablePhysicalAddress(l3_table);
 
             /* Set the entries in the L3 table. */
-            const u64 entry_template = l2_entry->GetEntryTemplate();
             for (size_t i = 0; i < L2BlockSize / L3BlockSize; i++) {
-                *(impl.GetL3EntryFromTable(l3_table, block_virt_addr + L3BlockSize * i)) = L3PageTableEntry(PageTableEntry::BlockTag{}, block_phys_addr + L3BlockSize * i, PageTableEntry(entry_template), true);
+                const u64 entry_template = l2_entry->GetEntryTemplateForL3Block(i);
+                *(impl.GetL3EntryFromTable(l3_table, block_virt_addr + L3BlockSize * i)) = L3PageTableEntry(PageTableEntry::BlockTag{}, block_phys_addr + L3BlockSize * i, PageTableEntry(entry_template), PageTableEntry::SoftwareReservedBit_None, true);
             }
 
             /* Open references to the L3 table. */
@@ -1101,10 +1163,13 @@ namespace ams::kern::arch::arm64 {
         L3PageTableEntry *l3_entry = impl.GetL3Entry(l2_entry, virt_addr);
         if (l3_entry->IsBlock() && l3_entry->IsContiguous()) {
             const KProcessAddress block_virt_addr  = util::AlignDown(GetInteger(virt_addr), L3ContiguousBlockSize);
+            const KPhysicalAddress block_phys_addr = util::AlignDown(GetInteger(l3_entry->GetBlock()), L3ContiguousBlockSize);
 
             /* Mark the entries as non-contiguous. */
             for (size_t i = 0; i < L3ContiguousBlockSize / L3BlockSize; i++) {
-                impl.GetL3Entry(l2_entry, block_virt_addr + L3BlockSize * i)->SetContiguous(false);
+                L3PageTableEntry *target = impl.GetL3Entry(l2_entry, block_virt_addr + L3BlockSize * i);
+                const u64 entry_template = target->GetEntryTemplateForL3Block(i);
+                *target = L3PageTableEntry(PageTableEntry::BlockTag{}, block_phys_addr + L3BlockSize * i, PageTableEntry(entry_template), PageTableEntry::SoftwareReservedBit_None, false);
             }
             this->NoteUpdated();
         }
@@ -1124,7 +1189,7 @@ namespace ams::kern::arch::arm64 {
         return ResultSuccess();
     }
 
-    Result KPageTable::ChangePermissions(KProcessAddress virt_addr, size_t num_pages, PageTableEntry entry_template, bool refresh_mapping, PageLinkedList *page_list, bool reuse_ll) {
+    Result KPageTable::ChangePermissions(KProcessAddress virt_addr, size_t num_pages, PageTableEntry entry_template, DisableMergeAttribute disable_merge_attr, bool refresh_mapping, PageLinkedList *page_list, bool reuse_ll) {
         MESOSPHERE_ASSERT(this->IsLockedByCurrentThread());
 
         /* Separate pages before we change permissions. */
@@ -1149,23 +1214,75 @@ namespace ams::kern::arch::arm64 {
             ApplyOption_MergeMappings  = (1u << 1),
         };
 
-        auto ApplyEntryTemplate = [this, virt_addr, num_pages, page_list](PageTableEntry entry_template, u32 apply_option) -> void {
+        auto ApplyEntryTemplate = [this, virt_addr, disable_merge_attr, num_pages, page_list](PageTableEntry entry_template, u32 apply_option) -> void {
             /* Create work variables for us to use. */
+            const KProcessAddress orig_virt_addr = virt_addr;
+            const KProcessAddress end_virt_addr  = orig_virt_addr + (num_pages * PageSize);
             KProcessAddress cur_virt_addr = virt_addr;
             size_t remaining_pages = num_pages;
 
             auto &impl = this->GetImpl();
 
+            /* Parse the disable merge attrs. */
+            const bool attr_disable_head           = (disable_merge_attr & DisableMergeAttribute_DisableHead) != 0;
+            const bool attr_disable_head_body      = (disable_merge_attr & DisableMergeAttribute_DisableHeadAndBody) != 0;
+            const bool attr_enable_head_body       = (disable_merge_attr & DisableMergeAttribute_EnableHeadAndBody) != 0;
+            const bool attr_disable_tail           = (disable_merge_attr & DisableMergeAttribute_DisableTail) != 0;
+            const bool attr_enable_tail            = (disable_merge_attr & DisableMergeAttribute_EnableTail) != 0;
+            const bool attr_enable_and_merge       = (disable_merge_attr & DisableMergeAttribute_EnableAndMergeHeadBodyTail) != 0;
+
             /* Begin traversal. */
             TraversalContext context;
             TraversalEntry   next_entry;
             MESOSPHERE_ABORT_UNLESS(impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), cur_virt_addr));
 
             /* Continue changing properties until we've changed them for all pages. */
+            bool cleared_disable_merge_bits = false;
             while (remaining_pages > 0) {
                 MESOSPHERE_ABORT_UNLESS(util::IsAligned(GetInteger(next_entry.phys_addr), next_entry.block_size));
                 MESOSPHERE_ABORT_UNLESS(next_entry.block_size <= remaining_pages * PageSize);
 
+                /* Determine if we're at the start. */
+                const bool is_start = (cur_virt_addr == orig_virt_addr);
+                const bool is_end   = ((cur_virt_addr + next_entry.block_size) == end_virt_addr);
+
+                /* Determine the relevant merge attributes. */
+                bool disable_head_merge, disable_head_body_merge, disable_tail_merge;
+                if (next_entry.IsHeadMergeDisabled()) {
+                    disable_head_merge = true;
+                } else if (attr_disable_head) {
+                    disable_head_merge = is_start;
+                } else {
+                    disable_head_merge = false;
+                }
+                if (is_start) {
+                    if (attr_disable_head_body) {
+                        disable_head_body_merge = true;
+                    } else if (attr_enable_head_body) {
+                        disable_head_body_merge = false;
+                    } else {
+                        disable_head_body_merge = (!attr_enable_and_merge && next_entry.IsHeadAndBodyMergeDisabled());
+                    }
+                } else {
+                    disable_head_body_merge = (!attr_enable_and_merge && next_entry.IsHeadAndBodyMergeDisabled());
+                    cleared_disable_merge_bits |= (attr_enable_and_merge && next_entry.IsHeadAndBodyMergeDisabled());
+                }
+                if (is_end) {
+                    if (attr_disable_tail) {
+                        disable_tail_merge = true;
+                    } else if (attr_enable_tail) {
+                        disable_tail_merge = false;
+                    } else {
+                        disable_tail_merge = (!attr_enable_and_merge && next_entry.IsTailMergeDisabled());
+                    }
+                } else {
+                    disable_tail_merge = (!attr_enable_and_merge && next_entry.IsTailMergeDisabled());
+                    cleared_disable_merge_bits |= (attr_enable_and_merge && next_entry.IsTailMergeDisabled());
+                }
+
+                /* Encode the merge disable flags into the software reserved bits. */
+                u8 sw_reserved_bits = PageTableEntry::EncodeSoftwareReservedBits(disable_head_merge, disable_head_body_merge, disable_tail_merge);
+
                 /* If we should flush entries, do so. */
                 if ((apply_option & ApplyOption_FlushDataCache) != 0) {
                     if (IsHeapPhysicalAddress(next_entry.phys_addr)) {
@@ -1179,7 +1296,7 @@ namespace ams::kern::arch::arm64 {
                     case L1BlockSize:
                         {
                             /* Write the updated entry. */
-                            *l1_entry = L1PageTableEntry(PageTableEntry::BlockTag{}, next_entry.phys_addr, entry_template, false);
+                            *l1_entry = L1PageTableEntry(PageTableEntry::BlockTag{}, next_entry.phys_addr, entry_template, sw_reserved_bits, false);
                         }
                         break;
                     case L2ContiguousBlockSize:
@@ -1196,7 +1313,8 @@ namespace ams::kern::arch::arm64 {
                             /* Write the updated entry. */
                             const bool contig = next_entry.block_size == L2ContiguousBlockSize;
                             for (size_t i = 0; i < num_l2_blocks; i++) {
-                                *impl.GetL2EntryFromTable(l2_virt, cur_virt_addr + L2BlockSize * i) = L2PageTableEntry(PageTableEntry::BlockTag{}, next_entry.phys_addr + L2BlockSize * i, entry_template, contig);
+                                *impl.GetL2EntryFromTable(l2_virt, cur_virt_addr + L2BlockSize * i) = L2PageTableEntry(PageTableEntry::BlockTag{}, next_entry.phys_addr + L2BlockSize * i, entry_template, sw_reserved_bits, contig);
+                                sw_reserved_bits &= ~(PageTableEntry::SoftwareReservedBit_DisableMergeHead);
                             }
                         }
                         break;
@@ -1220,7 +1338,8 @@ namespace ams::kern::arch::arm64 {
                             /* Write the updated entry. */
                             const bool contig = next_entry.block_size == L3ContiguousBlockSize;
                             for (size_t i = 0; i < num_l3_blocks; i++) {
-                                *impl.GetL3EntryFromTable(l3_virt, cur_virt_addr + L3BlockSize * i) = L3PageTableEntry(PageTableEntry::BlockTag{}, next_entry.phys_addr + L3BlockSize * i, entry_template, contig);
+                                *impl.GetL3EntryFromTable(l3_virt, cur_virt_addr + L3BlockSize * i) = L3PageTableEntry(PageTableEntry::BlockTag{}, next_entry.phys_addr + L3BlockSize * i, entry_template, sw_reserved_bits, contig);
+                                sw_reserved_bits &= ~(PageTableEntry::SoftwareReservedBit_DisableMergeHead);
                             }
                         }
                         break;
@@ -1228,12 +1347,12 @@ namespace ams::kern::arch::arm64 {
                 }
 
                 /* If our option asks us to, try to merge mappings. */
-                bool merge = ((apply_option & ApplyOption_MergeMappings) != 0) && next_entry.block_size < L1BlockSize;
+                bool merge = ((apply_option & ApplyOption_MergeMappings) != 0 || cleared_disable_merge_bits) && next_entry.block_size < L1BlockSize;
                 if (merge) {
                     const size_t larger_align = GetLargerAlignment(next_entry.block_size);
                     if (util::IsAligned(GetInteger(cur_virt_addr) + next_entry.block_size, larger_align)) {
                         const uintptr_t aligned_start = util::AlignDown(GetInteger(cur_virt_addr), larger_align);
-                        if (virt_addr <= aligned_start && aligned_start + larger_align - 1 < GetInteger(virt_addr) + (num_pages * PageSize) - 1) {
+                        if (orig_virt_addr <= aligned_start && aligned_start + larger_align - 1 < GetInteger(orig_virt_addr) + (num_pages * PageSize) - 1) {
                             merge = this->MergePages(cur_virt_addr, page_list);
                         } else {
                             merge = false;

libraries/libmesosphere/source/arch/arm64/kern_k_page_table_impl.cpp

@@ -45,11 +45,13 @@ namespace ams::kern::arch::arm64 {
             } else {
                 out_entry->block_size = L3BlockSize;
             }
+            out_entry->sw_reserved_bits = l3_entry->GetSoftwareReservedBits();
 
             return true;
         } else {
-            out_entry->phys_addr  = Null<KPhysicalAddress>;
+            out_entry->phys_addr        = Null<KPhysicalAddress>;
-            out_entry->block_size = L3BlockSize;
+            out_entry->block_size       = L3BlockSize;
+            out_entry->sw_reserved_bits = 0;
             return false;
         }
     }
@@ -66,14 +68,17 @@ namespace ams::kern::arch::arm64 {
             } else {
                 out_entry->block_size = L2BlockSize;
             }
+            out_entry->sw_reserved_bits = l2_entry->GetSoftwareReservedBits();
+
             /* Set the output context. */
             out_context->l3_entry = nullptr;
             return true;
         } else if (l2_entry->IsTable()) {
             return this->ExtractL3Entry(out_entry, out_context, this->GetL3EntryFromTable(GetPageTableVirtualAddress(l2_entry->GetTable()), virt_addr), virt_addr);
         } else {
-            out_entry->phys_addr  = Null<KPhysicalAddress>;
+            out_entry->phys_addr        = Null<KPhysicalAddress>;
-            out_entry->block_size = L2BlockSize;
+            out_entry->block_size       = L2BlockSize;
+            out_entry->sw_reserved_bits = 0;
             out_context->l3_entry = nullptr;
             return false;
         }
@@ -91,6 +96,8 @@ namespace ams::kern::arch::arm64 {
             } else {
                 out_entry->block_size = L1BlockSize;
             }
+            out_entry->sw_reserved_bits = l1_entry->GetSoftwareReservedBits();
+
             /* Set the output context. */
             out_context->l2_entry = nullptr;
             out_context->l3_entry = nullptr;
@@ -98,8 +105,9 @@ namespace ams::kern::arch::arm64 {
         } else if (l1_entry->IsTable()) {
             return this->ExtractL2Entry(out_entry, out_context, this->GetL2EntryFromTable(GetPageTableVirtualAddress(l1_entry->GetTable()), virt_addr), virt_addr);
         } else {
-            out_entry->phys_addr  = Null<KPhysicalAddress>;
+            out_entry->phys_addr        = Null<KPhysicalAddress>;
-            out_entry->block_size = L1BlockSize;
+            out_entry->block_size       = L1BlockSize;
+            out_entry->sw_reserved_bits = 0;
             out_context->l2_entry = nullptr;
             out_context->l3_entry = nullptr;
             return false;
@@ -108,8 +116,9 @@ namespace ams::kern::arch::arm64 {
 
     bool KPageTableImpl::BeginTraversal(TraversalEntry *out_entry, TraversalContext *out_context, KProcessAddress address) const {
         /* Setup invalid defaults. */
-        out_entry->phys_addr  = Null<KPhysicalAddress>;
+        out_entry->phys_addr        = Null<KPhysicalAddress>;
-        out_entry->block_size = L1BlockSize;
+        out_entry->block_size       = L1BlockSize;
+        out_entry->sw_reserved_bits = 0;
         out_context->l1_entry = this->table + this->num_entries;
         out_context->l2_entry = nullptr;
         out_context->l3_entry = nullptr;
@@ -208,8 +217,9 @@ namespace ams::kern::arch::arm64 {
                 valid = this->ExtractL1Entry(out_entry, context, context->l1_entry, Null<KProcessAddress>);
             } else {
                 /* Invalid, end traversal. */
-                out_entry->phys_addr  = Null<KPhysicalAddress>;
+                out_entry->phys_addr        = Null<KPhysicalAddress>;
-                out_entry->block_size = L1BlockSize;
+                out_entry->block_size       = L1BlockSize;
+                out_entry->sw_reserved_bits = 0;
                 context->l1_entry = this->table + this->num_entries;
                 context->l2_entry = nullptr;
                 context->l3_entry = nullptr;

libraries/libmesosphere/source/arch/arm64/kern_k_thread_context.cpp

@@ -26,7 +26,7 @@ namespace ams::kern::arch::arm64 {
         /* Send KDebug event for this thread's creation. */
         {
             KScopedInterruptEnable ei;
-            KDebug::OnDebugEvent(ams::svc::DebugEvent_CreateThread, GetCurrentThread().GetId(), GetInteger(GetCurrentThread().GetThreadLocalRegionAddress()), GetCurrentThread().GetEntrypoint());
+            KDebug::OnDebugEvent(ams::svc::DebugEvent_CreateThread, GetCurrentThread().GetId(), GetInteger(GetCurrentThread().GetThreadLocalRegionAddress()));
         }
 
         /* Handle any pending dpc. */

libraries/libmesosphere/source/arch/arm64/svc/kern_svc_handlers_asm.s

@@ -32,7 +32,7 @@ _ZN3ams4kern4arch5arm6412SvcHandler64Ev:
     mrs     x9, elr_el1
     mrs     x10, spsr_el1
     mrs     x11, tpidr_el0
-    mrs     x18, tpidr_el1
+    ldr     x18, [sp, #(0x120 + 0x28)]
 
     /* Save callee-saved registers. */
     stp     x19, x20, [sp, #(8 * 19)]
@@ -66,7 +66,7 @@ _ZN3ams4kern4arch5arm6412SvcHandler64Ev:
     b.eq    3f
 
     /* Check if our disable count allows us to call SVCs. */
-    ldr     x10, [x18, #0x30]
+    mrs     x10, tpidrro_el0
     ldrh    w10, [x10, #0x100]
     cbz     w10, 1f
 
@@ -138,7 +138,7 @@ _ZN3ams4kern4arch5arm6412SvcHandler64Ev:
     stp     xzr, xzr, [sp, #(8 * 12)]
     stp     xzr, xzr, [sp, #(8 * 14)]
     stp     xzr, xzr, [sp, #(8 * 16)]
-    stp     xzr, x19, [sp, #(8 * 18)]
+    str          x19, [sp, #(8 * 19)]
     stp     x20, x21, [sp, #(8 * 20)]
     stp     x22, x23, [sp, #(8 * 22)]
     stp     x24, x25, [sp, #(8 * 24)]
@@ -146,7 +146,6 @@ _ZN3ams4kern4arch5arm6412SvcHandler64Ev:
     stp     x28, x29, [sp, #(8 * 28)]
 
     /* Call ams::kern::arch::arm64::HandleException(ams::kern::arch::arm64::KExceptionContext *) */
-    mrs     x18, tpidr_el1
     mov     x0,  sp
     bl      _ZN3ams4kern4arch5arm6415HandleExceptionEPNS2_17KExceptionContextE
 
@@ -246,7 +245,7 @@ _ZN3ams4kern4arch5arm6412SvcHandler32Ev:
     mrs     x17, elr_el1
     mrs     x20, spsr_el1
     mrs     x19, tpidr_el0
-    mrs     x18, tpidr_el1
+    ldr     x18, [sp, #(0x120 + 0x28)]
     stp     x17, x20, [sp, #(8 * 32)]
     str     x19,      [sp, #(8 * 34)]
 
@@ -276,7 +275,7 @@ _ZN3ams4kern4arch5arm6412SvcHandler32Ev:
     b.eq    3f
 
     /* Check if our disable count allows us to call SVCs. */
-    ldr     x15, [x18, #0x30]
+    mrs     x15, tpidrro_el0
     ldrh    w15, [x15, #0x100]
     cbz     w15, 1f
 
@@ -353,7 +352,6 @@ _ZN3ams4kern4arch5arm6412SvcHandler32Ev:
     stp     xzr, xzr, [sp, #(8 * 30)]
 
     /* Call ams::kern::arch::arm64::HandleException(ams::kern::arch::arm64::KExceptionContext *) */
-    mrs     x18, tpidr_el1
     mov     x0,  sp
     bl      _ZN3ams4kern4arch5arm6415HandleExceptionEPNS2_17KExceptionContextE
 

libraries/libmesosphere/source/arch/arm64/svc/kern_svc_tables.cpp

@@ -59,6 +59,7 @@ namespace ams::kern::svc {
 
         /* Set omit-frame-pointer to prevent GCC from emitting MOV X29, SP instructions. */
         #pragma GCC push_options
+        #pragma GCC optimize ("-O2")
         #pragma GCC optimize ("omit-frame-pointer")
 
             AMS_SVC_FOREACH_KERN_DEFINITION(DECLARE_SVC_STRUCT, _)

libraries/libmesosphere/source/board/nintendo/nx/kern_k_sleep_manager_asm.s

@@ -29,8 +29,8 @@
 .type       _ZN3ams4kern5board8nintendo2nx13KSleepManager15CpuSleepHandlerEmm, %function
 _ZN3ams4kern5board8nintendo2nx13KSleepManager15CpuSleepHandlerEmm:
     /* Save arguments. */
-    mov     x17, x0
+    mov     x16, x0
-    mov     x18, x1
+    mov     x17, x1
 
     /* Enable access to FPU registers. */
     mrs     x1, cpacr_el1
@@ -40,12 +40,13 @@ _ZN3ams4kern5board8nintendo2nx13KSleepManager15CpuSleepHandlerEmm:
     isb
 
     /* Save callee-save registers. */
-    stp     x19, x20, [x0], #0x10
+    stp     x18, x19, [x0], #0x10
-    stp     x21, x22, [x0], #0x10
+    stp     x20, x21, [x0], #0x10
-    stp     x23, x24, [x0], #0x10
+    stp     x22, x23, [x0], #0x10
-    stp     x25, x26, [x0], #0x10
+    stp     x24, x25, [x0], #0x10
-    stp     x27, x28, [x0], #0x10
+    stp     x26, x27, [x0], #0x10
-    stp     x29, x30, [x0], #0x10
+    stp     x28, x29, [x0], #0x10
+    stp     x30, xzr, [x0], #0x10
 
     /* Save stack pointer. */
     mov     x1, sp
@@ -113,8 +114,8 @@ _ZN3ams4kern5board8nintendo2nx13KSleepManager15CpuSleepHandlerEmm:
 1:  /* Suspend. */
     LOAD_IMMEDIATE_32(x0, 0xC4000001)
     LOAD_IMMEDIATE_32(x1, 0x0201001B)
-    mov     x2, x18
+    mov     x2, x17
-    mov     x3, x17
+    mov     x3, x16
     smc     #1
 0:  b       0b
 
@@ -190,12 +191,13 @@ _ZN3ams4kern5board8nintendo2nx13KSleepManager11ResumeEntryEm:
     isb
 
     /* Restore callee-save registers. */
-    ldp     x19, x20, [x0], #0x10
+    ldp     x18, x19, [x0], #0x10
-    ldp     x21, x22, [x0], #0x10
+    ldp     x20, x21, [x0], #0x10
-    ldp     x23, x24, [x0], #0x10
+    ldp     x22, x23, [x0], #0x10
-    ldp     x25, x26, [x0], #0x10
+    ldp     x24, x25, [x0], #0x10
-    ldp     x27, x28, [x0], #0x10
+    ldp     x26, x27, [x0], #0x10
-    ldp     x29, x30, [x0], #0x10
+    ldp     x28, x29, [x0], #0x10
+    ldp     x30, xzr, [x0], #0x10
 
     /* Restore stack pointer. */
     ldr     x1, [x0], #8
@@ -249,7 +251,6 @@ _ZN3ams4kern5board8nintendo2nx13KSleepManager11ResumeEntryEm:
 
     /* Set the global context back into x18/tpidr. */
     msr     tpidr_el1, x2
-    mov     x18, x2
     dsb     sy
     isb
 

libraries/libmesosphere/source/board/nintendo/nx/kern_k_system_control.cpp

@@ -182,6 +182,9 @@ namespace ams::kern::board::nintendo::nx {
             const KMemoryRegion *region = KMemoryLayout::Find(KPhysicalAddress(address));
             if (AMS_LIKELY(region != nullptr)) {
                 if (AMS_LIKELY(region->IsDerivedFrom(KMemoryRegionType_MemoryController))) {
+                    /* Check the region is valid. */
+                    MESOSPHERE_ABORT_UNLESS(region->GetEndAddress() != 0);
+
                     /* Get the offset within the region. */
                     const size_t offset = address - region->GetAddress();
                     MESOSPHERE_ABORT_UNLESS(offset < region->GetSize());
@@ -210,6 +213,9 @@ namespace ams::kern::board::nintendo::nx {
                     region->IsDerivedFrom(KMemoryRegionType_MemoryController0) ||
                     region->IsDerivedFrom(KMemoryRegionType_MemoryController1))
                 {
+                    /* Check the region is valid. */
+                    MESOSPHERE_ABORT_UNLESS(region->GetEndAddress() != 0);
+
                     /* Get the offset within the region. */
                     const size_t offset = address - region->GetAddress();
                     MESOSPHERE_ABORT_UNLESS(offset < region->GetSize());
@@ -449,6 +455,8 @@ namespace ams::kern::board::nintendo::nx {
         /* Configure the Kernel Carveout region. */
         {
             const auto carveout = KMemoryLayout::GetCarveoutRegionExtents();
+            MESOSPHERE_ABORT_UNLESS(carveout.GetEndAddress() != 0);
+
             smc::ConfigureCarveout(0, carveout.GetAddress(), carveout.GetSize());
         }
 
@@ -483,7 +491,7 @@ namespace ams::kern::board::nintendo::nx {
             MESOSPHERE_ABORT_UNLESS(Kernel::GetSystemResourceLimit().Reserve(ams::svc::LimitableResource_PhysicalMemoryMax, SecureAppletMemorySize));
 
             constexpr auto SecureAppletAllocateOption = KMemoryManager::EncodeOption(KMemoryManager::Pool_System, KMemoryManager::Direction_FromFront);
-            g_secure_applet_memory_address = Kernel::GetMemoryManager().AllocateContinuous(SecureAppletMemorySize / PageSize, 1, SecureAppletAllocateOption);
+            g_secure_applet_memory_address = Kernel::GetMemoryManager().AllocateAndOpenContinuous(SecureAppletMemorySize / PageSize, 1, SecureAppletAllocateOption);
             MESOSPHERE_ABORT_UNLESS(g_secure_applet_memory_address != Null<KVirtualAddress>);
         }
 
@@ -545,7 +553,7 @@ namespace ams::kern::board::nintendo::nx {
         if (arg != nullptr) {
             /* Get the address of the legacy IRAM region. */
             const KVirtualAddress iram_address = KMemoryLayout::GetDeviceVirtualAddress(KMemoryRegionType_LegacyLpsIram) + 64_KB;
-            constexpr size_t RebootPayloadSize = 0x2E000;
+            constexpr size_t RebootPayloadSize = 0x24000;
 
             /* NOTE: Atmosphere extension; if we received an exception context from Panic(), */
             /*       generate a fatal error report using it. */
@@ -570,11 +578,16 @@ namespace ams::kern::board::nintendo::nx {
             f_ctx->afsr0 = 0;
             f_ctx->afsr1 = GetVersionIdentifier();
 
+            /* Set efsr/far. */
+            f_ctx->far = cpu::GetFarEl1();
+            f_ctx->esr = cpu::GetEsrEl1();
+
             /* Copy registers. */
             for (size_t i = 0; i < util::size(e_ctx->x); ++i) {
                 f_ctx->gprs[i] = e_ctx->x[i];
             }
             f_ctx->sp = e_ctx->sp;
+            f_ctx->pc = cpu::GetElrEl1();
 
             /* Dump stack trace. */
             {
@@ -691,12 +704,9 @@ namespace ams::kern::board::nintendo::nx {
 
         /* Allocate the memory. */
         const size_t num_pages = size / PageSize;
-        const KVirtualAddress vaddr = Kernel::GetMemoryManager().AllocateContinuous(num_pages, alignment / PageSize, KMemoryManager::EncodeOption(static_cast<KMemoryManager::Pool>(pool), KMemoryManager::Direction_FromFront));
+        const KVirtualAddress vaddr = Kernel::GetMemoryManager().AllocateAndOpenContinuous(num_pages, alignment / PageSize, KMemoryManager::EncodeOption(static_cast<KMemoryManager::Pool>(pool), KMemoryManager::Direction_FromFront));
         R_UNLESS(vaddr != Null<KVirtualAddress>, svc::ResultOutOfMemory());
 
-        /* Open a reference to the memory. */
-        Kernel::GetMemoryManager().Open(vaddr, num_pages);
-
         /* Ensure we don't leak references to the memory on error. */
         auto mem_guard = SCOPE_GUARD { Kernel::GetMemoryManager().Close(vaddr, num_pages); };
 

libraries/libmesosphere/source/board/nintendo/nx/kern_secure_monitor.cpp

@@ -57,25 +57,31 @@ namespace ams::kern::board::nintendo::nx::smc {
             {
                 /* Disable interrupts while making the call. */
                 KScopedInterruptDisable intr_disable;
-                __asm__ __volatile__("smc #1"
-                                    : "+r"(x0), "+r"(x1), "+r"(x2), "+r"(x3), "+r"(x4), "+r"(x5), "+r"(x6), "+r"(x7)
-                                    :
-                                    : "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x18", "cc", "memory"
-                                    );
 
-                /* Restore the CoreLocalRegion into X18. */
+                {
-                cpu::SetCoreLocalRegionAddress(cpu::GetTpidrEl1());
+                    /* Backup the current thread pointer. */
+                    const uintptr_t current_thread_pointer_value = cpu::GetCurrentThreadPointerValue();
+
+                    __asm__ __volatile__("smc #1"
+                                        : "+r"(x0), "+r"(x1), "+r"(x2), "+r"(x3), "+r"(x4), "+r"(x5), "+r"(x6), "+r"(x7)
+                                        :
+                                        : "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x18", "cc", "memory"
+                                        );
+
+                    /* Restore the current thread pointer into X18. */
+                    cpu::SetCurrentThreadPointerValue(current_thread_pointer_value);
+
+                    /* Store arguments to output. */
+                    args.x[0] = x0;
+                    args.x[1] = x1;
+                    args.x[2] = x2;
+                    args.x[3] = x3;
+                    args.x[4] = x4;
+                    args.x[5] = x5;
+                    args.x[6] = x6;
+                    args.x[7] = x7;
+                }
             }
-
-            /* Store arguments to output. */
-            args.x[0] = x0;
-            args.x[1] = x1;
-            args.x[2] = x2;
-            args.x[3] = x3;
-            args.x[4] = x4;
-            args.x[5] = x5;
-            args.x[6] = x6;
-            args.x[7] = x7;
         }
 
         void CallUserSecureMonitorFunction(ams::svc::lp64::SecureMonitorArguments *args) {
@@ -93,25 +99,31 @@ namespace ams::kern::board::nintendo::nx::smc {
             {
                 /* Disable interrupts while making the call. */
                 KScopedInterruptDisable intr_disable;
-                __asm__ __volatile__("smc #0"
-                                    : "+r"(x0), "+r"(x1), "+r"(x2), "+r"(x3), "+r"(x4), "+r"(x5), "+r"(x6), "+r"(x7)
-                                    :
-                                    : "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x18", "cc", "memory"
-                                    );
 
-                /* Restore the CoreLocalRegion into X18. */
+                {
-                cpu::SetCoreLocalRegionAddress(cpu::GetTpidrEl1());
+                    /* Backup the current thread pointer. */
+                    const uintptr_t current_thread_pointer_value = cpu::GetCurrentThreadPointerValue();
+
+                    __asm__ __volatile__("smc #0"
+                                        : "+r"(x0), "+r"(x1), "+r"(x2), "+r"(x3), "+r"(x4), "+r"(x5), "+r"(x6), "+r"(x7)
+                                        :
+                                        : "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x18", "cc", "memory"
+                                        );
+
+                    /* Restore the current thread pointer into X18. */
+                    cpu::SetCurrentThreadPointerValue(current_thread_pointer_value);
+
+                    /* Store arguments to output. */
+                    args->r[0] = x0;
+                    args->r[1] = x1;
+                    args->r[2] = x2;
+                    args->r[3] = x3;
+                    args->r[4] = x4;
+                    args->r[5] = x5;
+                    args->r[6] = x6;
+                    args->r[7] = x7;
+                }
             }
-
-            /* Store arguments to output. */
-            args->r[0] = x0;
-            args->r[1] = x1;
-            args->r[2] = x2;
-            args->r[3] = x3;
-            args->r[4] = x4;
-            args->r[5] = x5;
-            args->r[6] = x6;
-            args->r[7] = x7;
         }
 
         void CallPrivilegedSecureMonitorFunctionForInit(SecureMonitorArguments &args) {

libraries/libmesosphere/source/init/kern_init_slab_setup.cpp

@@ -39,7 +39,9 @@ namespace ams::kern::init {
         HANDLER(KObjectName,         (SLAB_COUNT(KObjectName)),                                                 ## __VA_ARGS__) \
         HANDLER(KResourceLimit,      (SLAB_COUNT(KResourceLimit)),                                              ## __VA_ARGS__) \
         HANDLER(KEventInfo,          (SLAB_COUNT(KThread) + SLAB_COUNT(KDebug)),                                ## __VA_ARGS__) \
-        HANDLER(KDebug,              (SLAB_COUNT(KDebug)),                                                      ## __VA_ARGS__)
+        HANDLER(KDebug,              (SLAB_COUNT(KDebug)),                                                      ## __VA_ARGS__) \
+        HANDLER(KAlpha,              (SLAB_COUNT(KAlpha)),                                                      ## __VA_ARGS__) \
+        HANDLER(KBeta,               (SLAB_COUNT(KBeta)),                                                       ## __VA_ARGS__)
 
     namespace {
 
@@ -68,6 +70,8 @@ namespace ams::kern::init {
         constexpr size_t SlabCountKObjectName           = 7;
         constexpr size_t SlabCountKResourceLimit        = 5;
         constexpr size_t SlabCountKDebug                = cpu::NumCores;
+        constexpr size_t SlabCountKAlpha                = 1;
+        constexpr size_t SlabCountKBeta                 = 6;
 
         constexpr size_t SlabCountExtraKThread          = 160;
 
@@ -94,6 +98,8 @@ namespace ams::kern::init {
             .num_KObjectName            = SlabCountKObjectName,
             .num_KResourceLimit         = SlabCountKResourceLimit,
             .num_KDebug                 = SlabCountKDebug,
+            .num_KAlpha                 = SlabCountKAlpha,
+            .num_KBeta                  = SlabCountKBeta,
         };
 
         template<typename T>
@@ -158,12 +164,9 @@ namespace ams::kern::init {
 
         /* Allocate memory for the slab. */
         constexpr auto AllocateOption = KMemoryManager::EncodeOption(KMemoryManager::Pool_System, KMemoryManager::Direction_FromFront);
-        const KVirtualAddress slab_address = Kernel::GetMemoryManager().AllocateContinuous(num_pages, 1, AllocateOption);
+        const KVirtualAddress slab_address = Kernel::GetMemoryManager().AllocateAndOpenContinuous(num_pages, 1, AllocateOption);
         MESOSPHERE_ABORT_UNLESS(slab_address != Null<KVirtualAddress>);
 
-        /* Open references to the slab. */
-        Kernel::GetMemoryManager().Open(slab_address, num_pages);
-
         /* Initialize the slabheap. */
         KPageBuffer::InitializeSlabHeap(GetVoidPointer(slab_address), slab_size);
     }

libraries/libmesosphere/source/kern_debug_log_impl.board.nintendo_nx.cpp

@@ -51,37 +51,22 @@ namespace ams::kern {
     }
 
     bool KDebugLogImpl::Initialize() {
+        /* Get the uart memory region. */
+        const KMemoryRegion *uart_region = KMemoryLayout::GetPhysicalMemoryRegionTree().FindFirstDerived(KMemoryRegionType_Uart);
+        if (uart_region == nullptr) {
+            return false;
+        }
+
         /* Set the uart register base address. */
-        g_uart_address = KMemoryLayout::GetDeviceVirtualAddress(KMemoryRegionType_Uart);
+        g_uart_address = uart_region->GetPairAddress();
+        if (g_uart_address == Null<KVirtualAddress>) {
+            return false;
+        }
 
-        /* Parameters for uart. */
+        /* NOTE: We assume here that UART init/config has been done by the Secure Monitor. */
-        constexpr u32 BaudRate = 115200;
+        /* As such, we only need to disable interrupts. */
-        constexpr u32 Pllp = 408000000;
-        constexpr u32 Rate = 16 * BaudRate;
-        constexpr u32 Divisor = (Pllp + Rate / 2) / Rate;
-
-        /* Initialize the UART registers. */
-        /* Set Divisor Latch Access bit, to allow access to DLL/DLH */
-        WriteUartRegister(UartRegister_LCR, 0x80);
-        ReadUartRegister(UartRegister_LCR);
-
-        /* Program the divisor into DLL/DLH. */
-        WriteUartRegister(UartRegister_DLL, Divisor & 0xFF);
-        WriteUartRegister(UartRegister_DLH, (Divisor >> 8) & 0xFF);
-        ReadUartRegister(UartRegister_DLH);
-
-        /* Set word length to 3, clear Divisor Latch Access. */
-        WriteUartRegister(UartRegister_LCR, 0x03);
-        ReadUartRegister(UartRegister_LCR);
-
-        /* Disable UART interrupts. */
         WriteUartRegister(UartRegister_IER, 0x00);
 
-        /* Configure the FIFO to be enabled and clear receive. */
-        WriteUartRegister(UartRegister_FCR, 0x03);
-        WriteUartRegister(UartRegister_IRDA_CSR, 0x02);
-        ReadUartRegister(UartRegister_FCR);
-
         return true;
     }
 

libraries/libmesosphere/source/kern_initial_process.cpp

@@ -26,7 +26,9 @@ namespace ams::kern {
         };
 
         KVirtualAddress GetInitialProcessBinaryAddress() {
-            return KMemoryLayout::GetPageTableHeapRegion().GetEndAddress() - InitialProcessBinarySizeMax;
+            const uintptr_t end_address = KMemoryLayout::GetPageTableHeapRegion().GetEndAddress();
+            MESOSPHERE_ABORT_UNLESS(end_address != 0);
+            return end_address - InitialProcessBinarySizeMax;
         }
 
         void LoadInitialProcessBinaryHeader(InitialProcessBinaryHeader *header) {
@@ -91,14 +93,17 @@ namespace ams::kern {
                     /* Allocate memory for the process. */
                     auto &mm = Kernel::GetMemoryManager();
                     const auto pool = reader.UsesSecureMemory() ? secure_pool : unsafe_pool;
-                    MESOSPHERE_R_ABORT_UNLESS(mm.Allocate(std::addressof(pg), params.code_num_pages, KMemoryManager::EncodeOption(pool, KMemoryManager::Direction_FromFront)));
+                    MESOSPHERE_R_ABORT_UNLESS(mm.AllocateAndOpen(std::addressof(pg), params.code_num_pages, KMemoryManager::EncodeOption(pool, KMemoryManager::Direction_FromFront)));
 
                     {
                         /* Ensure that we do not leak pages. */
-                        KScopedPageGroup spg(pg);
+                        ON_SCOPE_EXIT { pg.Close(); };
+
+                        /* Get the temporary region. */
+                        const auto &temp_region = KMemoryLayout::GetTempRegion();
+                        MESOSPHERE_ABORT_UNLESS(temp_region.GetEndAddress() != 0);
 
                         /* Map the process's memory into the temporary region. */
-                        const auto &temp_region = KMemoryLayout::GetTempRegion();
                         KProcessAddress temp_address = Null<KProcessAddress>;
                         MESOSPHERE_R_ABORT_UNLESS(Kernel::GetKernelPageTable().MapPageGroup(std::addressof(temp_address), pg, temp_region.GetAddress(), temp_region.GetSize() / PageSize, KMemoryState_Kernel, KMemoryPermission_KernelReadWrite));
 
@@ -170,9 +175,8 @@ namespace ams::kern {
             /* Allocate memory for the image. */
             const KMemoryManager::Pool pool = static_cast<KMemoryManager::Pool>(KSystemControl::GetCreateProcessMemoryPool());
             const auto allocate_option = KMemoryManager::EncodeOption(pool, KMemoryManager::Direction_FromFront);
-            KVirtualAddress allocated_memory = mm.AllocateContinuous(num_pages, 1, allocate_option);
+            KVirtualAddress allocated_memory = mm.AllocateAndOpenContinuous(num_pages, 1, allocate_option);
             MESOSPHERE_ABORT_UNLESS(allocated_memory != Null<KVirtualAddress>);
-            mm.Open(allocated_memory, num_pages);
 
             /* Relocate the image. */
             std::memmove(GetVoidPointer(allocated_memory), GetVoidPointer(GetInitialProcessBinaryAddress()), g_initial_process_binary_header.size);

libraries/libmesosphere/source/kern_k_address_arbiter.cpp

@@ -50,9 +50,8 @@ namespace ams::kern {
         s32 num_waiters = 0;
         {
             KScopedSchedulerLock sl;
-            g_cv_arbiter_compare_thread.SetupForAddressArbiterCompare(addr, -1);
 
-            auto it = this->tree.nfind(g_cv_arbiter_compare_thread);
+            auto it = this->tree.nfind_light({ addr, -1 });
             while ((it != this->tree.end()) && (count <= 0 || num_waiters < count) && (it->GetAddressArbiterKey() == addr)) {
                 KThread *target_thread = std::addressof(*it);
                 target_thread->SetSyncedObject(nullptr, ResultSuccess());
@@ -79,10 +78,7 @@ namespace ams::kern {
             R_UNLESS(UpdateIfEqual(std::addressof(user_value), addr, value, value + 1), svc::ResultInvalidCurrentMemory());
             R_UNLESS(user_value == value,                                               svc::ResultInvalidState());
 
-            g_cv_arbiter_compare_thread.SetupForAddressArbiterCompare(addr, -1);
+            auto it = this->tree.nfind_light({ addr, -1 });
-
-            auto it = this->tree.nfind(g_cv_arbiter_compare_thread);
-
             while ((it != this->tree.end()) && (count <= 0 || num_waiters < count) && (it->GetAddressArbiterKey() == addr)) {
                 KThread *target_thread = std::addressof(*it);
                 target_thread->SetSyncedObject(nullptr, ResultSuccess());
@@ -103,10 +99,8 @@ namespace ams::kern {
         s32 num_waiters = 0;
         {
             KScopedSchedulerLock sl;
-            g_cv_arbiter_compare_thread.SetupForAddressArbiterCompare(addr, -1);
-
-            auto it = this->tree.nfind(g_cv_arbiter_compare_thread);
 
+            auto it = this->tree.nfind_light({ addr, -1 });
             /* Determine the updated value. */
             s32 new_value;
             if (GetTargetFirmware() >= TargetFirmware_7_0_0) {

libraries/libmesosphere/source/kern_k_auto_object_container.cpp

@@ -18,24 +18,20 @@
 namespace ams::kern {
 
 
-    Result KAutoObjectWithListContainer::Register(KAutoObjectWithList *obj) {
+    void KAutoObjectWithListContainer::Register(KAutoObjectWithList *obj) {
         MESOSPHERE_ASSERT_THIS();
 
         KScopedLightLock lk(this->lock);
 
         this->object_list.insert(*obj);
-
-        return ResultSuccess();
     }
 
-    Result KAutoObjectWithListContainer::Unregister(KAutoObjectWithList *obj) {
+    void KAutoObjectWithListContainer::Unregister(KAutoObjectWithList *obj) {
         MESOSPHERE_ASSERT_THIS();
 
         KScopedLightLock lk(this->lock);
 
         this->object_list.erase(this->object_list.iterator_to(*obj));
-
-        return ams::svc::ResultNotFound();
     }
 
     size_t KAutoObjectWithListContainer::GetOwnedCount(KProcess *owner) {

libraries/libmesosphere/source/kern_k_condition_variable.cpp

@@ -17,8 +17,6 @@
 
 namespace ams::kern {
 
-    constinit KThread g_cv_arbiter_compare_thread;
-
     namespace {
 
         ALWAYS_INLINE bool ReadFromUser(u32 *out, KProcessAddress address) {
@@ -179,9 +177,8 @@ namespace ams::kern {
         int num_waiters = 0;
         {
             KScopedSchedulerLock sl;
-            g_cv_arbiter_compare_thread.SetupForConditionVariableCompare(cv_key, -1);
 
-            auto it = this->tree.nfind(g_cv_arbiter_compare_thread);
+            auto it = this->tree.nfind_light({ cv_key, -1 });
             while ((it != this->tree.end()) && (count <= 0 || num_waiters < count) && (it->GetConditionVariableKey() == cv_key)) {
                 KThread *target_thread = std::addressof(*it);
 
@@ -197,6 +194,12 @@ namespace ams::kern {
                 target_thread->ClearConditionVariable();
                 ++num_waiters;
             }
+
+            /* If we have no waiters, clear the has waiter flag. */
+            if (it == this->tree.end() || it->GetConditionVariableKey() != cv_key) {
+                const u32 has_waiter_flag = 0;
+                WriteToUser(cv_key, std::addressof(has_waiter_flag));
+            }
         }
 
         /* Close threads in the array. */
@@ -247,6 +250,13 @@ namespace ams::kern {
                     next_owner_thread->Wakeup();
                 }
 
+                /* Write to the cv key. */
+                {
+                    const u32 has_waiter_flag = 1;
+                    WriteToUser(key, std::addressof(has_waiter_flag));
+                    cpu::DataMemoryBarrier();
+                }
+
                 /* Write the value to userspace. */
                 if (!WriteToUser(addr, std::addressof(next_value))) {
                     slp.CancelSleep();

libraries/libmesosphere/source/kern_k_debug_base.cpp

@@ -327,7 +327,7 @@ namespace ams::kern {
                             it->SetDebugAttached();
 
                             /* Send the event. */
-                            this->PushDebugEvent(ams::svc::DebugEvent_CreateThread, it->GetId(), GetInteger(it->GetThreadLocalRegionAddress()), it->GetEntrypoint());
+                            this->PushDebugEvent(ams::svc::DebugEvent_CreateThread, it->GetId(), GetInteger(it->GetThreadLocalRegionAddress()));
                         }
                     }
                 }
@@ -496,7 +496,7 @@ namespace ams::kern {
             if (thread->GetRawState() != KThread::ThreadState_Runnable) {
                 bool current = false;
                 for (auto i = 0; i < static_cast<s32>(cpu::NumCores); ++i) {
-                    if (thread == Kernel::GetCurrentContext(i).current_thread) {
+                    if (thread == Kernel::GetScheduler(i).GetSchedulerCurrentThread()) {
                         current = true;
                     }
                     break;
@@ -543,7 +543,7 @@ namespace ams::kern {
             if (thread->GetRawState() != KThread::ThreadState_Runnable) {
                 bool current = false;
                 for (auto i = 0; i < static_cast<s32>(cpu::NumCores); ++i) {
-                    if (thread == Kernel::GetCurrentContext(i).current_thread) {
+                    if (thread == Kernel::GetScheduler(i).GetSchedulerCurrentThread()) {
                         current = true;
                     }
                     break;
@@ -682,7 +682,6 @@ namespace ams::kern {
                         /* Set the thread creation info. */
                         info->info.create_thread.thread_id   = param0;
                         info->info.create_thread.tls_address = param1;
-                        info->info.create_thread.entrypoint  = param2;
                     }
                     break;
                 case ams::svc::DebugEvent_ExitProcess:
@@ -842,7 +841,6 @@ namespace ams::kern {
                 {
                     out->info.create_thread.thread_id   = info->info.create_thread.thread_id;
                     out->info.create_thread.tls_address = info->info.create_thread.tls_address;
-                    out->info.create_thread.entrypoint  = info->info.create_thread.entrypoint;
                 }
                 break;
             case ams::svc::DebugEvent_ExitProcess:

libraries/libmesosphere/source/kern_k_device_address_space.cpp

@@ -79,25 +79,30 @@ namespace ams::kern {
         ON_SCOPE_EXIT { pg.Close(); };
 
         /* Ensure that if we fail, we don't keep unmapped pages locked. */
-        ON_SCOPE_EXIT {
+        auto unlock_guard = SCOPE_GUARD { MESOSPHERE_R_ABORT_UNLESS(page_table->UnlockForDeviceAddressSpace(process_address, size)); };
-            if (*out_mapped_size != size) {
-                page_table->UnlockForDeviceAddressSpace(process_address + *out_mapped_size, size - *out_mapped_size);
-            };
-        };
 
         /* Map the pages. */
         {
             /* Clear the output size to zero on failure. */
-            auto map_guard = SCOPE_GUARD { *out_mapped_size = 0; };
+            auto mapped_size_guard = SCOPE_GUARD { *out_mapped_size = 0; };
 
             /* Perform the mapping. */
             R_TRY(this->table.Map(out_mapped_size, pg, device_address, device_perm, refresh_mappings));
 
-            /* We succeeded, so cancel our guard. */
+            /* Ensure that we unmap the pages if we fail to update the protections. */
+            /* NOTE: Nintendo does not check the result of this unmap call. */
+            auto map_guard = SCOPE_GUARD { this->table.Unmap(device_address, *out_mapped_size); };
+
+            /* Update the protections in accordance with how much we mapped. */
+            R_TRY(page_table->UnlockForDeviceAddressSpacePartialMap(process_address, size, *out_mapped_size));
+
+            /* We succeeded, so cancel our guards. */
             map_guard.Cancel();
+            mapped_size_guard.Cancel();
         }
 
-
+        /* We succeeded, so we don't need to unlock our pages. */
+        unlock_guard.Cancel();
         return ResultSuccess();
     }
 
@@ -110,19 +115,23 @@ namespace ams::kern {
 
         /* Make and open a page group for the unmapped region. */
         KPageGroup pg(page_table->GetBlockInfoManager());
-        R_TRY(page_table->MakeAndOpenPageGroupContiguous(std::addressof(pg), process_address, size / PageSize,
+        R_TRY(page_table->MakePageGroupForUnmapDeviceAddressSpace(std::addressof(pg), process_address, size));
-                                                         KMemoryState_FlagCanDeviceMap, KMemoryState_FlagCanDeviceMap,
-                                                         KMemoryPermission_None, KMemoryPermission_None,
-                                                         KMemoryAttribute_AnyLocked | KMemoryAttribute_DeviceShared | KMemoryAttribute_Locked, KMemoryAttribute_DeviceShared));
 
         /* Ensure the page group is closed on scope exit. */
         ON_SCOPE_EXIT { pg.Close(); };
 
-        /* Unmap. */
+        /* If we fail to unmap, we want to do a partial unlock. */
-        R_TRY(this->table.Unmap(pg, device_address));
+        {
+            auto unlock_guard = SCOPE_GUARD { page_table->UnlockForDeviceAddressSpacePartialMap(process_address, size, size); };
+
+            /* Unmap. */
+            R_TRY(this->table.Unmap(pg, device_address));
+
+            unlock_guard.Cancel();
+        }
 
         /* Unlock the pages. */
-        R_TRY(page_table->UnlockForDeviceAddressSpace(process_address, size));
+        MESOSPHERE_R_ABORT_UNLESS(page_table->UnlockForDeviceAddressSpace(process_address, size));
 
         return ResultSuccess();
     }

libraries/libmesosphere/source/kern_k_initial_process_reader.cpp

@@ -140,6 +140,9 @@ namespace ams::kern {
             out->flags |= ams::svc::CreateProcessFlag_AddressSpace32Bit;
         }
 
+        /* All initial processes should disable device address space merge. */
+        out->flags |= ams::svc::CreateProcessFlag_DisableDeviceAddressSpaceMerge;
+
         return ResultSuccess();
     }
 

libraries/libmesosphere/source/kern_k_interrupt_event.cpp

@@ -34,7 +34,7 @@ namespace ams::kern {
         KReadableEvent::Initialize(nullptr);
 
         /* Try to register the task. */
-        R_TRY(KInterruptEventTask::Register(std::addressof(this->task), this->interrupt_id, type == ams::svc::InterruptType_Level, this));
+        R_TRY(KInterruptEventTask::Register(this->interrupt_id, type == ams::svc::InterruptType_Level, this));
 
         /* Mark initialized. */
         this->is_initialized = true;
@@ -44,15 +44,17 @@ namespace ams::kern {
     void KInterruptEvent::Finalize() {
         MESOSPHERE_ASSERT_THIS();
 
-        MESOSPHERE_ASSERT(this->task != nullptr);
+        g_interrupt_event_task_table[this->interrupt_id]->Unregister(this->interrupt_id);
-        this->task->Unregister();
+
+        /* Perform inherited finalization. */
+        KAutoObjectWithSlabHeapAndContainer<KInterruptEvent, KReadableEvent>::Finalize();
     }
 
     Result KInterruptEvent::Reset() {
         MESOSPHERE_ASSERT_THIS();
 
-        /* Lock the task table. */
+        /* Lock the task. */
-        KScopedLightLock lk(g_interrupt_event_lock);
+        KScopedLightLock lk(g_interrupt_event_task_table[this->interrupt_id]->GetLock());
 
         /* Clear the event. */
         R_TRY(KReadableEvent::Reset());
@@ -63,7 +65,7 @@ namespace ams::kern {
         return ResultSuccess();
     }
 
-    Result KInterruptEventTask::Register(KInterruptEventTask **out, s32 interrupt_id, bool level, KInterruptEvent *event) {
+    Result KInterruptEventTask::Register(s32 interrupt_id, bool level, KInterruptEvent *event) {
         /* Verify the interrupt id is defined and global. */
         R_UNLESS(Kernel::GetInterruptManager().IsInterruptDefined(interrupt_id), svc::ResultOutOfRange());
         R_UNLESS(Kernel::GetInterruptManager().IsGlobal(interrupt_id),           svc::ResultOutOfRange());
@@ -85,45 +87,47 @@ namespace ams::kern {
             allocated = true;
         }
 
+        /* Ensure that the task is cleaned up if anything goes wrong. */
+        auto task_guard = SCOPE_GUARD { if (allocated) { KInterruptEventTask::Free(task); } };
+
         /* Register/bind the interrupt task. */
         {
-            /* Ensure that the task is cleaned up if anything goes wrong. */
+            /* Acqquire exclusive access to the task. */
-            auto task_guard = SCOPE_GUARD { if (allocated) { KInterruptEventTask::Free(task); } };
+            KScopedLightLock tlk(task->lock);
 
             /* Bind the interrupt handler. */
             R_TRY(Kernel::GetInterruptManager().BindHandler(task, interrupt_id, GetCurrentCoreId(), KInterruptController::PriorityLevel_High, true, level));
 
-            /* We successfully registered, so we don't need to free the task. */
+            /* Set the event. */
-            task_guard.Cancel();
+            task->event = event;
         }
 
-        /* Set the event. */
-        task->event = event;
-
         /* If we allocated, set the event in the table. */
         if (allocated) {
-            task->interrupt_id = interrupt_id;
             g_interrupt_event_task_table[interrupt_id] = task;
         }
 
-        /* Set the output. */
+        /* We successfully registered, so we don't need to free the task. */
-        *out = task;
+        task_guard.Cancel();
         return ResultSuccess();
     }
 
-    void KInterruptEventTask::Unregister() {
+    void KInterruptEventTask::Unregister(s32 interrupt_id) {
         MESOSPHERE_ASSERT_THIS();
 
         /* Lock the task table. */
         KScopedLightLock lk(g_interrupt_event_lock);
 
+        /* Lock the task. */
+        KScopedLightLock tlk(this->lock);
+
         /* Ensure we can unregister. */
-        MESOSPHERE_ABORT_UNLESS(g_interrupt_event_task_table[this->interrupt_id] == this);
+        MESOSPHERE_ABORT_UNLESS(g_interrupt_event_task_table[interrupt_id] == this);
         MESOSPHERE_ABORT_UNLESS(this->event != nullptr);
-        this->event = nullptr;
 
         /* Unbind the interrupt. */
-        Kernel::GetInterruptManager().UnbindHandler(this->interrupt_id, GetCurrentCoreId());
+        this->event = nullptr;
+        Kernel::GetInterruptManager().UnbindHandler(interrupt_id, GetCurrentCoreId());
     }
 
     KInterruptTask *KInterruptEventTask::OnInterrupt(s32 interrupt_id) {
@@ -136,7 +140,7 @@ namespace ams::kern {
         MESOSPHERE_ASSERT_THIS();
 
         /* Lock the task table. */
-        KScopedLightLock lk(g_interrupt_event_lock);
+        KScopedLightLock lk(this->lock);
 
         if (this->event != nullptr) {
             this->event->Signal();

libraries/libmesosphere/source/kern_k_light_server_session.cpp

@@ -74,7 +74,7 @@ namespace ams::kern {
             /* If we can reply, do so. */
             if (!this->current_request->IsTerminationRequested()) {
                 MESOSPHERE_ASSERT(this->current_request->GetState() == KThread::ThreadState_Waiting);
-                MESOSPHERE_ASSERT(this->current_request == this->request_queue.GetFront());
+                MESOSPHERE_ASSERT(this->request_queue.begin() != this->request_queue.end() && this->current_request == std::addressof(*this->request_queue.begin()));
                 std::memcpy(this->current_request->GetLightSessionData(), server_thread->GetLightSessionData(), KLightSession::DataSize);
                 this->request_queue.WakeupThread(this->current_request);
             }
@@ -110,8 +110,8 @@ namespace ams::kern {
             R_UNLESS(!this->parent->IsServerClosed(), svc::ResultSessionClosed());
 
             /* If we have a request available, use it. */
-            if (this->current_request == nullptr && this->request_queue.IsEmpty()) {
+            if (this->current_request == nullptr && !this->request_queue.IsEmpty()) {
-                this->current_request = this->request_queue.GetFront();
+                this->current_request = std::addressof(*this->request_queue.begin());
                 this->current_request->Open();
                 this->server_thread   = server_thread;
                 break;
@@ -148,7 +148,7 @@ namespace ams::kern {
                 /* Reply to the current request. */
                 if (!this->current_request->IsTerminationRequested()) {
                     MESOSPHERE_ASSERT(this->current_request->GetState() == KThread::ThreadState_Waiting);
-                    MESOSPHERE_ASSERT(this->current_request == this->request_queue.GetFront());
+                    MESOSPHERE_ASSERT(this->request_queue.begin() != this->request_queue.end() && this->current_request == std::addressof(*this->request_queue.begin()));
                     this->request_queue.WakeupThread(this->current_request);
                     this->current_request->SetSyncedObject(nullptr, svc::ResultSessionClosed());
                 }

libraries/libmesosphere/source/kern_k_memory_block_manager.cpp

@@ -147,7 +147,35 @@ namespace ams::kern {
         return Null<KProcessAddress>;
     }
 
-    void KMemoryBlockManager::Update(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages, KMemoryState state, KMemoryPermission perm, KMemoryAttribute attr) {
+    void KMemoryBlockManager::CoalesceForUpdate(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages) {
+        /* Find the iterator now that we've updated. */
+        iterator it = this->FindIterator(address);
+        if (address != this->start_address) {
+            it--;
+        }
+
+        /* Coalesce blocks that we can. */
+        while (true) {
+            iterator prev = it++;
+            if (it == this->memory_block_tree.end()) {
+                break;
+            }
+
+            if (prev->CanMergeWith(*it)) {
+                KMemoryBlock *block = std::addressof(*it);
+                this->memory_block_tree.erase(it);
+                prev->Add(*block);
+                allocator->Free(block);
+                it = prev;
+            }
+
+            if (address + num_pages * PageSize < it->GetMemoryInfo().GetEndAddress()) {
+                break;
+            }
+        }
+    }
+
+    void KMemoryBlockManager::Update(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages, KMemoryState state, KMemoryPermission perm, KMemoryAttribute attr, KMemoryBlockDisableMergeAttribute set_disable_attr, KMemoryBlockDisableMergeAttribute clear_disable_attr) {
         /* Ensure for auditing that we never end up with an invalid tree. */
         KScopedMemoryBlockManagerAuditor auditor(this);
         MESOSPHERE_ASSERT(util::IsAligned(GetInteger(address), PageSize));
@@ -193,39 +221,14 @@ namespace ams::kern {
                 }
 
                 /* Update block state. */
-                it->Update(state, perm, attr);
+                it->Update(state, perm, attr, cur_address == address, set_disable_attr, clear_disable_attr);
                 cur_address += cur_info.GetSize();
                 remaining_pages -= cur_info.GetNumPages();
             }
             it++;
         }
 
-        /* Find the iterator now that we've updated. */
+        this->CoalesceForUpdate(allocator, address, num_pages);
-        it = this->FindIterator(address);
-        if (address != this->start_address) {
-            it--;
-        }
-
-        /* Coalesce blocks that we can. */
-        while (true) {
-            iterator prev = it++;
-            if (it == this->memory_block_tree.end()) {
-                break;
-            }
-
-            if (prev->HasSameProperties(*it)) {
-                KMemoryBlock *block = std::addressof(*it);
-                const size_t pages = it->GetNumPages();
-                this->memory_block_tree.erase(it);
-                allocator->Free(block);
-                prev->Add(pages);
-                it = prev;
-            }
-
-            if (address + num_pages * PageSize < it->GetMemoryInfo().GetEndAddress()) {
-                break;
-            }
-        }
     }
 
     void KMemoryBlockManager::UpdateIfMatch(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages, KMemoryState test_state, KMemoryPermission test_perm, KMemoryAttribute test_attr, KMemoryState state, KMemoryPermission perm, KMemoryAttribute attr) {
@@ -265,7 +268,7 @@ namespace ams::kern {
                 }
 
                 /* Update block state. */
-                it->Update(state, perm, attr);
+                it->Update(state, perm, attr, false, KMemoryBlockDisableMergeAttribute_None, KMemoryBlockDisableMergeAttribute_None);
                 cur_address     += cur_info.GetSize();
                 remaining_pages -= cur_info.GetNumPages();
             } else {
@@ -281,35 +284,10 @@ namespace ams::kern {
             it++;
         }
 
-        /* Find the iterator now that we've updated. */
+        this->CoalesceForUpdate(allocator, address, num_pages);
-        it = this->FindIterator(address);
-        if (address != this->start_address) {
-            it--;
-        }
-
-        /* Coalesce blocks that we can. */
-        while (true) {
-            iterator prev = it++;
-            if (it == this->memory_block_tree.end()) {
-                break;
-            }
-
-            if (prev->HasSameProperties(*it)) {
-                KMemoryBlock *block = std::addressof(*it);
-                const size_t pages = it->GetNumPages();
-                this->memory_block_tree.erase(it);
-                allocator->Free(block);
-                prev->Add(pages);
-                it = prev;
-            }
-
-            if (address + num_pages * PageSize < it->GetMemoryInfo().GetEndAddress()) {
-                break;
-            }
-        }
     }
 
-    void KMemoryBlockManager::UpdateLock(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages, void (KMemoryBlock::*lock_func)(KMemoryPermission new_perm), KMemoryPermission perm) {
+    void KMemoryBlockManager::UpdateLock(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages, MemoryBlockLockFunction lock_func, KMemoryPermission perm) {
         /* Ensure for auditing that we never end up with an invalid tree. */
         KScopedMemoryBlockManagerAuditor auditor(this);
         MESOSPHERE_ASSERT(util::IsAligned(GetInteger(address), PageSize));
@@ -317,8 +295,8 @@ namespace ams::kern {
         KProcessAddress cur_address = address;
         size_t remaining_pages = num_pages;
         iterator it   = this->FindIterator(address);
-        iterator prev = it, next = it;
+
-        bool check_coalesce_prev = false, check_coalesce_next = false;
+        const KProcessAddress end_address = address + (num_pages * PageSize);
 
         while (remaining_pages > 0) {
             const size_t remaining_size = remaining_pages * PageSize;
@@ -334,10 +312,6 @@ namespace ams::kern {
 
                 cur_info = it->GetMemoryInfo();
                 cur_address = cur_info.GetAddress();
-            } else if (cur_address == address && cur_address != this->start_address) {
-                /* If there's a previous, we should check for coalescing. */
-                check_coalesce_prev = true;
-                prev--;
             }
 
             if (cur_info.GetSize() > remaining_size) {
@@ -348,47 +322,16 @@ namespace ams::kern {
                 it = this->memory_block_tree.insert(*new_block);
 
                 cur_info = it->GetMemoryInfo();
-            } else if (cur_info.GetSize() == remaining_size) {
-                /* Otherwise if we can map precisely, we may need to check for coalescing against next block. */
-                next = it;
-                ++next;
-                if (next != this->memory_block_tree.end()) {
-                    check_coalesce_next = true;
-                }
             }
 
             /* Call the locked update function. */
-            (std::addressof(*it)->*lock_func)(perm);
+            (std::addressof(*it)->*lock_func)(perm, cur_info.GetAddress() == address, cur_info.GetEndAddress() == end_address);
             cur_address += cur_info.GetSize();
             remaining_pages -= cur_info.GetNumPages();
             it++;
         }
 
-        /* If we should try to coalesce prev, do so. */
+        this->CoalesceForUpdate(allocator, address, num_pages);
-        if (check_coalesce_prev) {
-            it = prev;
-            it++;
-            if (prev->HasSameProperties(*it)) {
-                KMemoryBlock *block = std::addressof(*it);
-                const size_t pages = it->GetNumPages();
-                this->memory_block_tree.erase(it);
-                allocator->Free(block);
-                prev->Add(pages);
-            }
-        }
-
-        /* If we should try to coalesce next, do so. */
-        if (check_coalesce_next) {
-            it = next;
-            it--;
-            if (it->HasSameProperties(*next)) {
-                KMemoryBlock *block = std::addressof(*next);
-                const size_t pages = next->GetNumPages();
-                this->memory_block_tree.erase(next);
-                allocator->Free(block);
-                it->Add(pages);
-            }
-        }
     }
 
     /* Debug. */
@@ -403,8 +346,8 @@ namespace ams::kern {
             const KMemoryInfo prev_info = prev->GetMemoryInfo();
             const KMemoryInfo cur_info  = it->GetMemoryInfo();
 
-            /* Sequential blocks with same properties should be coalesced. */
+            /* Sequential blocks which can be merged should be merged. */
-            if (prev->HasSameProperties(*it)) {
+            if (prev->CanMergeWith(*it)) {
                 return false;
             }
 

libraries/libmesosphere/source/kern_k_memory_layout.board.nintendo_nx.cpp

@@ -55,6 +55,7 @@ namespace ams::kern {
             MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetPhysicalMemoryRegionTree().Insert(start, size, phys_type, attr));
             const KMemoryRegion *phys = KMemoryLayout::GetPhysicalMemoryRegionTree().FindByTypeAndAttribute(phys_type, attr);
             MESOSPHERE_INIT_ABORT_UNLESS(phys != nullptr);
+            MESOSPHERE_INIT_ABORT_UNLESS(phys->GetEndAddress() != 0);
             MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetVirtualMemoryRegionTree().Insert(phys->GetPairAddress(), size, virt_type, attr));
         }
 
@@ -104,6 +105,7 @@ namespace ams::kern {
         void SetupPoolPartitionMemoryRegions() {
             /* Start by identifying the extents of the DRAM memory region. */
             const auto dram_extents = KMemoryLayout::GetMainMemoryPhysicalExtents();
+            MESOSPHERE_INIT_ABORT_UNLESS(dram_extents.GetEndAddress() != 0);
 
             /* Determine the end of the pool region. */
             const uintptr_t pool_end = dram_extents.GetEndAddress() - KTraceBufferSize;

libraries/libmesosphere/source/kern_k_memory_layout.cpp

@@ -23,7 +23,7 @@ namespace ams::kern {
             NON_COPYABLE(KMemoryRegionAllocator);
             NON_MOVEABLE(KMemoryRegionAllocator);
             public:
-                static constexpr size_t MaxMemoryRegions = 1000;
+                static constexpr size_t MaxMemoryRegions = 200;
             private:
                 KMemoryRegion region_heap[MaxMemoryRegions];
                 size_t num_regions;
@@ -40,8 +40,6 @@ namespace ams::kern {
                     new (region) KMemoryRegion(std::forward<Args>(args)...);
 
                     return region;
-
-                    return &this->region_heap[this->num_regions++];
                 }
         };
 
@@ -55,8 +53,8 @@ namespace ams::kern {
 
     }
 
-    void KMemoryRegionTree::InsertDirectly(uintptr_t address, size_t size, u32 attr, u32 type_id) {
+    void KMemoryRegionTree::InsertDirectly(uintptr_t address, uintptr_t last_address, u32 attr, u32 type_id) {
-        this->insert(*AllocateRegion(address, size, attr, type_id));
+        this->insert(*AllocateRegion(address, last_address, attr, type_id));
     }
 
     bool KMemoryRegionTree::Insert(uintptr_t address, size_t size, u32 type_id, u32 new_attr, u32 old_attr) {
@@ -82,9 +80,7 @@ namespace ams::kern {
 
         /* Cache information from the region before we remove it. */
         const uintptr_t old_address = found->GetAddress();
-        const size_t    old_size    = found->GetSize();
+        const uintptr_t old_last    = found->GetLastAddress();
-        const uintptr_t old_end     = old_address + old_size;
-        const uintptr_t old_last    = old_end - 1;
         const uintptr_t old_pair    = found->GetPairAddress();
         const u32       old_type    = found->GetType();
 
@@ -92,24 +88,24 @@ namespace ams::kern {
         this->erase(this->iterator_to(*found));
 
         /* Insert the new region into the tree. */
-        const uintptr_t new_pair = (old_pair != std::numeric_limits<uintptr_t>::max()) ? old_pair + (address - old_address) : old_pair;
         if (old_address == address) {
             /* Reuse the old object for the new region, if we can. */
-            found->Reset(address, size, new_pair, new_attr, type_id);
+            found->Reset(address, inserted_region_last, old_pair, new_attr, type_id);
             this->insert(*found);
         } else {
             /* If we can't re-use, adjust the old region. */
-            found->Reset(old_address, address - old_address, old_pair, old_attr, old_type);
+            found->Reset(old_address, address - 1, old_pair, old_attr, old_type);
             this->insert(*found);
 
             /* Insert a new region for the split. */
-            this->insert(*AllocateRegion(address, size, new_pair, new_attr, type_id));
+            const uintptr_t new_pair = (old_pair != std::numeric_limits<uintptr_t>::max()) ? old_pair + (address - old_address) : old_pair;
+            this->insert(*AllocateRegion(address, inserted_region_last, new_pair, new_attr, type_id));
         }
 
         /* If we need to insert a region after the region, do so. */
         if (old_last != inserted_region_last) {
             const uintptr_t after_pair = (old_pair != std::numeric_limits<uintptr_t>::max()) ? old_pair + (inserted_region_end - old_address) : old_pair;
-            this->insert(*AllocateRegion(inserted_region_end, old_end - inserted_region_end, after_pair, old_attr, old_type));
+            this->insert(*AllocateRegion(inserted_region_end, old_last, after_pair, old_attr, old_type));
         }
 
         return true;
@@ -125,8 +121,11 @@ namespace ams::kern {
         const uintptr_t first_address = extents.GetAddress();
         const uintptr_t last_address  = extents.GetLastAddress();
 
+        const uintptr_t first_index = first_address / alignment;
+        const uintptr_t last_index  = last_address / alignment;
+
         while (true) {
-            const uintptr_t candidate = util::AlignDown(KSystemControl::Init::GenerateRandomRange(first_address, last_address), alignment);
+            const uintptr_t candidate = KSystemControl::Init::GenerateRandomRange(first_index, last_index) * alignment;
 
             /* Ensure that the candidate doesn't overflow with the size. */
             if (!(candidate < candidate + size)) {
@@ -157,13 +156,13 @@ namespace ams::kern {
         /* Initialize linear trees. */
         for (auto &region : GetPhysicalMemoryRegionTree()) {
             if (region.HasTypeAttribute(KMemoryRegionAttr_LinearMapped)) {
-                GetPhysicalLinearMemoryRegionTree().InsertDirectly(region.GetAddress(), region.GetSize(), region.GetAttributes(), region.GetType());
+                GetPhysicalLinearMemoryRegionTree().InsertDirectly(region.GetAddress(), region.GetLastAddress(), region.GetAttributes(), region.GetType());
             }
         }
 
         for (auto &region : GetVirtualMemoryRegionTree()) {
             if (region.IsDerivedFrom(KMemoryRegionType_Dram)) {
-                GetVirtualLinearMemoryRegionTree().InsertDirectly(region.GetAddress(), region.GetSize(), region.GetAttributes(), region.GetType());
+                GetVirtualLinearMemoryRegionTree().InsertDirectly(region.GetAddress(), region.GetLastAddress(), region.GetAttributes(), region.GetType());
             }
         }
     }
@@ -181,89 +180,4 @@ namespace ams::kern {
         return resource_region_size;
     }
 
-    namespace init {
-
-        namespace {
-
-            constexpr PageTableEntry KernelRwDataAttribute(PageTableEntry::Permission_KernelRW, PageTableEntry::PageAttribute_NormalMemory, PageTableEntry::Shareable_InnerShareable, PageTableEntry::MappingFlag_Mapped);
-
-            constexpr size_t CoreLocalRegionAlign          = PageSize;
-            constexpr size_t CoreLocalRegionSize           = PageSize * (1 + cpu::NumCores);
-            constexpr size_t CoreLocalRegionSizeWithGuards = CoreLocalRegionSize + 2 * PageSize;
-            constexpr size_t CoreLocalRegionBoundsAlign    = 1_GB;
-            static_assert(CoreLocalRegionSize == sizeof(KCoreLocalRegion));
-
-            KVirtualAddress GetCoreLocalRegionVirtualAddress() {
-                while (true) {
-                    const uintptr_t candidate_start = GetInteger(KMemoryLayout::GetVirtualMemoryRegionTree().GetRandomAlignedRegion(CoreLocalRegionSizeWithGuards, CoreLocalRegionAlign, KMemoryRegionType_None));
-                    const uintptr_t candidate_end   = candidate_start + CoreLocalRegionSizeWithGuards;
-                    const uintptr_t candidate_last  = candidate_end - 1;
-
-                    const auto &containing_region = *KMemoryLayout::GetVirtualMemoryRegionTree().Find(candidate_start);
-
-                    if (candidate_last > containing_region.GetLastAddress()) {
-                        continue;
-                    }
-
-                    if (containing_region.GetType() != KMemoryRegionType_None) {
-                        continue;
-                    }
-
-                    if (util::AlignDown(candidate_start, CoreLocalRegionBoundsAlign) != util::AlignDown(candidate_last, CoreLocalRegionBoundsAlign)) {
-                        continue;
-                    }
-
-                    if (containing_region.GetAddress() > util::AlignDown(candidate_start, CoreLocalRegionBoundsAlign)) {
-                        continue;
-                    }
-
-                    if (util::AlignUp(candidate_last, CoreLocalRegionBoundsAlign) - 1 > containing_region.GetLastAddress()) {
-                        continue;
-                    }
-
-                    return candidate_start + PageSize;
-                }
-
-            }
-
-        }
-
-        void SetupCoreLocalRegionMemoryRegions(KInitialPageTable &page_table, KInitialPageAllocator &page_allocator) {
-            /* NOTE: Nintendo passes page table here to use num_l1_entries; we don't use this at present. */
-            MESOSPHERE_UNUSED(page_table);
-
-            /* Get the virtual address of the core local reigon. */
-            const KVirtualAddress core_local_virt_start = GetCoreLocalRegionVirtualAddress();
-            MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetVirtualMemoryRegionTree().Insert(GetInteger(core_local_virt_start), CoreLocalRegionSize, KMemoryRegionType_CoreLocalRegion));
-
-            /* Allocate a page for each core. */
-            KPhysicalAddress core_local_region_start_phys[cpu::NumCores] = {};
-            for (size_t i = 0; i < cpu::NumCores; i++) {
-                core_local_region_start_phys[i] = page_allocator.Allocate();
-            }
-
-            /* Allocate an l1 page table for each core. */
-            KPhysicalAddress core_l1_ttbr1_phys[cpu::NumCores] = {};
-            core_l1_ttbr1_phys[0] = util::AlignDown(cpu::GetTtbr1El1(), PageSize);
-            for (size_t i = 1; i < cpu::NumCores; i++) {
-                core_l1_ttbr1_phys[i] = page_allocator.Allocate();
-                std::memcpy(reinterpret_cast<void *>(GetInteger(core_l1_ttbr1_phys[i])), reinterpret_cast<void *>(GetInteger(core_l1_ttbr1_phys[0])), PageSize);
-            }
-
-            /* Use the l1 page table for each core to map the core local region for each core. */
-            for (size_t i = 0; i < cpu::NumCores; i++) {
-                KInitialPageTable temp_pt(core_l1_ttbr1_phys[i], KInitialPageTable::NoClear{});
-                temp_pt.Map(core_local_virt_start, PageSize, core_local_region_start_phys[i], KernelRwDataAttribute, page_allocator);
-                for (size_t j = 0; j < cpu::NumCores; j++) {
-                    temp_pt.Map(core_local_virt_start + (j + 1) * PageSize, PageSize, core_local_region_start_phys[j], KernelRwDataAttribute, page_allocator);
-                }
-
-                /* Setup the InitArguments. */
-                SetInitArguments(static_cast<s32>(i), core_local_region_start_phys[i], GetInteger(core_l1_ttbr1_phys[i]));
-            }
-        }
-
-    }
-
-
 }

libraries/libmesosphere/source/kern_k_memory_manager.cpp

@@ -20,12 +20,16 @@ namespace ams::kern {
     namespace {
 
         constexpr KMemoryManager::Pool GetPoolFromMemoryRegionType(u32 type) {
-            switch (type) {
+            if ((type | KMemoryRegionType_VirtualDramApplicationPool) == type) {
-                case KMemoryRegionType_VirtualDramApplicationPool:     return KMemoryManager::Pool_Application;
+                return KMemoryManager::Pool_Application;
-                case KMemoryRegionType_VirtualDramAppletPool:          return KMemoryManager::Pool_Applet;
+            } else if ((type | KMemoryRegionType_VirtualDramAppletPool) == type) {
-                case KMemoryRegionType_VirtualDramSystemPool:          return KMemoryManager::Pool_System;
+                return KMemoryManager::Pool_Applet;
-                case KMemoryRegionType_VirtualDramSystemNonSecurePool: return KMemoryManager::Pool_SystemNonSecure;
+            } else if ((type | KMemoryRegionType_VirtualDramSystemPool) == type) {
-                MESOSPHERE_UNREACHABLE_DEFAULT_CASE();
+                return KMemoryManager::Pool_System;
+            } else if ((type | KMemoryRegionType_VirtualDramSystemNonSecurePool) == type) {
+                return KMemoryManager::Pool_SystemNonSecure;
+            } else {
+                MESOSPHERE_PANIC("InvalidMemoryRegionType for conversion to Pool");
             }
         }
 
@@ -37,9 +41,11 @@ namespace ams::kern {
         std::memset(GetVoidPointer(management_region), 0, management_region_size);
 
         /* Traverse the virtual memory layout tree, initializing each manager as appropriate. */
-        while (true) {
+        while (this->num_managers != MaxManagerCount) {
             /* Locate the region that should initialize the current manager. */
-            const KMemoryRegion *region = nullptr;
+            uintptr_t region_address = 0;
+            size_t region_size = 0;
+            Pool region_pool = Pool_Count;
             for (const auto &it : KMemoryLayout::GetVirtualMemoryRegionTree()) {
                 /* We only care about regions that we need to create managers for. */
                 if (!it.IsDerivedFrom(KMemoryRegionType_VirtualDramUserPool)) {
@@ -51,39 +57,62 @@ namespace ams::kern {
                     continue;
                 }
 
-                region = std::addressof(it);
+                /* Validate the region. */
-                break;
+                MESOSPHERE_ABORT_UNLESS(it.GetEndAddress() != 0);
+                MESOSPHERE_ASSERT(it.GetAddress() != Null<decltype(it.GetAddress())>);
+                MESOSPHERE_ASSERT(it.GetSize()    > 0);
+
+                /* Update the region's extents. */
+                if (region_address == 0) {
+                    region_address = it.GetAddress();
+                    region_size    = it.GetSize();
+                    region_pool    = GetPoolFromMemoryRegionType(it.GetType());
+                } else {
+                    MESOSPHERE_ASSERT(it.GetAddress() == region_address + region_size);
+
+                    /* Update the size. */
+                    region_size = it.GetEndAddress() - region_address;
+                    MESOSPHERE_ABORT_UNLESS(GetPoolFromMemoryRegionType(it.GetType()) == region_pool);
+                }
             }
 
-            /* If we didn't find a region, then we're done initializing managers. */
+            /* If we didn't find a region, we're done. */
-            if (region == nullptr) {
+            if (region_size == 0) {
                 break;
             }
 
-            /* Ensure that the region is correct. */
-            MESOSPHERE_ASSERT(region->GetAddress() != Null<decltype(region->GetAddress())>);
-            MESOSPHERE_ASSERT(region->GetSize()    > 0);
-            MESOSPHERE_ASSERT(region->GetEndAddress() >= region->GetAddress());
-            MESOSPHERE_ASSERT(region->IsDerivedFrom(KMemoryRegionType_VirtualDramUserPool));
-            MESOSPHERE_ASSERT(region->GetAttributes() == this->num_managers);
-
             /* Initialize a new manager for the region. */
-            const Pool pool = GetPoolFromMemoryRegionType(region->GetType());
             Impl *manager = std::addressof(this->managers[this->num_managers++]);
             MESOSPHERE_ABORT_UNLESS(this->num_managers <= util::size(this->managers));
 
-            const size_t cur_size = manager->Initialize(region, pool, management_region, management_region_end);
+            const size_t cur_size = manager->Initialize(region_address, region_size, management_region, management_region_end, region_pool);
             management_region += cur_size;
             MESOSPHERE_ABORT_UNLESS(management_region <= management_region_end);
 
             /* Insert the manager into the pool list. */
-            if (this->pool_managers_tail[pool] == nullptr) {
+            if (this->pool_managers_tail[region_pool] == nullptr) {
-                this->pool_managers_head[pool] = manager;
+                this->pool_managers_head[region_pool] = manager;
             } else {
-                this->pool_managers_tail[pool]->SetNext(manager);
+                this->pool_managers_tail[region_pool]->SetNext(manager);
-                manager->SetPrev(this->pool_managers_tail[pool]);
+                manager->SetPrev(this->pool_managers_tail[region_pool]);
             }
-            this->pool_managers_tail[pool] = manager;
+            this->pool_managers_tail[region_pool] = manager;
+        }
+
+        /* Free each region to its corresponding heap. */
+        for (const auto &it : KMemoryLayout::GetVirtualMemoryRegionTree()) {
+            if (it.IsDerivedFrom(KMemoryRegionType_VirtualDramUserPool)) {
+                /* Check the region. */
+                MESOSPHERE_ABORT_UNLESS(it.GetEndAddress() != 0);
+
+                /* Free the memory to the heap. */
+                this->managers[it.GetAttributes()].Free(it.GetAddress(), it.GetSize() / PageSize);
+            }
+        }
+
+        /* Update the used size for all managers. */
+        for (size_t i = 0; i < this->num_managers; ++i) {
+            this->managers[i].UpdateUsedHeapSize();
         }
     }
 
@@ -117,7 +146,7 @@ namespace ams::kern {
     }
 
 
-    KVirtualAddress KMemoryManager::AllocateContinuous(size_t num_pages, size_t align_pages, u32 option) {
+    KVirtualAddress KMemoryManager::AllocateAndOpenContinuous(size_t num_pages, size_t align_pages, u32 option) {
         /* Early return if we're allocating no pages. */
         if (num_pages == 0) {
             return Null<KVirtualAddress>;
@@ -156,6 +185,9 @@ namespace ams::kern {
             chosen_manager->TrackUnoptimizedAllocation(allocated_block, num_pages);
         }
 
+        /* Open the first reference to the pages. */
+        chosen_manager->OpenFirst(allocated_block, num_pages);
+
         return allocated_block;
     }
 
@@ -210,7 +242,7 @@ namespace ams::kern {
         return ResultSuccess();
     }
 
-    Result KMemoryManager::Allocate(KPageGroup *out, size_t num_pages, u32 option) {
+    Result KMemoryManager::AllocateAndOpen(KPageGroup *out, size_t num_pages, u32 option) {
         MESOSPHERE_ASSERT(out != nullptr);
         MESOSPHERE_ASSERT(out->GetNumPages() == 0);
 
@@ -222,10 +254,30 @@ namespace ams::kern {
         KScopedLightLock lk(this->pool_locks[pool]);
 
         /* Allocate the page group. */
-        return this->AllocatePageGroupImpl(out, num_pages, pool, dir, this->has_optimized_process[pool], true);
+        R_TRY(this->AllocatePageGroupImpl(out, num_pages, pool, dir, this->has_optimized_process[pool], true));
+
+        /* Open the first reference to the pages. */
+        for (const auto &block : *out) {
+            KVirtualAddress cur_address = block.GetAddress();
+            size_t remaining_pages      = block.GetNumPages();
+            while (remaining_pages > 0) {
+                /* Get the manager for the current address. */
+                auto &manager = this->GetManager(cur_address);
+
+                /* Process part or all of the block. */
+                const size_t cur_pages = std::min(remaining_pages, manager.GetPageOffsetToEnd(cur_address));
+                manager.OpenFirst(cur_address, cur_pages);
+
+                /* Advance. */
+                cur_address     += cur_pages * PageSize;
+                remaining_pages -= cur_pages;
+            }
+        }
+
+        return ResultSuccess();
     }
 
-    Result KMemoryManager::AllocateForProcess(KPageGroup *out, size_t num_pages, u32 option, u64 process_id, u8 fill_pattern) {
+    Result KMemoryManager::AllocateAndOpenForProcess(KPageGroup *out, size_t num_pages, u32 option, u64 process_id, u8 fill_pattern) {
         MESOSPHERE_ASSERT(out != nullptr);
         MESOSPHERE_ASSERT(out->GetNumPages() == 0);
 
@@ -247,6 +299,24 @@ namespace ams::kern {
 
             /* Set whether we should optimize. */
             optimized = has_optimized && is_optimized;
+
+            /* Open the first reference to the pages. */
+            for (const auto &block : *out) {
+                KVirtualAddress cur_address = block.GetAddress();
+                size_t remaining_pages      = block.GetNumPages();
+                while (remaining_pages > 0) {
+                    /* Get the manager for the current address. */
+                    auto &manager = this->GetManager(cur_address);
+
+                    /* Process part or all of the block. */
+                    const size_t cur_pages = std::min(remaining_pages, manager.GetPageOffsetToEnd(cur_address));
+                    manager.OpenFirst(cur_address, cur_pages);
+
+                    /* Advance. */
+                    cur_address     += cur_pages * PageSize;
+                    remaining_pages -= cur_pages;
+                }
+            }
         }
 
         /* Perform optimized memory tracking, if we should. */
@@ -313,12 +383,12 @@ namespace ams::kern {
         return ResultSuccess();
     }
 
-    size_t KMemoryManager::Impl::Initialize(const KMemoryRegion *region, Pool p, KVirtualAddress management, KVirtualAddress management_end) {
+    size_t KMemoryManager::Impl::Initialize(uintptr_t address, size_t size, KVirtualAddress management, KVirtualAddress management_end, Pool p) {
         /* Calculate management sizes. */
-        const size_t ref_count_size      = (region->GetSize() / PageSize) * sizeof(u16);
+        const size_t ref_count_size      = (size / PageSize) * sizeof(u16);
-        const size_t optimize_map_size   = CalculateOptimizedProcessOverheadSize(region->GetSize());
+        const size_t optimize_map_size   = CalculateOptimizedProcessOverheadSize(size);
         const size_t manager_size        = util::AlignUp(optimize_map_size + ref_count_size, PageSize);
-        const size_t page_heap_size      = KPageHeap::CalculateManagementOverheadSize(region->GetSize());
+        const size_t page_heap_size      = KPageHeap::CalculateManagementOverheadSize(size);
         const size_t total_management_size = manager_size + page_heap_size;
         MESOSPHERE_ABORT_UNLESS(manager_size <= total_management_size);
         MESOSPHERE_ABORT_UNLESS(management + total_management_size <= management_end);
@@ -331,13 +401,7 @@ namespace ams::kern {
         MESOSPHERE_ABORT_UNLESS(util::IsAligned(GetInteger(this->management_region), PageSize));
 
         /* Initialize the manager's KPageHeap. */
-        this->heap.Initialize(region->GetAddress(), region->GetSize(), management + manager_size, page_heap_size);
+        this->heap.Initialize(address, size, management + manager_size, page_heap_size);
-
-        /* Free the memory to the heap. */
-        this->heap.Free(region->GetAddress(), region->GetSize() / PageSize);
-
-        /* Update the heap's used size. */
-        this->heap.UpdateUsedSize();
 
         return total_management_size;
     }

libraries/libmesosphere/source/kern_k_process.cpp

@@ -78,11 +78,6 @@ namespace ams::kern {
     }
 
     void KProcess::Finalize() {
-        /* Ensure we're not executing on any core. */
-        for (size_t i = 0; i < cpu::NumCores; ++i) {
-            MESOSPHERE_ASSERT(Kernel::GetCurrentContext(static_cast<s32>(i)).current_process.load(std::memory_order_relaxed) != this);
-        }
-
         /* Delete the process local region. */
         this->DeleteThreadLocalRegion(this->plr_address);
 
@@ -134,6 +129,17 @@ namespace ams::kern {
             }
         }
 
+        /* Close all references to our betas. */
+        {
+            auto it = this->beta_list.begin();
+            while (it != this->beta_list.end()) {
+                KBeta *beta = std::addressof(*it);
+                it = this->beta_list.erase(it);
+
+                beta->Close();
+            }
+        }
+
         /* Our thread local page list must be empty at this point. */
         MESOSPHERE_ABORT_UNLESS(this->partially_used_tlp_tree.empty());
         MESOSPHERE_ABORT_UNLESS(this->fully_used_tlp_tree.empty());
@@ -237,13 +243,14 @@ namespace ams::kern {
         /* NOTE: Nintendo passes process ID despite not having set it yet. */
         /* This goes completely unused, but even so... */
         {
-            const auto as_type       = static_cast<ams::svc::CreateProcessFlag>(params.flags & ams::svc::CreateProcessFlag_AddressSpaceMask);
+            const auto as_type          = static_cast<ams::svc::CreateProcessFlag>(params.flags & ams::svc::CreateProcessFlag_AddressSpaceMask);
-            const bool enable_aslr   = (params.flags & ams::svc::CreateProcessFlag_EnableAslr);
+            const bool enable_aslr      = (params.flags & ams::svc::CreateProcessFlag_EnableAslr) != 0;
-            const bool is_app        = (params.flags & ams::svc::CreateProcessFlag_IsApplication);
+            const bool enable_das_merge = (params.flags & ams::svc::CreateProcessFlag_DisableDeviceAddressSpaceMerge) == 0;
-            auto *mem_block_manager  = std::addressof(is_app ? Kernel::GetApplicationMemoryBlockManager() : Kernel::GetSystemMemoryBlockManager());
+            const bool is_app           = (params.flags & ams::svc::CreateProcessFlag_IsApplication) != 0;
-            auto *block_info_manager = std::addressof(Kernel::GetBlockInfoManager());
+            auto *mem_block_manager     = std::addressof(is_app ? Kernel::GetApplicationMemoryBlockManager() : Kernel::GetSystemMemoryBlockManager());
-            auto *pt_manager         = std::addressof(Kernel::GetPageTableManager());
+            auto *block_info_manager    = std::addressof(Kernel::GetBlockInfoManager());
-            R_TRY(this->page_table.Initialize(this->process_id, as_type, enable_aslr, !enable_aslr, pool, params.code_address, params.code_num_pages * PageSize, mem_block_manager, block_info_manager, pt_manager));
+            auto *pt_manager            = std::addressof(Kernel::GetPageTableManager());
+            R_TRY(this->page_table.Initialize(this->process_id, as_type, enable_aslr, enable_das_merge, !enable_aslr, pool, params.code_address, params.code_num_pages * PageSize, mem_block_manager, block_info_manager, pt_manager));
         }
         auto pt_guard = SCOPE_GUARD { this->page_table.Finalize(); };
 
@@ -344,9 +351,10 @@ namespace ams::kern {
         /* NOTE: Nintendo passes process ID despite not having set it yet. */
         /* This goes completely unused, but even so... */
         {
-            const auto as_type     = static_cast<ams::svc::CreateProcessFlag>(params.flags & ams::svc::CreateProcessFlag_AddressSpaceMask);
+            const auto as_type          = static_cast<ams::svc::CreateProcessFlag>(params.flags & ams::svc::CreateProcessFlag_AddressSpaceMask);
-            const bool enable_aslr = (params.flags & ams::svc::CreateProcessFlag_EnableAslr);
+            const bool enable_aslr      = (params.flags & ams::svc::CreateProcessFlag_EnableAslr) != 0;
-            R_TRY(this->page_table.Initialize(this->process_id, as_type, enable_aslr, !enable_aslr, pool, params.code_address, code_size, mem_block_manager, block_info_manager, pt_manager));
+            const bool enable_das_merge = (params.flags & ams::svc::CreateProcessFlag_DisableDeviceAddressSpaceMerge) == 0;
+            R_TRY(this->page_table.Initialize(this->process_id, as_type, enable_aslr, enable_das_merge, !enable_aslr, pool, params.code_address, code_size, mem_block_manager, block_info_manager, pt_manager));
         }
         auto pt_guard = SCOPE_GUARD { this->page_table.Finalize(); };
 
@@ -925,7 +933,7 @@ namespace ams::kern {
         mem_reservation.Commit();
 
         /* Note for debug that we're running a new process. */
-        MESOSPHERE_LOG("KProcess::Run() pid=%ld name=%-12s thread=%ld affinity=0x%lx ideal_core=%d active_core=%d\n", this->process_id, this->name, main_thread->GetId(), main_thread->GetAffinityMask().GetAffinityMask(), main_thread->GetIdealCore(), main_thread->GetActiveCore());
+        MESOSPHERE_LOG("KProcess::Run() pid=%ld name=%-12s thread=%ld affinity=0x%lx ideal_core=%d active_core=%d\n", this->process_id, this->name, main_thread->GetId(), main_thread->GetVirtualAffinityMask(), main_thread->GetIdealVirtualCore(), main_thread->GetActiveCore());
 
         return ResultSuccess();
     }

libraries/libmesosphere/source/kern_k_resource_limit.cpp

@@ -32,6 +32,7 @@ namespace ams::kern {
                 this->limit_values[i]    = 0;
                 this->current_values[i]  = 0;
                 this->current_hints[i]   = 0;
+                this->peak_values[i]     = 0;
             }
         */
     }
@@ -70,6 +71,21 @@ namespace ams::kern {
         return value;
     }
 
+    s64 KResourceLimit::GetPeakValue(ams::svc::LimitableResource which) const {
+        MESOSPHERE_ASSERT_THIS();
+
+        s64 value;
+        {
+            KScopedLightLock lk(this->lock);
+            value = this->peak_values[which];
+            MESOSPHERE_ASSERT(value >= 0);
+            MESOSPHERE_ASSERT(this->current_values[which] <= this->limit_values[which]);
+            MESOSPHERE_ASSERT(this->current_hints[which]  <= this->current_values[which]);
+        }
+
+        return value;
+    }
+
     s64 KResourceLimit::GetFreeValue(ams::svc::LimitableResource which) const {
         MESOSPHERE_ASSERT_THIS();
 
@@ -124,6 +140,7 @@ namespace ams::kern {
             if (this->current_values[which] + value <= this->limit_values[which]) {
                 this->current_values[which] += value;
                 this->current_hints[which]  += value;
+                this->peak_values[which]    = std::max(this->peak_values[which], this->current_values[which]);
                 return true;
             }
 

libraries/libmesosphere/source/kern_k_scheduler.cpp

@@ -66,6 +66,9 @@ namespace ams::kern {
 
         /* Bind interrupt handler. */
         Kernel::GetInterruptManager().BindHandler(GetSchedulerInterruptTask(), KInterruptName_Scheduler, this->core_id, KInterruptController::PriorityLevel_Scheduler, false, false);
+
+        /* Set the current thread. */
+        this->current_thread = GetCurrentThreadPointer();
     }
 
     void KScheduler::Activate() {
@@ -259,18 +262,21 @@ namespace ams::kern {
 
         MESOSPHERE_KTRACE_THREAD_SWITCH(next_thread);
 
+        if (next_thread->GetCurrentCore() != this->core_id) {
+            next_thread->SetCurrentCore(this->core_id);
+        }
+
         /* Switch the current process, if we're switching processes. */
         if (KProcess *next_process = next_thread->GetOwnerProcess(); next_process != cur_process) {
-            /* MESOSPHERE_LOG("!!! PROCESS SWITCH !!! %s -> %s\n", cur_process != nullptr ? cur_process->GetName() : nullptr, next_process != nullptr ? next_process->GetName() : nullptr); */
             KProcess::Switch(cur_process, next_process);
         }
 
         /* Set the new thread. */
         SetCurrentThread(next_thread);
+        this->current_thread = next_thread;
 
         /* Set the new Thread Local region. */
         cpu::SwitchThreadLocalRegion(GetInteger(next_thread->GetThreadLocalRegionAddress()));
-        SetCurrentThreadLocalRegion(next_thread->GetThreadLocalRegionHeapAddress());
     }
 
     void KScheduler::ClearPreviousThread(KThread *thread) {

libraries/libmesosphere/source/kern_k_server_session.cpp

@@ -17,6 +17,9 @@
 
 namespace ams::kern {
 
+    #pragma GCC push_options
+    #pragma GCC optimize ("-O3")
+
     namespace ipc {
 
         using MessageBuffer = ams::svc::ipc::MessageBuffer;
@@ -257,7 +260,7 @@ namespace ams::kern {
                     R_TRY(src_page_table.CopyMemoryFromHeapToHeapWithoutCheckDestination(dst_page_table, recv_pointer, recv_size,
                                                                                          KMemoryState_FlagReferenceCounted, KMemoryState_FlagReferenceCounted,
                                                                                          static_cast<KMemoryPermission>(KMemoryPermission_NotMapped | KMemoryPermission_KernelReadWrite),
-                                                                                         KMemoryAttribute_AnyLocked | KMemoryAttribute_Uncached | KMemoryAttribute_Locked, KMemoryAttribute_AnyLocked | KMemoryAttribute_Locked,
+                                                                                         KMemoryAttribute_Uncached | KMemoryAttribute_Locked, KMemoryAttribute_Locked,
                                                                                          src_pointer,
                                                                                          KMemoryState_FlagReferenceCounted, KMemoryState_FlagReferenceCounted,
                                                                                          KMemoryPermission_UserRead,
@@ -291,15 +294,15 @@ namespace ams::kern {
             switch (state) {
                 case KMemoryState_Ipc:
                     out_state     = KMemoryState_FlagCanUseIpc;
-                    out_attr_mask = KMemoryAttribute_AnyLocked | KMemoryAttribute_Uncached | KMemoryAttribute_DeviceShared | KMemoryAttribute_Locked;
+                    out_attr_mask = KMemoryAttribute_Uncached | KMemoryAttribute_DeviceShared | KMemoryAttribute_Locked;
                     break;
                 case KMemoryState_NonSecureIpc:
                     out_state     = KMemoryState_FlagCanUseNonSecureIpc;
-                    out_attr_mask = KMemoryAttribute_AnyLocked | KMemoryAttribute_Uncached | KMemoryAttribute_Locked;
+                    out_attr_mask = KMemoryAttribute_Uncached | KMemoryAttribute_Locked;
                     break;
                 case KMemoryState_NonDeviceIpc:
                     out_state     = KMemoryState_FlagCanUseNonDeviceIpc;
-                    out_attr_mask = KMemoryAttribute_AnyLocked | KMemoryAttribute_Uncached | KMemoryAttribute_Locked;
+                    out_attr_mask = KMemoryAttribute_Uncached | KMemoryAttribute_Locked;
                     break;
                 default:
                     return svc::ResultInvalidCombination();
@@ -654,7 +657,7 @@ namespace ams::kern {
                         R_TRY(src_page_table.CopyMemoryFromHeapToHeap(dst_page_table, dst_message_buffer + max_fast_size, raw_size - fast_size,
                                                                       KMemoryState_FlagReferenceCounted, KMemoryState_FlagReferenceCounted,
                                                                       static_cast<KMemoryPermission>(KMemoryPermission_NotMapped | KMemoryPermission_KernelReadWrite),
-                                                                      KMemoryAttribute_AnyLocked | KMemoryAttribute_Uncached | KMemoryAttribute_Locked, KMemoryAttribute_AnyLocked | KMemoryAttribute_Locked,
+                                                                      KMemoryAttribute_Uncached | KMemoryAttribute_Locked, KMemoryAttribute_Locked,
                                                                       src_message_buffer + max_fast_size,
                                                                       KMemoryState_FlagReferenceCounted, KMemoryState_FlagReferenceCounted,
                                                                       src_perm,
@@ -911,7 +914,7 @@ namespace ams::kern {
                                                                       src_message_buffer + max_fast_size,
                                                                       KMemoryState_FlagReferenceCounted, KMemoryState_FlagReferenceCounted,
                                                                       static_cast<KMemoryPermission>(KMemoryPermission_NotMapped | KMemoryPermission_KernelRead),
-                                                                      KMemoryAttribute_AnyLocked | KMemoryAttribute_Uncached | KMemoryAttribute_Locked, KMemoryAttribute_AnyLocked | KMemoryAttribute_Locked));
+                                                                      KMemoryAttribute_Uncached | KMemoryAttribute_Locked, KMemoryAttribute_Locked));
                     }
                 } else /* if (dst_user) */ {
                     /* The destination is a user buffer, so it should be unmapped + readable. */
@@ -1359,7 +1362,9 @@ namespace ams::kern {
         }
 
         /* Notify. */
-        this->NotifyAbort(svc::ResultSessionClosed());
+        this->NotifyAvailable(svc::ResultSessionClosed());
     }
 
+    #pragma GCC pop_options
+
 }

libraries/libmesosphere/source/kern_k_shared_memory.cpp

@@ -37,7 +37,7 @@ namespace ams::kern {
         R_UNLESS(memory_reservation.Succeeded(), svc::ResultLimitReached());
 
         /* Allocate the memory. */
-        R_TRY(Kernel::GetMemoryManager().Allocate(std::addressof(this->page_group), num_pages, owner->GetAllocateOption()));
+        R_TRY(Kernel::GetMemoryManager().AllocateAndOpen(std::addressof(this->page_group), num_pages, owner->GetAllocateOption()));
 
         /* Commit our reservation. */
         memory_reservation.Commit();
@@ -46,9 +46,6 @@ namespace ams::kern {
         this->resource_limit = reslimit;
         this->resource_limit->Open();
 
-        /* Open the memory. */
-        this->page_group.Open();
-
         /* Mark initialized. */
         this->is_initialized = true;
 

libraries/libmesosphere/source/kern_k_synchronization.cpp

@@ -1,139 +0,0 @@
-/*
- * Copyright (c) 2018-2020 Atmosphère-NX
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-#include <mesosphere.hpp>
-
-namespace ams::kern {
-
-    Result KSynchronization::Wait(s32 *out_index, KSynchronizationObject **objects, const s32 num_objects, s64 timeout) {
-        MESOSPHERE_ASSERT_THIS();
-
-        /* Allocate space on stack for thread iterators. */
-        KSynchronizationObject::iterator *thread_iters = static_cast<KSynchronizationObject::iterator *>(__builtin_alloca(sizeof(KSynchronizationObject::iterator) * num_objects));
-
-        /* Prepare for wait. */
-        KThread *thread = GetCurrentThreadPointer();
-        s32 sync_index = -1;
-        KHardwareTimer *timer;
-
-        {
-            /* Setup the scheduling lock and sleep. */
-            KScopedSchedulerLockAndSleep slp(std::addressof(timer), thread, timeout);
-
-            /* Check if any of the objects are already signaled. */
-            for (auto i = 0; i < num_objects; ++i) {
-                AMS_ASSERT(objects[i] != nullptr);
-
-                if (objects[i]->IsSignaled()) {
-                    *out_index = i;
-                    slp.CancelSleep();
-                    return ResultSuccess();
-                }
-            }
-
-            /* Check if the timeout is zero. */
-            if (timeout == 0) {
-                slp.CancelSleep();
-                return svc::ResultTimedOut();
-            }
-
-            /* Check if the thread should terminate. */
-            if (thread->IsTerminationRequested()) {
-                slp.CancelSleep();
-                return svc::ResultTerminationRequested();
-            }
-
-            /* Check if waiting was canceled. */
-            if (thread->IsWaitCancelled()) {
-                slp.CancelSleep();
-                thread->ClearWaitCancelled();
-                return svc::ResultCancelled();
-            }
-
-            /* Add the waiters. */
-            for (auto i = 0; i < num_objects; ++i) {
-                thread_iters[i] = objects[i]->RegisterWaitingThread(thread);
-            }
-
-            /* Mark the thread as waiting. */
-            thread->SetCancellable();
-            thread->SetSyncedObject(nullptr, svc::ResultTimedOut());
-            thread->SetState(KThread::ThreadState_Waiting);
-        }
-
-        /* The lock/sleep is done, so we should be able to get our result. */
-
-        /* Thread is no longer cancellable. */
-        thread->ClearCancellable();
-
-        /* Cancel the timer as needed. */
-        if (timer != nullptr) {
-            timer->CancelTask(thread);
-        }
-
-        /* Get the wait result. */
-        Result wait_result;
-        {
-            KScopedSchedulerLock lk;
-            KSynchronizationObject *synced_obj;
-            wait_result = thread->GetWaitResult(std::addressof(synced_obj));
-
-            for (auto i = 0; i < num_objects; ++i) {
-                objects[i]->UnregisterWaitingThread(thread_iters[i]);
-                if (objects[i] == synced_obj) {
-                    sync_index = i;
-                }
-            }
-        }
-
-        /* Set output. */
-        *out_index = sync_index;
-        return wait_result;
-    }
-
-    void KSynchronization::OnAvailable(KSynchronizationObject *object) {
-        MESOSPHERE_ASSERT_THIS();
-
-        KScopedSchedulerLock sl;
-
-        /* If we're not signaled, we've nothing to notify. */
-        if (!object->IsSignaled()) {
-            return;
-        }
-
-        /* Iterate over each thread. */
-        for (auto &thread : *object) {
-            if (thread.GetState() == KThread::ThreadState_Waiting) {
-                thread.SetSyncedObject(object, ResultSuccess());
-                thread.SetState(KThread::ThreadState_Runnable);
-            }
-        }
-    }
-
-    void KSynchronization::OnAbort(KSynchronizationObject *object, Result abort_reason) {
-        MESOSPHERE_ASSERT_THIS();
-
-        KScopedSchedulerLock sl;
-
-        /* Iterate over each thread. */
-        for (auto &thread : *object) {
-            if (thread.GetState() == KThread::ThreadState_Waiting) {
-                thread.SetSyncedObject(object, abort_reason);
-                thread.SetState(KThread::ThreadState_Runnable);
-            }
-        }
-    }
-
-}

libraries/libmesosphere/source/kern_k_synchronization_object.cpp

@@ -17,18 +17,6 @@
 
 namespace ams::kern {
 
-    void KSynchronizationObject::NotifyAvailable() {
-        MESOSPHERE_ASSERT_THIS();
-
-        Kernel::GetSynchronization().OnAvailable(this);
-    }
-
-    void KSynchronizationObject::NotifyAbort(Result abort_reason) {
-        MESOSPHERE_ASSERT_THIS();
-
-        Kernel::GetSynchronization().OnAbort(this, abort_reason);
-    }
-
     void KSynchronizationObject::Finalize() {
         MESOSPHERE_ASSERT_THIS();
 
@@ -37,9 +25,8 @@ namespace ams::kern {
         {
             KScopedSchedulerLock sl;
 
-            auto end = this->end();
+            for (auto *cur_node = this->thread_list_root; cur_node != nullptr; cur_node = cur_node->next) {
-            for (auto it = this->begin(); it != end; ++it) {
+                KThread *thread = cur_node->thread;
-                KThread *thread = std::addressof(*it);
                 MESOSPHERE_LOG("KSynchronizationObject::Finalize(%p) with %p (id=%ld) waiting.\n", this, thread, thread->GetId());
             }
         }
@@ -49,6 +36,118 @@ namespace ams::kern {
         KAutoObject::Finalize();
     }
 
+    Result KSynchronizationObject::Wait(s32 *out_index, KSynchronizationObject **objects, const s32 num_objects, s64 timeout) {
+        /* Allocate space on stack for thread nodes. */
+        ThreadListNode *thread_nodes = static_cast<ThreadListNode *>(__builtin_alloca(sizeof(ThreadListNode) * num_objects));
+
+        /* Prepare for wait. */
+        KThread *thread = GetCurrentThreadPointer();
+        KHardwareTimer *timer;
+
+        {
+            /* Setup the scheduling lock and sleep. */
+            KScopedSchedulerLockAndSleep slp(std::addressof(timer), thread, timeout);
+
+            /* Check if any of the objects are already signaled. */
+            for (auto i = 0; i < num_objects; ++i) {
+                AMS_ASSERT(objects[i] != nullptr);
+
+                if (objects[i]->IsSignaled()) {
+                    *out_index = i;
+                    slp.CancelSleep();
+                    return ResultSuccess();
+                }
+            }
+
+            /* Check if the timeout is zero. */
+            if (timeout == 0) {
+                slp.CancelSleep();
+                return svc::ResultTimedOut();
+            }
+
+            /* Check if the thread should terminate. */
+            if (thread->IsTerminationRequested()) {
+                slp.CancelSleep();
+                return svc::ResultTerminationRequested();
+            }
+
+            /* Check if waiting was canceled. */
+            if (thread->IsWaitCancelled()) {
+                slp.CancelSleep();
+                thread->ClearWaitCancelled();
+                return svc::ResultCancelled();
+            }
+
+            /* Add the waiters. */
+            for (auto i = 0; i < num_objects; ++i) {
+                thread_nodes[i].thread      = thread;
+                thread_nodes[i].next        = objects[i]->thread_list_root;
+                objects[i]->thread_list_root = std::addressof(thread_nodes[i]);
+            }
+
+            /* Mark the thread as waiting. */
+            thread->SetCancellable();
+            thread->SetSyncedObject(nullptr, svc::ResultTimedOut());
+            thread->SetState(KThread::ThreadState_Waiting);
+        }
+
+        /* The lock/sleep is done, so we should be able to get our result. */
+
+        /* Thread is no longer cancellable. */
+        thread->ClearCancellable();
+
+        /* Cancel the timer as needed. */
+        if (timer != nullptr) {
+            timer->CancelTask(thread);
+        }
+
+        /* Get the wait result. */
+        Result wait_result;
+        s32 sync_index = -1;
+        {
+            KScopedSchedulerLock lk;
+            KSynchronizationObject *synced_obj;
+            wait_result = thread->GetWaitResult(std::addressof(synced_obj));
+
+            for (auto i = 0; i < num_objects; ++i) {
+                /* Unlink the object from the list. */
+                ThreadListNode **link = std::addressof(objects[i]->thread_list_root);
+                while (*link != std::addressof(thread_nodes[i])) {
+                    link = std::addressof((*link)->next);
+                }
+                *link = thread_nodes[i].next;
+
+                if (objects[i] == synced_obj) {
+                    sync_index = i;
+                }
+            }
+        }
+
+        /* Set output. */
+        *out_index = sync_index;
+        return wait_result;
+    }
+
+    void KSynchronizationObject::NotifyAvailable(Result result) {
+        MESOSPHERE_ASSERT_THIS();
+
+        KScopedSchedulerLock sl;
+
+        /* If we're not signaled, we've nothing to notify. */
+        if (!this->IsSignaled()) {
+            return;
+        }
+
+        /* Iterate over each thread. */
+        for (auto *cur_node = this->thread_list_root; cur_node != nullptr; cur_node = cur_node->next) {
+            KThread *thread = cur_node->thread;
+            if (thread->GetState() == KThread::ThreadState_Waiting) {
+                thread->SetSyncedObject(this, result);
+                thread->SetState(KThread::ThreadState_Runnable);
+            }
+        }
+    }
+
     void KSynchronizationObject::DebugWaiters() {
         MESOSPHERE_ASSERT_THIS();
 
@@ -60,9 +159,8 @@ namespace ams::kern {
             MESOSPHERE_RELEASE_LOG("Threads waiting on %p:\n", this);
 
             bool has_waiters = false;
-            auto end = this->end();
+            for (auto *cur_node = this->thread_list_root; cur_node != nullptr; cur_node = cur_node->next) {
-            for (auto it = this->begin(); it != end; ++it) {
+                KThread *thread = cur_node->thread;
-                KThread *thread = std::addressof(*it);
 
                 if (KProcess *process = thread->GetOwnerProcess(); process != nullptr) {
                     MESOSPHERE_RELEASE_LOG("    %p tid=%ld pid=%ld (%s)\n", thread, thread->GetId(), process->GetId(), process->GetName());
@@ -81,28 +179,4 @@ namespace ams::kern {
         #endif
     }
 
-    KSynchronizationObject::iterator KSynchronizationObject::RegisterWaitingThread(KThread *thread) {
-        MESOSPHERE_ASSERT_THIS();
-
-        return this->thread_list.insert(this->thread_list.end(), *thread);
-    }
-
-    KSynchronizationObject::iterator KSynchronizationObject::UnregisterWaitingThread(KSynchronizationObject::iterator it) {
-        MESOSPHERE_ASSERT_THIS();
-
-        return this->thread_list.erase(it);
-    }
-
-    KSynchronizationObject::iterator KSynchronizationObject::begin() {
-        MESOSPHERE_ASSERT_THIS();
-
-        return this->thread_list.begin();
-    }
-
-    KSynchronizationObject::iterator KSynchronizationObject::end() {
-        MESOSPHERE_ASSERT_THIS();
-
-        return this->thread_list.end();
-    }
-
 }

libraries/libmesosphere/source/kern_k_thread.cpp

@@ -38,13 +38,17 @@ namespace ams::kern {
 
     }
 
-    Result KThread::Initialize(KThreadFunction func, uintptr_t arg, void *kern_stack_top, KProcessAddress user_stack_top, s32 prio, s32 core, KProcess *owner, ThreadType type) {
+    Result KThread::Initialize(KThreadFunction func, uintptr_t arg, void *kern_stack_top, KProcessAddress user_stack_top, s32 prio, s32 virt_core, KProcess *owner, ThreadType type) {
         /* Assert parameters are valid. */
         MESOSPHERE_ASSERT_THIS();
         MESOSPHERE_ASSERT(kern_stack_top != nullptr);
         MESOSPHERE_ASSERT((type == ThreadType_Main) || (ams::svc::HighestThreadPriority <= prio && prio <= ams::svc::LowestThreadPriority));
         MESOSPHERE_ASSERT((owner != nullptr) || (type != ThreadType_User));
-        MESOSPHERE_ASSERT(0 <= core && core < static_cast<s32>(cpu::NumCores));
+        MESOSPHERE_ASSERT(0 <= virt_core && virt_core < static_cast<s32>(BITSIZEOF(u64)));
+
+        /* Convert the virtual core to a physical core. */
+        const s32 phys_core = cpu::VirtualToPhysicalCoreMap[virt_core];
+        MESOSPHERE_ASSERT(0 <= phys_core && phys_core < static_cast<s32>(cpu::NumCores));
 
         /* First, clear the TLS address. */
         this->tls_address = Null<KProcessAddress>;
@@ -59,8 +63,8 @@ namespace ams::kern {
                 }
                 [[fallthrough]];
             case ThreadType_HighPriority:
-                {
+                if (type != ThreadType_Main) {
-                    MESOSPHERE_ASSERT(core == GetCurrentCoreId());
+                    MESOSPHERE_ASSERT(phys_core == GetCurrentCoreId());
                 }
                 [[fallthrough]];
             case ThreadType_Kernel:
@@ -71,8 +75,8 @@ namespace ams::kern {
                 [[fallthrough]];
             case ThreadType_User:
                 {
-                    MESOSPHERE_ASSERT(((owner == nullptr) || (owner->GetCoreMask()     | (1ul << core)) == owner->GetCoreMask()));
+                    MESOSPHERE_ASSERT(((owner == nullptr) || (owner->GetCoreMask()     | (1ul << virt_core)) == owner->GetCoreMask()));
-                    MESOSPHERE_ASSERT(((owner == nullptr) || (owner->GetPriorityMask() | (1ul << prio)) == owner->GetPriorityMask()));
+                    MESOSPHERE_ASSERT(((owner == nullptr) || (owner->GetPriorityMask() | (1ul << prio))      == owner->GetPriorityMask()));
                 }
                 break;
             default:
@@ -81,8 +85,10 @@ namespace ams::kern {
         }
 
         /* Set the ideal core ID and affinity mask. */
-        this->ideal_core_id                 = core;
+        this->virtual_ideal_core_id  = virt_core;
-        this->affinity_mask.SetAffinity(core, true);
+        this->physical_ideal_core_id = phys_core;
+        this->virtual_affinity_mask = (static_cast<u64>(1) << virt_core);
+        this->physical_affinity_mask.SetAffinity(phys_core, true);
 
         /* Set the thread state. */
         this->thread_state                  = (type == ThreadType_Main) ? ThreadState_Runnable : ThreadState_Initialized;
@@ -98,13 +104,12 @@ namespace ams::kern {
 
         /* Set sync booleans. */
         this->signaled                      = false;
-        this->ipc_cancelled                 = false;
         this->termination_requested         = false;
         this->wait_cancelled                = false;
         this->cancellable                   = false;
 
         /* Set core ID and wait result. */
-        this->core_id                       = this->ideal_core_id;
+        this->core_id                       = phys_core;
         this->wait_result                   = svc::ResultNoSynchronizationObject();
 
         /* Set the stack top. */
@@ -119,7 +124,6 @@ namespace ams::kern {
         this->waiting_lock                  = nullptr;
 
         /* Initialize sleeping queue. */
-        this->sleeping_queue_entry.Initialize();
         this->sleeping_queue                = nullptr;
 
         /* Set suspend flags. */
@@ -141,7 +145,9 @@ namespace ams::kern {
 
         /* We have no waiters, but we do have an entrypoint. */
         this->num_kernel_waiters            = 0;
-        this->entrypoint                    = reinterpret_cast<uintptr_t>(func);
+
+        /* Set our current core id. */
+        this->current_core_id               = phys_core;
 
         /* We haven't released our resource limit hint, and we've spent no time on the cpu. */
         this->resource_limit_release_hint   = 0;
@@ -169,7 +175,7 @@ namespace ams::kern {
         const bool is_64_bit = this->parent ? this->parent->Is64Bit() : IsDefault64Bit;
         const bool is_user = (type == ThreadType_User);
         const bool is_main = (type == ThreadType_Main);
-        this->thread_context.Initialize(this->entrypoint, reinterpret_cast<uintptr_t>(this->GetStackTop()), GetInteger(user_stack_top), arg, is_user, is_64_bit, is_main);
+        this->thread_context.Initialize(reinterpret_cast<uintptr_t>(func), reinterpret_cast<uintptr_t>(this->GetStackTop()), GetInteger(user_stack_top), arg, is_user, is_64_bit, is_main);
 
         /* Setup the stack parameters. */
         StackParameters &sp = this->GetStackParameters();
@@ -177,6 +183,7 @@ namespace ams::kern {
             this->parent->CopySvcPermissionsTo(sp);
         }
         sp.context = std::addressof(this->thread_context);
+        sp.cur_thread = this;
         sp.disable_count = 1;
         this->SetInExceptionHandler();
 
@@ -200,6 +207,7 @@ namespace ams::kern {
     Result KThread::InitializeThread(KThread *thread, KThreadFunction func, uintptr_t arg, KProcessAddress user_stack_top, s32 prio, s32 core, KProcess *owner, ThreadType type) {
         /* Get stack region for the thread. */
         const auto &stack_region = KMemoryLayout::GetKernelStackRegion();
+        MESOSPHERE_ABORT_UNLESS(stack_region.GetEndAddress() != 0);
 
         /* Allocate a page to use as the thread. */
         KPageBuffer *page = KPageBuffer::Allocate();
@@ -362,7 +370,7 @@ namespace ams::kern {
             for (size_t i = 0; i < cpu::NumCores; ++i) {
                 KThread *core_thread;
                 do {
-                    core_thread = Kernel::GetCurrentContext(i).current_thread.load(std::memory_order_acquire);
+                    core_thread = Kernel::GetScheduler(i).GetSchedulerCurrentThread();
                 } while (core_thread == this);
             }
         }
@@ -389,20 +397,19 @@ namespace ams::kern {
             ++this->num_core_migration_disables;
 
             /* Save our ideal state to restore when we're unpinned. */
-            this->original_ideal_core_id = this->ideal_core_id;
+            this->original_physical_ideal_core_id = this->physical_ideal_core_id;
-            this->original_affinity_mask = this->affinity_mask;
+            this->original_physical_affinity_mask = this->physical_affinity_mask;
 
             /* Bind ourselves to this core. */
             const s32 active_core  = this->GetActiveCore();
             const s32 current_core = GetCurrentCoreId();
 
             this->SetActiveCore(current_core);
-            this->ideal_core_id = current_core;
+            this->physical_ideal_core_id = current_core;
+            this->physical_affinity_mask.SetAffinityMask(1ul << current_core);
 
-            this->affinity_mask.SetAffinityMask(1ul << current_core);
+            if (active_core != current_core || this->physical_affinity_mask.GetAffinityMask() != this->original_physical_affinity_mask.GetAffinityMask()) {
-
+                KScheduler::OnThreadAffinityMaskChanged(this, this->original_physical_affinity_mask, active_core);
-            if (active_core != current_core || this->affinity_mask.GetAffinityMask() != this->original_affinity_mask.GetAffinityMask()) {
-                KScheduler::OnThreadAffinityMaskChanged(this, this->original_affinity_mask, active_core);
             }
         }
 
@@ -437,19 +444,19 @@ namespace ams::kern {
             --this->num_core_migration_disables;
 
             /* Restore our original state. */
-            const KAffinityMask old_mask = this->affinity_mask;
+            const KAffinityMask old_mask = this->physical_affinity_mask;
 
-            this->ideal_core_id = this->original_ideal_core_id;
+            this->physical_ideal_core_id = this->original_physical_ideal_core_id;
-            this->affinity_mask = this->original_affinity_mask;
+            this->physical_affinity_mask = this->original_physical_affinity_mask;
 
-            if (this->affinity_mask.GetAffinityMask() != old_mask.GetAffinityMask()) {
+            if (this->physical_affinity_mask.GetAffinityMask() != old_mask.GetAffinityMask()) {
                 const s32 active_core = this->GetActiveCore();
 
-                if (!this->affinity_mask.GetAffinity(active_core)) {
+                if (!this->physical_affinity_mask.GetAffinity(active_core)) {
-                    if (this->ideal_core_id >= 0) {
+                    if (this->physical_ideal_core_id >= 0) {
-                        this->SetActiveCore(this->ideal_core_id);
+                        this->SetActiveCore(this->physical_ideal_core_id);
                     } else {
-                        this->SetActiveCore(BITSIZEOF(unsigned long long) - 1 - __builtin_clzll(this->affinity_mask.GetAffinityMask()));
+                        this->SetActiveCore(BITSIZEOF(unsigned long long) - 1 - __builtin_clzll(this->physical_affinity_mask.GetAffinityMask()));
                     }
                 }
                 KScheduler::OnThreadAffinityMaskChanged(this, old_mask, active_core);
@@ -491,16 +498,16 @@ namespace ams::kern {
         MESOSPHERE_ASSERT(this->num_core_migration_disables >= 0);
         if ((this->num_core_migration_disables++) == 0) {
             /* Save our ideal state to restore when we can migrate again. */
-            this->original_ideal_core_id = this->ideal_core_id;
+            this->original_physical_ideal_core_id = this->physical_ideal_core_id;
-            this->original_affinity_mask = this->affinity_mask;
+            this->original_physical_affinity_mask = this->physical_affinity_mask;
 
             /* Bind ourselves to this core. */
             const s32 active_core = this->GetActiveCore();
-            this->ideal_core_id = active_core;
+            this->physical_ideal_core_id = active_core;
-            this->affinity_mask.SetAffinityMask(1ul << active_core);
+            this->physical_affinity_mask.SetAffinityMask(1ul << active_core);
 
-            if (this->affinity_mask.GetAffinityMask() != this->original_affinity_mask.GetAffinityMask()) {
+            if (this->physical_affinity_mask.GetAffinityMask() != this->original_physical_affinity_mask.GetAffinityMask()) {
-                KScheduler::OnThreadAffinityMaskChanged(this, this->original_affinity_mask, active_core);
+                KScheduler::OnThreadAffinityMaskChanged(this, this->original_physical_affinity_mask, active_core);
             }
         }
     }
@@ -512,20 +519,20 @@ namespace ams::kern {
         KScopedSchedulerLock sl;
         MESOSPHERE_ASSERT(this->num_core_migration_disables > 0);
         if ((--this->num_core_migration_disables) == 0) {
-            const KAffinityMask old_mask = this->affinity_mask;
+            const KAffinityMask old_mask = this->physical_affinity_mask;
 
             /* Restore our ideals. */
-            this->ideal_core_id = this->original_ideal_core_id;
+            this->physical_ideal_core_id = this->original_physical_ideal_core_id;
-            this->affinity_mask = this->original_affinity_mask;
+            this->physical_affinity_mask = this->original_physical_affinity_mask;
 
-            if (this->affinity_mask.GetAffinityMask() != old_mask.GetAffinityMask()) {
+            if (this->physical_affinity_mask.GetAffinityMask() != old_mask.GetAffinityMask()) {
                 const s32 active_core = this->GetActiveCore();
 
-                if (!this->affinity_mask.GetAffinity(active_core)) {
+                if (!this->physical_affinity_mask.GetAffinity(active_core)) {
-                    if (this->ideal_core_id >= 0) {
+                    if (this->physical_ideal_core_id >= 0) {
-                        this->SetActiveCore(this->ideal_core_id);
+                        this->SetActiveCore(this->physical_ideal_core_id);
                     } else {
-                        this->SetActiveCore(BITSIZEOF(unsigned long long) - 1 - __builtin_clzll(this->affinity_mask.GetAffinityMask()));
+                        this->SetActiveCore(BITSIZEOF(unsigned long long) - 1 - __builtin_clzll(this->physical_affinity_mask.GetAffinityMask()));
                     }
                 }
                 KScheduler::OnThreadAffinityMaskChanged(this, old_mask, active_core);
@@ -534,6 +541,19 @@ namespace ams::kern {
     }
 
     Result KThread::GetCoreMask(int32_t *out_ideal_core, u64 *out_affinity_mask) {
+        MESOSPHERE_ASSERT_THIS();
+        {
+            KScopedSchedulerLock sl;
+
+            /* Get the virtual mask. */
+            *out_ideal_core    = this->virtual_ideal_core_id;
+            *out_affinity_mask = this->virtual_affinity_mask;
+        }
+
+        return ResultSuccess();
+    }
+
+    Result KThread::GetPhysicalCoreMask(int32_t *out_ideal_core, u64 *out_affinity_mask) {
         MESOSPHERE_ASSERT_THIS();
         {
             KScopedSchedulerLock sl;
@@ -541,63 +561,72 @@ namespace ams::kern {
 
             /* Select between core mask and original core mask. */
             if (this->num_core_migration_disables == 0) {
-                *out_ideal_core    = this->ideal_core_id;
+                *out_ideal_core    = this->physical_ideal_core_id;
-                *out_affinity_mask = this->affinity_mask.GetAffinityMask();
+                *out_affinity_mask = this->physical_affinity_mask.GetAffinityMask();
             } else {
-                *out_ideal_core    = this->original_ideal_core_id;
+                *out_ideal_core    = this->original_physical_ideal_core_id;
-                *out_affinity_mask = this->original_affinity_mask.GetAffinityMask();
+                *out_affinity_mask = this->original_physical_affinity_mask.GetAffinityMask();
             }
         }
 
         return ResultSuccess();
     }
 
-    Result KThread::SetCoreMask(int32_t ideal_core, u64 affinity_mask) {
+    Result KThread::SetCoreMask(int32_t core_id, u64 v_affinity_mask) {
         MESOSPHERE_ASSERT_THIS();
         MESOSPHERE_ASSERT(this->parent != nullptr);
-        MESOSPHERE_ASSERT(affinity_mask != 0);
+        MESOSPHERE_ASSERT(v_affinity_mask != 0);
         KScopedLightLock lk(this->activity_pause_lock);
 
         /* Set the core mask. */
+        u64 p_affinity_mask = 0;
         {
             KScopedSchedulerLock sl;
             MESOSPHERE_ASSERT(this->num_core_migration_disables >= 0);
 
             /* If the core id is no-update magic, preserve the ideal core id. */
-            if (ideal_core == ams::svc::IdealCoreNoUpdate) {
+            if (core_id == ams::svc::IdealCoreNoUpdate) {
-                if (this->num_core_migration_disables == 0) {
+                core_id = this->virtual_ideal_core_id;
-                    ideal_core = this->ideal_core_id;
+                R_UNLESS(((1ul << core_id) & v_affinity_mask) != 0, svc::ResultInvalidCombination());
-                } else {
+            }
-                    ideal_core = this->original_ideal_core_id;
-                }
 
-                R_UNLESS(((1ul << ideal_core) & affinity_mask) != 0, svc::ResultInvalidCombination());
+            /* Set the virtual core/affinity mask. */
+            this->virtual_ideal_core_id = core_id;
+            this->virtual_affinity_mask = v_affinity_mask;
+
+            /* Translate the virtual core to a physical core. */
+            if (core_id >= 0) {
+                core_id = cpu::VirtualToPhysicalCoreMap[core_id];
+            }
+
+            /* Translate the virtual affinity mask to a physical one. */
+            while (v_affinity_mask != 0) {
+                const u64 next = __builtin_ctzll(v_affinity_mask);
+                v_affinity_mask &= ~(1ul << next);
+                p_affinity_mask |=  (1ul << cpu::VirtualToPhysicalCoreMap[next]);
             }
 
             /* If we haven't disabled migration, perform an affinity change. */
             if (this->num_core_migration_disables == 0) {
-                const KAffinityMask old_mask = this->affinity_mask;
+                const KAffinityMask old_mask = this->physical_affinity_mask;
 
                 /* Set our new ideals. */
-                this->ideal_core_id = ideal_core;
+                this->physical_ideal_core_id = core_id;
-                this->affinity_mask.SetAffinityMask(affinity_mask);
+                this->physical_affinity_mask.SetAffinityMask(p_affinity_mask);
 
-                if (this->affinity_mask.GetAffinityMask() != old_mask.GetAffinityMask()) {
+                if (this->physical_affinity_mask.GetAffinityMask() != old_mask.GetAffinityMask()) {
                     const s32 active_core = this->GetActiveCore();
 
-                    if (active_core >= 0) {
+                    if (active_core >= 0 && !this->physical_affinity_mask.GetAffinity(active_core)) {
-                        if (!this->affinity_mask.GetAffinity(active_core)) {
+                        const s32 new_core = this->physical_ideal_core_id >= 0 ? this->physical_ideal_core_id : BITSIZEOF(unsigned long long) - 1 - __builtin_clzll(this->physical_affinity_mask.GetAffinityMask());
-                            this->SetActiveCore(this->ideal_core_id);
+                        this->SetActiveCore(new_core);
-                        } else {
-                            this->SetActiveCore(BITSIZEOF(unsigned long long) - 1 - __builtin_clzll(this->affinity_mask.GetAffinityMask()));
-                        }
                     }
                     KScheduler::OnThreadAffinityMaskChanged(this, old_mask, active_core);
                 }
             } else {
                 /* Otherwise, we edit the original affinity for restoration later. */
-                this->original_ideal_core_id = ideal_core;
+                this->original_physical_ideal_core_id = core_id;
-                this->original_affinity_mask.SetAffinityMask(affinity_mask);
+                this->original_physical_affinity_mask.SetAffinityMask(p_affinity_mask);
             }
         }
 
@@ -619,14 +648,14 @@ namespace ams::kern {
                 bool thread_is_current = false;
                 s32 thread_core;
                 for (thread_core = 0; thread_core < static_cast<s32>(cpu::NumCores); ++thread_core) {
-                    if (Kernel::GetCurrentContext(thread_core).current_thread == this) {
+                    if (Kernel::GetScheduler(thread_core).GetSchedulerCurrentThread() == this) {
                         thread_is_current = true;
                         break;
                     }
                 }
 
                 /* If the thread is currently running, check whether it's no longer allowed under the new mask. */
-                if (thread_is_current && ((1ul << thread_core) & affinity_mask) == 0) {
+                if (thread_is_current && ((1ul << thread_core) & p_affinity_mask) == 0) {
                     /* If the thread is pinned, we want to wait until it's not pinned. */
                     if (this->GetStackParameters().is_pinned) {
                         /* Verify that the current thread isn't terminating. */
@@ -834,7 +863,7 @@ namespace ams::kern {
                     thread_is_current = false;
 
                     for (auto i = 0; i < static_cast<s32>(cpu::NumCores); ++i) {
-                        if (Kernel::GetCurrentContext(i).current_thread == this) {
+                        if (Kernel::GetScheduler(i).GetSchedulerCurrentThread() == this) {
                             thread_is_current = true;
                             break;
                         }
@@ -1085,7 +1114,7 @@ namespace ams::kern {
             /* If the thread isn't terminated, wait for it to terminate. */
             s32 index;
             KSynchronizationObject *objects[] = { this };
-            Kernel::GetSynchronization().Wait(std::addressof(index), objects, 1, ams::svc::WaitInfinite);
+            KSynchronizationObject::Wait(std::addressof(index), objects, 1, ams::svc::WaitInfinite);
         }
     }
 
@@ -1126,7 +1155,7 @@ namespace ams::kern {
 
             /* If the thread is runnable, send a termination interrupt to other cores. */
             if (this->GetState() == ThreadState_Runnable) {
-                if (const u64 core_mask = this->affinity_mask.GetAffinityMask() & ~(1ul << GetCurrentCoreId()); core_mask != 0) {
+                if (const u64 core_mask = this->physical_affinity_mask.GetAffinityMask() & ~(1ul << GetCurrentCoreId()); core_mask != 0) {
                     cpu::DataSynchronizationBarrier();
                     Kernel::GetInterruptManager().SendInterProcessorInterrupt(KInterruptName_ThreadTerminate, core_mask);
                 }