docs: fix partial changelog

subrepo:
  subdir:   "emummc"
  merged:   "b168ddf5"
upstream:
  origin:   "https://github.com/m4xw/emuMMC"
  branch:   "develop"
  commit:   "b168ddf5"
git-subrepo:
  version:  "0.4.1"
  origin:   "???"
  commit:   "???"

.gitignore

@@ -64,6 +64,7 @@ dkms.conf
 # Distribution files
 *.tgz
 *.zip
+*.bz2
 
 # IDA binaries
 *.id0

Makefile

@@ -58,6 +58,7 @@ dist-no-debug: all
 	mkdir atmosphere-$(AMSVER)/switch
 	mkdir -p atmosphere-$(AMSVER)/atmosphere/contents/0100000000000008
 	mkdir -p atmosphere-$(AMSVER)/atmosphere/contents/010000000000000D
+	mkdir -p atmosphere-$(AMSVER)/atmosphere/contents/010000000000002B
 	mkdir -p atmosphere-$(AMSVER)/atmosphere/contents/0100000000000032
 	mkdir -p atmosphere-$(AMSVER)/atmosphere/contents/0100000000000034
 	mkdir -p atmosphere-$(AMSVER)/atmosphere/contents/0100000000000036
@@ -82,6 +83,7 @@ dist-no-debug: all
 	cp -r config_templates/hbl_html atmosphere-$(AMSVER)/atmosphere/hbl_html
 	cp stratosphere/boot2/boot2.nsp atmosphere-$(AMSVER)/atmosphere/contents/0100000000000008/exefs.nsp
 	cp stratosphere/dmnt/dmnt.nsp atmosphere-$(AMSVER)/atmosphere/contents/010000000000000D/exefs.nsp
+	cp stratosphere/erpt/erpt.nsp atmosphere-$(AMSVER)/atmosphere/contents/010000000000002B/exefs.nsp
 	cp stratosphere/eclct.stub/eclct.stub.nsp atmosphere-$(AMSVER)/atmosphere/contents/0100000000000032/exefs.nsp
 	cp stratosphere/fatal/fatal.nsp atmosphere-$(AMSVER)/atmosphere/contents/0100000000000034/exefs.nsp
 	cp stratosphere/creport/creport.nsp atmosphere-$(AMSVER)/atmosphere/contents/0100000000000036/exefs.nsp
@@ -133,6 +135,7 @@ dist: dist-no-debug
 	cp stratosphere/ro/ro.elf atmosphere-$(AMSVER)-debug/ro.elf
 	cp stratosphere/sm/sm.elf atmosphere-$(AMSVER)-debug/sm.elf
 	cp stratosphere/spl/spl.elf atmosphere-$(AMSVER)-debug/spl.elf
+	cp stratosphere/erpt/erpt.elf atmosphere-$(AMSVER)-debug/erpt.elf
 	cd atmosphere-$(AMSVER)-debug; zip -r ../atmosphere-$(AMSVER)-debug.zip ./*; cd ../;
 	rm -r atmosphere-$(AMSVER)-debug
 	mv atmosphere-$(AMSVER)-debug.zip out/atmosphere-$(AMSVER)-debug.zip

README.md

@@ -28,6 +28,7 @@ You can find a copy of the license in the [LICENSE file](LICENSE).
 
 Exemptions:
 * The [yuzu emulator project](https://github.com/yuzu-emu/yuzu) is exempt from GPLv2 licensing and may (at its option) instead license any source code authored for the Atmosphère project as GPLv2 or later.
+* [Nintendo](https://github.com/Nintendo) is exempt from GPLv2 licensing and may (at its option) instead license any source code authored for the Atmosphère project under the Zero-Clause BSD license.
 
 Credits
 =====

config_templates/BCT.ini

@@ -18,3 +18,5 @@ enable_user_pmu_access = 0
 [stratosphere]
 ; To force-enable nogc, add nogc = 1
 ; To force-disable nogc, add nogc = 0
+
+; To opt in to using Atmosphere's NCM reimplementation, add enable_ncm = 1

docs/changelog.md

@@ -1,4 +1,75 @@
 # Changelog
+## 0.11.0
++ Support was added for 10.0.0.
+  + Exosphere has been updated to reflect the new key import semantics in 10.0.0.
+  + kernel_ldr now implements physical ASLR for the kernel's backing pages.
+  + Loader, NCM, and PM have been updated to reflect the changes Nintendo made in 10.0.0.
+  + Creport was updated to use the new `pgl` service to terminate processes instead of `ns:dev`.
++ A reimplementation of the `erpt` (error reports) system module was added.
+  + In previous versions of Atmosphere, a majority of error reports were prevented via a combination of custom creport, fatal, and stubbed eclct.
+  + However, error reports were still generated via some system actions.
+    + Most notably, any time the error applet appeared, an error report was generated.
+    + By default, atmosphere disabled the *uploading* of error reports, but going online in OFW after an error report occurred in Atmosphere could lead to undesirable telemetry.
+  + Atmosphere's `erpt` reimplementation allows the system to interact with existing error reports as expected.
+  + However, all new error reports are instead saved to the sd card (`/atmosphere/erpt_reports`), and are not committed to the system savegame.
+    + Users curious about what kind of telemetry is being prevented can view the reports as they're generated in there.
+    + Reports are saved as msgpack (as this is what Nintendo uses).
+  + Please note, not all telemetry is disabled. Play reports and System reports will continue to function unmodified.
+  + With atmosphere's `erpt` implementation, homebrew can now use the native error applet to display errors without worrying about generating undesirable telemetry.
++ libstratosphere and libvapours received a number of improvements.
+  + With thanks to @Adubbz for his work, the NCM namespace now has client code.
+    + This lays the groundwork for first-class system update/downgrade homebrew support in the near future.
+  + In particular, code implementing the os namespace is significantly more accurate.
+  + In addition, Nintendo's allocators were implemented, allowing for identical memory efficiency versus Nintendo's implementations.
++ General system stability improvements to enhance the user's experience.
+## 0.10.5
++ Changes were made to the way fs.mitm builds images when providing a layeredfs romfs.
+  + Building romfs metadata previously had a memory cost of about ~4-5x the file table size.
+  + This caused games that have particularly enormous file metadata tables (> 4 MB) to exhaust fs.mitm's 16 MB memory pool.
+  + The code that creates romfs images was thus changed to be significantly more memory efficient, again.
+  + Memory requirements have been lowered from ~4x file table size to ~2x file table size + 0.5 MB.
+  + There is a slight speed penalty to this, but testing on Football Manager 2020 only took an extra ~1.5 seconds for the game to boot with many modded files.
+    + This shouldn't be noticeable thanks to the async changes made in 0.10.2.
+  + If you encounter a game that exhausts ams.mitm's memory (crashing it) when loading layeredfs mods, please contact SciresM.
+    + Romfs building can be made even more memory efficient, but unless games show up with even more absurdly huge file tables it seems not worth the speed trade-off.
++ A bug was fixed that caused Atmosphere's fatal error context to not dump TLS for certain processes.
++ General system stability improvements to enhance the user's experience.
+## 0.10.4
++ With major thanks to @Adubbz for his work, the NCM system module has now been re-implemented.
+  + This was a major stepping stone towards the goal of having implementations everything in the Switch's package1/package2 firmware.
+  + This also lays the groundwork for libstratosphere providing bindings for changing the installed version of the Switch's OS.
+  + **Please Note**: The NCM implementation will initially be opt-in.
+    + The Atmosphere team is confident in our NCM implementation (and we have tested it on every firmware version).
+    + That said, this is our first system module that manages NAND savegames -- and caution is a habit.
+    + We do not anticipate any issues that didn't come up in testing, so this is just our being particularly careful.
+    + Users interested in opting in to using our implementation should set `stratosphere!ncm_enabled = 1` in BCT.ini.
+      + In the unlikely event that any issues are encountered, please report them to @SciresM.
+    + The NCM implementation will stop being opt-in in a future update, after thorough testing has been done in practice.
++ A bug was fixed in emummc that caused Nintendo path to be corrupted on 1.0.0.
+  + This manifested as the emummc folder being created inside the virtual NAND instead of the SD card.
+  + It's unlikely there are any negative consequences to this in practice.
+    + If you want to be truly sure, you can re-clone sysmmc before updating a 1.0.0 emummc to latest firmware.
++ Stratosphere system modules now use new Nintendo-style FS bindings instead of stdio.
+  + This saves a modest amount of memory due to leaner code, and greatly increases the accuracy of several components.
+  + These bindings will make it easier for other system modules using libstratosphere to interact with the filesystem.
+  + This also lays the groundwork for changes necessary to support per-emummc Atmosphere folders in a future update.
++ Atmosphere's fatal error context now dumps 0x100 of TLS.
+  + This will make it much easier to fix bugs when an error report is dumped for whatever caused the crash.
++ General system stability improvements to enhance the user's experience.
+## 0.10.3
++ Support was added for 9.2.0.
++ Support was added for redirecting manual html content for games.
+  + This works like normal layeredfs, replacing content placed in `/atmosphere/contents/<program id>/manual_html/`.
+  + This allows for game mods/translations to provide custom manual content, if they so choose.
++ A number of improvements were made to Atmosphere's memory usage, including:
+  + `fatal` now uses STB instead of freetype for rendering.
+    + This saves around 1 MB of memory, and makes our fatal substantially leaner than Nintendo's.
+  + `sm` no longer wastes 2 MiB unnecessarily.
++ fusee/sept's sdmmc access now better matches official behavior.
+  + This improves compatibility with some SD cards.
++ `ro` has been updated to reflect changes made in 9.1.0.
++ The temporary auto-migration added in 0.10.0 has been removed, since the transitionary period is well over.
++ General system stability improvements to enhance the user's experience.
 ## 0.10.2
 + hbl configuration was made more flexible.
   + Up to eight specific program ids can now be specified to have their own override keys.

emummc/.gitrepo

@@ -6,7 +6,7 @@
 [subrepo]
 	remote = https://github.com/m4xw/emuMMC
 	branch = develop
-	commit = bd81a674a946c30b566e1732a95c18f19b701558
-	parent = 6ee525201ccef107c61d81ba73c891e3eb5f0215
+	commit = b168ddf5fbb31013ff529a4859110c82b11eb361
+	parent = c07f54f3709a4710e0aead6c91139fa0893b5e5c
 	method = rebase
-	cmdver = 0.4.0
+	cmdver = 0.4.1

emummc/README.md

@@ -1,21 +1,21 @@
 # emuMMC
-*A SDMMC driver replacement for Nintendo's Filesystem Services, by **m4xw***
+*A SDMMC driver replacement for Nintendo's Filesystem Services, by **m4xw***  
 
 ### Supported Horizon Versions
-**1.0.0 - 9.1.0**
+**1.0.0 - 10.0.0**
 
 ## Features
-* Arbitrary SDMMC backend selection
+* Arbitrary SDMMC backend selection  
   **This allows loading eMMC from SD or even SD from eMMC**
-* On the fly hooking / patching, fully self-infesting
+* On the fly hooking / patching, fully self-infesting  
   **Only one payload required for all versions!**
-* File-based SDMMC backend support (from SD)
+* File-based SDMMC backend support (from SD)  
   **This allows loading eMMC images from hekate-backups (split or not)**
-* SDMMC device based sector offset (*currently eMMC only*)
+* SDMMC device based sector offset (*currently eMMC only*)  
   **Raw partition support for eMMC from SD with less performance overhead**
-* Full support for `/Nintendo` folder redirection to a arbitrary path
+* Full support for `/Nintendo` folder redirection to a arbitrary path  
   **No 8 char length restriction!**
-* exosphere based context configuration
+* exosphere based context configuration  
   **This includes full support for multiple emuMMC images**
 
 ## Compiling

emummc/source/FS/FS_offsets.c

@@ -45,6 +45,8 @@
 #include "offsets/900_exfat.h"
 #include "offsets/910.h"
 #include "offsets/910_exfat.h"
+#include "offsets/1000.h"
+#include "offsets/1000_exfat.h"
 #include "../utils/fatal.h"
 
 #define GET_OFFSET_STRUCT_NAME(vers) g_offsets##vers
@@ -100,6 +102,8 @@ DEFINE_OFFSET_STRUCT(_900);
 DEFINE_OFFSET_STRUCT(_900_EXFAT);
 DEFINE_OFFSET_STRUCT(_910);
 DEFINE_OFFSET_STRUCT(_910_EXFAT);
+DEFINE_OFFSET_STRUCT(_1000);
+DEFINE_OFFSET_STRUCT(_1000_EXFAT);
 
 const fs_offsets_t *get_fs_offsets(enum FS_VER version) {
     switch (version) {
@@ -161,6 +165,10 @@ const fs_offsets_t *get_fs_offsets(enum FS_VER version) {
             return &(GET_OFFSET_STRUCT_NAME(_910));
         case FS_VER_9_1_0_EXFAT:
             return &(GET_OFFSET_STRUCT_NAME(_910_EXFAT));
+        case FS_VER_10_0_0:
+            return &(GET_OFFSET_STRUCT_NAME(_1000));
+        case FS_VER_10_0_0_EXFAT:
+            return &(GET_OFFSET_STRUCT_NAME(_1000_EXFAT));
         default:
             fatal_abort(Fatal_UnknownVersion);
     }

emummc/source/FS/FS_versions.h

@@ -65,6 +65,9 @@ enum FS_VER
     FS_VER_9_1_0,
     FS_VER_9_1_0_EXFAT,
 
+    FS_VER_10_0_0,
+    FS_VER_10_0_0_EXFAT,
+
     FS_VER_MAX,
 };
 

emummc/source/FS/offsets/100.h

@@ -48,10 +48,8 @@
 // Nintendo Paths
 #define FS_OFFSET_100_NINTENDO_PATHS \
 { \
-    {.opcode_reg = 9, .adrp_offset = 0x00032C58, .add_rel_offset = 4}, \
-    {.opcode_reg = 8, .adrp_offset = 0x00032C60, .add_rel_offset = 4}, \
-    {.opcode_reg = 9, .adrp_offset = 0x00032F3C, .add_rel_offset = 4}, \
-    {.opcode_reg = 8, .adrp_offset = 0x00032F44, .add_rel_offset = 4}, \
+    {.opcode_reg = 8, .adrp_offset = 0x00032C58, .add_rel_offset = 8}, \
+    {.opcode_reg = 9, .adrp_offset = 0x00032F40, .add_rel_offset = 8}, \
     {.opcode_reg = 0, .adrp_offset = 0, .add_rel_offset = 0}, \
 }
 

emummc/source/FS/offsets/1000.h

@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2019 m4xw <m4x@m4xw.net>
+ * Copyright (c) 2019 Atmosphere-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef __FS_1000_H__
+#define __FS_1000_H__
+
+// Accessor vtable getters
+#define FS_OFFSET_1000_SDMMC_ACCESSOR_GC   0x14DC90
+#define FS_OFFSET_1000_SDMMC_ACCESSOR_SD   0x14BDA0
+#define FS_OFFSET_1000_SDMMC_ACCESSOR_NAND 0x146C20
+
+// Hooks
+#define FS_OFFSET_1000_SDMMC_WRAPPER_READ  0x142380
+#define FS_OFFSET_1000_SDMMC_WRAPPER_WRITE 0x142460
+#define FS_OFFSET_1000_RTLD                0x634
+#define FS_OFFSET_1000_RTLD_DESTINATION    0x9C
+
+#define FS_OFFSET_1000_CLKRST_SET_MIN_V_CLK_RATE 0x1415A0
+
+// Misc funcs
+#define FS_OFFSET_1000_LOCK_MUTEX          0x28910
+#define FS_OFFSET_1000_UNLOCK_MUTEX        0x28960
+
+#define FS_OFFSET_1000_SDMMC_WRAPPER_CONTROLLER_CLOSE 0x1422E0
+
+// Misc Data
+#define FS_OFFSET_1000_SD_MUTEX            0xE273E8
+#define FS_OFFSET_1000_NAND_MUTEX          0xE22DA0
+#define FS_OFFSET_1000_ACTIVE_PARTITION    0xE22DE0
+#define FS_OFFSET_1000_SDMMC_DAS_HANDLE    0xE0AB90
+
+// NOPs
+#define FS_OFFSET_1000_SD_DAS_INIT         0x151CEC
+
+// Nintendo Paths
+#define FS_OFFSET_1000_NINTENDO_PATHS \
+{ \
+	{.opcode_reg = 3, .adrp_offset = 0x0006BBA4, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 3, .adrp_offset = 0x00078520, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 3, .adrp_offset = 0x0007ED0C, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 4, .adrp_offset = 0x0009115C, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 0, .adrp_offset = 0, .add_rel_offset = 0}, \
+}
+
+#endif // __FS_1000_H__

emummc/source/FS/offsets/1000_exfat.h

@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2019 m4xw <m4x@m4xw.net>
+ * Copyright (c) 2019 Atmosphere-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef __FS_1000_EXFAT_H__
+#define __FS_1000_EXFAT_H__
+
+// Accessor vtable getters
+#define FS_OFFSET_1000_EXFAT_SDMMC_ACCESSOR_GC   0x14DC90
+#define FS_OFFSET_1000_EXFAT_SDMMC_ACCESSOR_SD   0x14BDA0
+#define FS_OFFSET_1000_EXFAT_SDMMC_ACCESSOR_NAND 0x146C20
+
+// Hooks
+#define FS_OFFSET_1000_EXFAT_SDMMC_WRAPPER_READ  0x142380
+#define FS_OFFSET_1000_EXFAT_SDMMC_WRAPPER_WRITE 0x142460
+#define FS_OFFSET_1000_EXFAT_RTLD                0x634
+#define FS_OFFSET_1000_EXFAT_RTLD_DESTINATION    0x9C
+
+#define FS_OFFSET_1000_EXFAT_CLKRST_SET_MIN_V_CLK_RATE 0x1415A0
+
+// Misc funcs
+#define FS_OFFSET_1000_EXFAT_LOCK_MUTEX          0x28910
+#define FS_OFFSET_1000_EXFAT_UNLOCK_MUTEX        0x28960
+
+#define FS_OFFSET_1000_EXFAT_SDMMC_WRAPPER_CONTROLLER_CLOSE 0x1422E0
+
+// Misc Data
+#define FS_OFFSET_1000_EXFAT_SD_MUTEX            0xE353E8
+#define FS_OFFSET_1000_EXFAT_NAND_MUTEX          0xE30DA0
+#define FS_OFFSET_1000_EXFAT_ACTIVE_PARTITION    0xE30DE0
+#define FS_OFFSET_1000_EXFAT_SDMMC_DAS_HANDLE    0xE18B90
+
+// NOPs
+#define FS_OFFSET_1000_EXFAT_SD_DAS_INIT         0x151CEC
+
+// Nintendo Paths
+#define FS_OFFSET_1000_EXFAT_NINTENDO_PATHS \
+{ \
+	{.opcode_reg = 3, .adrp_offset = 0x0006BBA4, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 3, .adrp_offset = 0x00078520, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 3, .adrp_offset = 0x0007ED0C, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 4, .adrp_offset = 0x0009115C, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 0, .adrp_offset = 0, .add_rel_offset = 0}, \
+}
+
+#endif // __FS_1000_EXFAT_H__

emummc/source/emuMMC/emummc.c

@@ -36,7 +36,6 @@ sdmmc_storage_t sd_storage;
 
 // init vars
 bool custom_driver = true;
-extern const volatile emuMMC_ctx_t emuMMC_ctx;
 
 // FS funcs
 _sdmmc_accessor_gc sdmmc_accessor_gc;
@@ -344,7 +343,7 @@ uint64_t sdmmc_wrapper_controller_close(int mmc_id)
         {
             return 0;
         }
-        
+
         if (mmc_id == FS_SDMMC_EMMC)
         {
             // Close file handles and unmount

emummc/source/main.c

@@ -48,6 +48,9 @@ extern char __argdata__;
 // TODO
 static char nintendo_path[0x80] = "Nintendo";
 
+// 1.0.0 requires special path handling because it has separate album and contents paths.
+#define FS_100_ALBUM_PATH    0
+#define FS_100_CONTENTS_PATH 1
 static char nintendo_path_album_100[0x100] = "/Nintendo/Album";
 static char nintendo_path_contents_100[0x100] = "/Nintendo/Contents";
 
@@ -275,23 +278,18 @@ void setup_nintendo_paths(void)
         // 1.0.0 needs special handling because it uses two paths.
         // Do album path
         {
-            int path_len = snprintf(nintendo_path_album_100, sizeof(nintendo_path_album_100), "/%s/Album", nintendo_path);
+            snprintf(nintendo_path_album_100, sizeof(nintendo_path_album_100), "/%s/Album", nintendo_path);
             intptr_t nintendo_album_path_location = (intptr_t)&nintendo_path_album_100;
-            intptr_t album_path_location = nintendo_album_path_location + path_len - 6; // "/Album"
-            uintptr_t fs_n_adrp_opcode_location = INJECT_OFFSET(uintptr_t, fs_offsets->nintendo_paths[0].adrp_offset);
-            uintptr_t fs_adrp_opcode_location = INJECT_OFFSET(uintptr_t, fs_offsets->nintendo_paths[1].adrp_offset);
-            write_adrp_add(fs_offsets->nintendo_paths[0].opcode_reg, fs_n_adrp_opcode_location, fs_offsets->nintendo_paths[0].add_rel_offset, nintendo_album_path_location);
-            write_adrp_add(fs_offsets->nintendo_paths[1].opcode_reg, fs_adrp_opcode_location, fs_offsets->nintendo_paths[1].add_rel_offset, album_path_location);
+            uintptr_t fs_adrp_opcode_location = INJECT_OFFSET(uintptr_t, fs_offsets->nintendo_paths[FS_100_ALBUM_PATH].adrp_offset);
+            write_adrp_add(fs_offsets->nintendo_paths[FS_100_ALBUM_PATH].opcode_reg, fs_adrp_opcode_location, fs_offsets->nintendo_paths[FS_100_ALBUM_PATH].add_rel_offset, nintendo_album_path_location);
         }
+
         // Do contents path
         {
-            int path_len = snprintf(nintendo_path_contents_100, sizeof(nintendo_path_contents_100), "/%s/Contents", nintendo_path);
+            snprintf(nintendo_path_contents_100, sizeof(nintendo_path_contents_100), "/%s/Contents", nintendo_path);
             intptr_t nintendo_contents_path_location = (intptr_t)&nintendo_path_contents_100;
-            intptr_t contents_path_location = nintendo_contents_path_location + path_len - 9; // "/Contents"
-            uintptr_t fs_n_adrp_opcode_location = INJECT_OFFSET(uintptr_t, fs_offsets->nintendo_paths[2].adrp_offset);
-            uintptr_t fs_adrp_opcode_location = INJECT_OFFSET(uintptr_t, fs_offsets->nintendo_paths[3].adrp_offset);
-            write_adrp_add(fs_offsets->nintendo_paths[2].opcode_reg, fs_n_adrp_opcode_location, fs_offsets->nintendo_paths[2].add_rel_offset, nintendo_contents_path_location);
-            write_adrp_add(fs_offsets->nintendo_paths[3].opcode_reg, fs_adrp_opcode_location, fs_offsets->nintendo_paths[3].add_rel_offset, contents_path_location);
+            uintptr_t fs_adrp_opcode_location = INJECT_OFFSET(uintptr_t, fs_offsets->nintendo_paths[FS_100_CONTENTS_PATH].adrp_offset);
+            write_adrp_add(fs_offsets->nintendo_paths[FS_100_CONTENTS_PATH].opcode_reg, fs_adrp_opcode_location, fs_offsets->nintendo_paths[FS_100_CONTENTS_PATH].add_rel_offset, nintendo_contents_path_location);
         }
     }
 }

emummc/source/nx/svc.h

@@ -50,8 +50,18 @@ extern "C" {
  * @return Result code.
  * @note Syscall number 0x55.
  * @warning This is a privileged syscall. Use \ref envIsSyscallHinted to check if it is available.
+ * @warning Only exists on [10.0.0+]. For older versions use \ref svcLegacyQueryIoMapping.
  */
-Result svcQueryIoMapping(u64* virtaddr, u64 physaddr, u64 size);
+Result svcQueryIoMapping(u64* virtaddr, u64* out_size, u64 physaddr, u64 size);
+
+/**
+ * @brief Returns a virtual address mapped to a given IO range.
+ * @return Result code.
+ * @note Syscall number 0x55.
+ * @warning This is a privileged syscall. Use \ref envIsSyscallHinted to check if it is available.
+ * @warning Only exists on [1.0.0-9.2.0]. For newer versions use \ref svcQueryIoMapping.
+ */
+Result svcLegacyQueryIoMapping(u64* virtaddr, u64 physaddr, u64 size);
 
 /**
  * @brief Attaches a device address space to a device.

emummc/source/nx/svc.s

@@ -17,6 +17,15 @@
 .endm
 
 SVC_BEGIN svcQueryIoMapping
+	STP X0, X1, [SP, #-16]!
+	SVC 0x55
+	LDP X3, X4, [SP], #16
+	STR X1, [X3]
+	STR X2, [X4]
+	RET
+SVC_END
+
+SVC_BEGIN svcLegacyQueryIoMapping
 	STR X0, [SP, #-16]!
 	SVC 0x55
 	LDR X2, [SP], #16

emummc/source/utils/fatal.h

@@ -25,11 +25,12 @@ enum FatalReason
     Fatal_InvalidAccessor,
     Fatal_ReadNoAccessor,
     Fatal_WriteNoAccessor,
-    Fatal_IoMapping,
+    Fatal_IoMappingLegacy,
     Fatal_UnknownVersion,
     Fatal_BadResult,
     Fatal_GetConfig,
     Fatal_CloseAccessor,
+    Fatal_IoMapping,
     Fatal_Max
 };
 

emummc/source/utils/util.c

@@ -38,8 +38,15 @@ static inline uintptr_t _GetIoMapping(u64 io_addr, u64 io_size)
     u64 vaddr;
     u64 aligned_addr = (io_addr & ~0xFFFul);
     u64 aligned_size = io_size + (io_addr - aligned_addr);
-    if (svcQueryIoMapping(&vaddr, aligned_addr, aligned_size) != 0) {
-        fatal_abort(Fatal_IoMapping);
+    if (emuMMC_ctx.fs_ver >= FS_VER_10_0_0) {
+        u64 out_size;
+        if (svcQueryIoMapping(&vaddr, &out_size, aligned_addr, aligned_size) != 0) {
+            fatal_abort(Fatal_IoMapping);
+        }
+    } else {
+        if (svcLegacyQueryIoMapping(&vaddr, aligned_addr, aligned_size) != 0) {
+            fatal_abort(Fatal_IoMappingLegacy);
+        }
     }
     return (uintptr_t)(vaddr + (io_addr - aligned_addr));
 }

emummc/source/utils/util.h

@@ -19,6 +19,7 @@
 #define _UTIL_H_
 
 #include "types.h"
+#include "../emuMMC/emummc_ctx.h"
 
 intptr_t QueryIoMapping(u64 addr, u64 size);
 #define byte_swap_32(num) ((num >> 24) & 0xff) | ((num << 8) & 0xff0000) | \
@@ -37,4 +38,6 @@ void usleep(u64 ticks);
 void msleep(u64 milliseconds);
 void exec_cfg(u32 *base, const cfg_op_t *ops, u32 num_ops);
 
+extern volatile emuMMC_ctx_t emuMMC_ctx;
+
 #endif

exosphere/src/bootup.c

@@ -315,7 +315,7 @@ void setup_current_core_state(void) {
     uint64_t temp_reg;
 
     /* Setup system registers. */
-    SET_SYSREG(spsr_el3, 0b1111 << 6 | 0b1001); /* use EL2h+DAIF set initially, may be overwritten later. Not in official code */
+    SET_SYSREG(spsr_el3, 0b1111 << 6 | 0b0101); /* use EL2h+DAIF set initially, may be overwritten later. Not in official code */
 
     SET_SYSREG(actlr_el3, 0x73ull);
     SET_SYSREG(actlr_el2, 0x73ull);

exosphere/src/fuse.c

@@ -263,19 +263,20 @@ uint32_t fuse_get_5x_key_generation(void) {
 /* Returns the fuse version expected for the firmware. */
 uint32_t fuse_get_expected_fuse_version(uint32_t target_firmware) {
     static const uint8_t expected_versions[ATMOSPHERE_TARGET_FIRMWARE_COUNT+1] = {
-        [ATMOSPHERE_TARGET_FIRMWARE_100] = 1,
-        [ATMOSPHERE_TARGET_FIRMWARE_200] = 2,
-        [ATMOSPHERE_TARGET_FIRMWARE_300] = 3,
-     /* [ATMOSPHERE_TARGET_FIRMWARE_302] = 4, */
-        [ATMOSPHERE_TARGET_FIRMWARE_400] = 5,
-        [ATMOSPHERE_TARGET_FIRMWARE_500] = 6,
-        [ATMOSPHERE_TARGET_FIRMWARE_600] = 7,
-        [ATMOSPHERE_TARGET_FIRMWARE_620] = 8,
-        [ATMOSPHERE_TARGET_FIRMWARE_700] = 9,
-        [ATMOSPHERE_TARGET_FIRMWARE_800] = 9,
-        [ATMOSPHERE_TARGET_FIRMWARE_810] = 10,
-        [ATMOSPHERE_TARGET_FIRMWARE_900] = 11,
-        [ATMOSPHERE_TARGET_FIRMWARE_910] = 12,
+        [ATMOSPHERE_TARGET_FIRMWARE_100]  = 1,
+        [ATMOSPHERE_TARGET_FIRMWARE_200]  = 2,
+        [ATMOSPHERE_TARGET_FIRMWARE_300]  = 3,
+     /* [ATMOSPHERE_TARGET_FIRMWARE_302]  = 4, */
+        [ATMOSPHERE_TARGET_FIRMWARE_400]  = 5,
+        [ATMOSPHERE_TARGET_FIRMWARE_500]  = 6,
+        [ATMOSPHERE_TARGET_FIRMWARE_600]  = 7,
+        [ATMOSPHERE_TARGET_FIRMWARE_620]  = 8,
+        [ATMOSPHERE_TARGET_FIRMWARE_700]  = 9,
+        [ATMOSPHERE_TARGET_FIRMWARE_800]  = 9,
+        [ATMOSPHERE_TARGET_FIRMWARE_810]  = 10,
+        [ATMOSPHERE_TARGET_FIRMWARE_900]  = 11,
+        [ATMOSPHERE_TARGET_FIRMWARE_910]  = 12,
+        [ATMOSPHERE_TARGET_FIRMWARE_1000] = 13,
     };
 
     if (target_firmware > ATMOSPHERE_TARGET_FIRMWARE_COUNT) {

exosphere/src/package2.c

@@ -149,6 +149,7 @@ static void setup_se(void) {
         case ATMOSPHERE_TARGET_FIRMWARE_810:
         case ATMOSPHERE_TARGET_FIRMWARE_900:
         case ATMOSPHERE_TARGET_FIRMWARE_910:
+        case ATMOSPHERE_TARGET_FIRMWARE_1000:
             derive_new_device_keys(KEYSLOT_SWITCH_5XNEWDEVICEKEYGENKEY);
             break;
     }
@@ -338,7 +339,7 @@ static bool validate_package2_metadata(package2_meta_t *metadata) {
 
     /* Perform version checks. */
     /* We will be compatible with all package2s released before current, but not newer ones. */
-    if (metadata->version_max >= PACKAGE2_MINVER_THEORETICAL && metadata->version_min < PACKAGE2_MAXVER_910_CURRENT) {
+    if (metadata->version_max >= PACKAGE2_MINVER_THEORETICAL && metadata->version_min < PACKAGE2_MAXVER_1000_CURRENT) {
         return true;
     }
 
@@ -466,6 +467,7 @@ static void copy_warmboot_bin_to_dram() {
         case ATMOSPHERE_TARGET_FIRMWARE_810:
         case ATMOSPHERE_TARGET_FIRMWARE_900:
         case ATMOSPHERE_TARGET_FIRMWARE_910:
+        case ATMOSPHERE_TARGET_FIRMWARE_1000:
             warmboot_src = (uint8_t *)0x4003E000;
             break;
     }
@@ -551,6 +553,9 @@ void load_package2(coldboot_crt0_reloc_list_t *reloc_list) {
             case ATMOSPHERE_TARGET_FIRMWARE_910:
                 MAKE_REG32(PMC_BASE + 0x360) = 0x18C;
                 break;
+            case ATMOSPHERE_TARGET_FIRMWARE_1000:
+                MAKE_REG32(PMC_BASE + 0x360) = 0x1AD;
+                break;
         }
     }
 

exosphere/src/package2.h

@@ -73,7 +73,8 @@ static inline uintptr_t get_nx_bootloader_mailbox_base(unsigned int targetfw) {
 #define PACKAGE2_MAXVER_700_800 0xA
 #define PACKAGE2_MAXVER_810 0xB
 #define PACKAGE2_MAXVER_900 0xC
-#define PACKAGE2_MAXVER_910_CURRENT 0xD
+#define PACKAGE2_MAXVER_910_920 0xD
+#define PACKAGE2_MAXVER_1000_CURRENT 0xE
 
 #define PACKAGE2_MINVER_100 0x3
 #define PACKAGE2_MINVER_200 0x4
@@ -86,7 +87,8 @@ static inline uintptr_t get_nx_bootloader_mailbox_base(unsigned int targetfw) {
 #define PACKAGE2_MINVER_700_800 0xB
 #define PACKAGE2_MINVER_810 0xC
 #define PACKAGE2_MINVER_900 0xD
-#define PACKAGE2_MINVER_910_CURRENT 0xE
+#define PACKAGE2_MINVER_910_920 0xE
+#define PACKAGE2_MINVER_1000_CURRENT 0xF
 
 typedef struct {
     union {

exosphere/src/rsa_common.c

@@ -0,0 +1,20 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "rsa_common.h"
+
+/* Instantiate the shared RSA data inside a single translation unit. */
+rsa_shared_data_t g_rsa_shared_data = {};

exosphere/src/rsa_common.h

@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef EXOSPHERE_RSA_COMMON_H
+#define EXOSPHERE_RSA_COMMON_H
+#include <stdint.h>
+
+typedef union {
+    struct {
+        uint8_t user_data[0x100];
+    } storage_exp_mod;
+    struct {
+        uint32_t master_key_rev;
+        uint32_t type;
+        uint64_t expected_label_hash[4];
+    } unwrap_titlekey;
+} rsa_shared_data_t __attribute__((aligned(4)));
+
+_Static_assert(sizeof(rsa_shared_data_t) == 0x100);
+
+extern rsa_shared_data_t g_rsa_shared_data;
+
+#endif

exosphere/src/se.c

@@ -13,7 +13,7 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
- 
+
 #include <string.h>
 
 #include "utils.h"
@@ -47,24 +47,25 @@ void ll_init(volatile se_ll_t *ll, void *buffer, size_t size) {
         ll->addr_info.address = 0;
         ll->addr_info.size = 0;
     }
-    
+
     flush_dcache_range((uint8_t *)ll, (uint8_t *)ll + sizeof(*ll));
 }
 
 void set_security_engine_callback(unsigned int (*callback)(void)) {
-    if (callback == NULL || g_se_callback != NULL) {
-        generic_panic();
-    }
-
+    /* Set the callback. */
     g_se_callback = callback;
+
+    /* Enable SE Interrupt firing for async op. */
+    se_get_regs()->SE_INT_ENABLE = 0x10;
 }
 
 /* Fires on Security Engine operation completion. */
 void se_operation_completed(void) {
     se_get_regs()->SE_INT_ENABLE = 0;
-    if (g_se_callback != NULL) {
-        g_se_callback();
+    unsigned int (*callback)(void) = g_se_callback;
+    if (callback != NULL) {
         g_se_callback = NULL;
+        callback();
     }
 }
 
@@ -103,7 +104,7 @@ void se_validate_stored_vector(void) {
 
     uint8_t calc_vector[0x10];
     se_generate_test_vector(calc_vector);
-    
+
     /* Ensure nobody's messed with the security engine while we slept. */
     if (memcmp(calc_vector, g_se_stored_test_vector, 0x10) != 0) {
         generic_panic();
@@ -122,7 +123,7 @@ void se_generate_stored_vector(void) {
 /* Set the flags for an AES keyslot. */
 void set_aes_keyslot_flags(unsigned int keyslot, unsigned int flags) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     if (keyslot >= KEYSLOT_AES_MAX) {
         generic_panic();
     }
@@ -141,7 +142,7 @@ void set_aes_keyslot_flags(unsigned int keyslot, unsigned int flags) {
 /* Set the flags for an RSA keyslot. */
 void set_rsa_keyslot_flags(unsigned int keyslot, unsigned int flags) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     if (keyslot >= KEYSLOT_RSA_MAX) {
         generic_panic();
     }
@@ -160,7 +161,7 @@ void set_rsa_keyslot_flags(unsigned int keyslot, unsigned int flags) {
 
 void clear_aes_keyslot(unsigned int keyslot) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     if (keyslot >= KEYSLOT_AES_MAX) {
         generic_panic();
     }
@@ -174,7 +175,7 @@ void clear_aes_keyslot(unsigned int keyslot) {
 
 void clear_rsa_keyslot(unsigned int keyslot) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     if (keyslot >= KEYSLOT_RSA_MAX) {
         generic_panic();
     }
@@ -194,7 +195,7 @@ void clear_rsa_keyslot(unsigned int keyslot) {
 
 void set_aes_keyslot(unsigned int keyslot, const void *key, size_t key_size) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     if (keyslot >= KEYSLOT_AES_MAX || key_size > KEYSIZE_AES_MAX) {
         generic_panic();
     }
@@ -207,7 +208,7 @@ void set_aes_keyslot(unsigned int keyslot, const void *key, size_t key_size) {
 
 void set_rsa_keyslot(unsigned int keyslot, const void  *modulus, size_t modulus_size, const void *exponent, size_t exp_size) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     if (keyslot >= KEYSLOT_RSA_MAX || modulus_size > KEYSIZE_RSA_MAX || exp_size > KEYSIZE_RSA_MAX) {
         generic_panic();
     }
@@ -228,7 +229,7 @@ void set_rsa_keyslot(unsigned int keyslot, const void  *modulus, size_t modulus_
 
 void set_aes_keyslot_iv(unsigned int keyslot, const void *iv, size_t iv_size) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     if (keyslot >= KEYSLOT_AES_MAX || iv_size > 0x10) {
         generic_panic();
     }
@@ -241,7 +242,7 @@ void set_aes_keyslot_iv(unsigned int keyslot, const void *iv, size_t iv_size) {
 
 void clear_aes_keyslot_iv(unsigned int keyslot) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     if (keyslot >= KEYSLOT_AES_MAX) {
         generic_panic();
     }
@@ -260,7 +261,7 @@ void set_se_ctr(const void *ctr) {
 
 void decrypt_data_into_keyslot(unsigned int keyslot_dst, unsigned int keyslot_src, const void *wrapped_key, size_t wrapped_key_size) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     if (keyslot_dst >= KEYSLOT_AES_MAX || keyslot_src >= KEYSIZE_AES_MAX || wrapped_key_size > KEYSIZE_AES_MAX) {
         generic_panic();
     }
@@ -276,7 +277,7 @@ void decrypt_data_into_keyslot(unsigned int keyslot_dst, unsigned int keyslot_sr
 
 void se_aes_crypt_insecure_internal(unsigned int keyslot, uint32_t out_ll_paddr, uint32_t in_ll_paddr, size_t size, unsigned int crypt_config, bool encrypt, unsigned int (*callback)(void)) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     if (keyslot >= KEYSLOT_AES_MAX) {
         generic_panic();
     }
@@ -304,9 +305,6 @@ void se_aes_crypt_insecure_internal(unsigned int keyslot, uint32_t out_ll_paddr,
     /* Set the callback, for after the async operation. */
     set_security_engine_callback(callback);
 
-    /* Enable SE Interrupt firing for async op. */
-    se->SE_INT_ENABLE = 0x10;
-
     /* Setup Input/Output lists */
     se->SE_IN_LL_ADDR = in_ll_paddr;
     se->SE_OUT_LL_ADDR = out_ll_paddr;
@@ -338,7 +336,7 @@ void se_aes_cbc_decrypt_insecure(unsigned int keyslot, uint32_t out_ll_paddr, ui
     se_aes_crypt_insecure_internal(keyslot, out_ll_paddr, in_ll_paddr, size, 0x66, false, callback);
 }
 
-void se_exp_mod(unsigned int keyslot, void *buf, size_t size, unsigned int (*callback)(void)) {
+void se_exp_mod(unsigned int keyslot, const void *buf, size_t size, unsigned int (*callback)(void)) {
     volatile tegra_se_t *se = se_get_regs();
     uint8_t stack_buf[KEYSIZE_RSA_MAX];
 
@@ -348,7 +346,7 @@ void se_exp_mod(unsigned int keyslot, void *buf, size_t size, unsigned int (*cal
 
     /* Endian swap the input. */
     for (size_t i = 0; i < size; i++) {
-        stack_buf[i] = *((uint8_t *)buf + size - i - 1);
+        stack_buf[i] = *((const uint8_t *)buf + size - i - 1);
     }
 
     se->SE_CONFIG = (ALG_RSA | DST_RSAREG);
@@ -358,9 +356,6 @@ void se_exp_mod(unsigned int keyslot, void *buf, size_t size, unsigned int (*cal
 
     set_security_engine_callback(callback);
 
-    /* Enable SE interrupt firing for async op. */
-    se->SE_INT_ENABLE = 0x10;
-
     flush_dcache_range(stack_buf, stack_buf + KEYSIZE_RSA_MAX);
     trigger_se_rsa_op(stack_buf, size);
 
@@ -468,7 +463,7 @@ bool se_rsa2048_pss_verify(const void *signature, size_t signature_size, const v
 void trigger_se_rsa_op(void *buf, size_t size) {
     volatile tegra_se_t *se = se_get_regs();
     se_ll_t in_ll;
-    
+
     ll_init(&in_ll, (void *)buf, size);
 
     /* Set the input LL. */
@@ -491,19 +486,19 @@ void trigger_se_blocking_op(unsigned int op, void *dst, size_t dst_size, const v
 
     ll_init(&in_ll, (void *)src, src_size);
     ll_init(&out_ll, dst, dst_size);
-    
+
     __dsb_sy();
 
     /* Set the LLs. */
     se->SE_IN_LL_ADDR = (uint32_t) get_physical_address(&in_ll);
     se->SE_OUT_LL_ADDR = (uint32_t) get_physical_address(&out_ll);
-    
+
     /* Set registers for operation. */
     se->SE_ERR_STATUS = se->SE_ERR_STATUS;
     se->SE_INT_STATUS = se->SE_INT_STATUS;
     se->SE_OPERATION = op;
     (void)(se->SE_OPERATION);
-    
+
     __dsb_ish();
 
     while (!(se->SE_INT_STATUS & 0x10)) { /* Wait a while */ }
@@ -538,7 +533,7 @@ void se_perform_aes_block_operation(void *dst, size_t dst_size, const void *src,
 
 void se_aes_ctr_crypt(unsigned int keyslot, void *dst, size_t dst_size, const void *src, size_t src_size, const void *ctr, size_t ctr_size) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     if (keyslot >= KEYSLOT_AES_MAX || ctr_size != 0x10) {
         generic_panic();
     }
@@ -548,7 +543,7 @@ void se_aes_ctr_crypt(unsigned int keyslot, void *dst, size_t dst_size, const vo
     }
     if (dst_size) {
         flush_dcache_range((uint8_t *)dst, (uint8_t *)dst + dst_size);
-    }  
+    }
 
     unsigned int num_blocks = src_size >> 4;
 
@@ -576,12 +571,12 @@ void se_aes_ctr_crypt(unsigned int keyslot, void *dst, size_t dst_size, const vo
 
     if (dst_size) {
         flush_dcache_range((uint8_t *)dst, (uint8_t *)dst + dst_size);
-    }  
+    }
 }
 
 void se_aes_ecb_encrypt_block(unsigned int keyslot, void *dst, size_t dst_size, const void *src, size_t src_size, unsigned int config_high) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     if (keyslot >= KEYSLOT_AES_MAX || dst_size != 0x10 || src_size != 0x10) {
         generic_panic();
     }
@@ -606,7 +601,7 @@ void se_aes_256_ecb_encrypt_block(unsigned int keyslot, void *dst, size_t dst_si
 
 void se_aes_ecb_decrypt_block(unsigned int keyslot, void *dst, size_t dst_size, const void *src, size_t src_size) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     if (keyslot >= KEYSLOT_AES_MAX || dst_size != 0x10 || src_size != 0x10) {
         generic_panic();
     }
@@ -632,15 +627,15 @@ void shift_left_xor_rb(uint8_t *key) {
 
 void se_compute_aes_cmac(unsigned int keyslot, void *cmac, size_t cmac_size, const void *data, size_t data_size, unsigned int config_high) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     if (keyslot >= KEYSLOT_AES_MAX) {
         generic_panic();
     }
-    
+
     if (data_size) {
         flush_dcache_range((uint8_t *)data, (uint8_t *)data + data_size);
     }
-    
+
     /* Generate the derived key, to be XOR'd with final output block. */
     uint8_t derived_key[0x10] = {0};
     se_aes_ecb_encrypt_block(keyslot, derived_key, sizeof(derived_key), derived_key, sizeof(derived_key), config_high);
@@ -652,7 +647,7 @@ void se_compute_aes_cmac(unsigned int keyslot, void *cmac, size_t cmac_size, con
     se->SE_CONFIG = (ALG_AES_ENC | DST_HASHREG) | (config_high << 16);
     se->SE_CRYPTO_CONFIG = (keyslot << 24) | (0x145);
     clear_aes_keyslot_iv(keyslot);
-    
+
     unsigned int num_blocks = (data_size + 0xF) >> 4;
     /* Handle aligned blocks. */
     if (num_blocks > 1) {
@@ -660,7 +655,7 @@ void se_compute_aes_cmac(unsigned int keyslot, void *cmac, size_t cmac_size, con
         trigger_se_blocking_op(OP_START, NULL, 0, data, data_size);
         se->SE_CRYPTO_CONFIG |= 0x80;
     }
-    
+
     /* Create final block. */
     uint8_t last_block[0x10] = {0};
     if (data_size & 0xF) {
@@ -669,11 +664,11 @@ void se_compute_aes_cmac(unsigned int keyslot, void *cmac, size_t cmac_size, con
     } else if (data_size >= 0x10) {
         memcpy(last_block, data + data_size - 0x10, 0x10);
     }
-    
+
     for (unsigned int i = 0; i < 0x10; i++) {
         last_block[i] ^= derived_key[i];
     }
-    
+
     /* Perform last operation. */
     se->SE_CRYPTO_LAST_BLOCK = 0;
     flush_dcache_range(last_block, last_block + sizeof(last_block));
@@ -694,11 +689,11 @@ void se_compute_aes_256_cmac(unsigned int keyslot, void *cmac, size_t cmac_size,
 
 void se_aes_256_cbc_encrypt(unsigned int keyslot, void *dst, size_t dst_size, const void *src, size_t src_size, const void *iv) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     if (keyslot >= KEYSLOT_AES_MAX || src_size < 0x10) {
         generic_panic();
     }
-    
+
     se->SE_CONFIG = (ALG_AES_ENC | DST_MEMORY) | (0x202 << 16);
     se->SE_CRYPTO_CONFIG = (keyslot << 24) | 0x144;
     set_aes_keyslot_iv(keyslot, iv, 0x10);
@@ -709,7 +704,7 @@ void se_aes_256_cbc_encrypt(unsigned int keyslot, void *dst, size_t dst_size, co
 /* SHA256 Implementation. */
 void se_calculate_sha256(void *dst, const void *src, size_t src_size) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     /* Setup config for SHA256, size = BITS(src_size) */
     se->SE_CONFIG = (ENCMODE_SHA256 | ALG_SHA | DST_HASHREG);
     se->SE_SHA_CONFIG = 1;
@@ -721,7 +716,7 @@ void se_calculate_sha256(void *dst, const void *src, size_t src_size) {
     se->SE_SHA_MSG_LEFT[1] = 0;
     se->SE_SHA_MSG_LEFT[2] = 0;
     se->SE_SHA_MSG_LEFT[3] = 0;
-    
+
     /* Trigger the operation. */
     trigger_se_blocking_op(OP_START, NULL, 0, src, src_size);
 
@@ -734,7 +729,7 @@ void se_calculate_sha256(void *dst, const void *src, size_t src_size) {
 /* RNG API */
 void se_initialize_rng(unsigned int keyslot) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     if (keyslot >= KEYSLOT_AES_MAX) {
         generic_panic();
     }
@@ -754,7 +749,7 @@ void se_initialize_rng(unsigned int keyslot) {
 
 void se_generate_random(unsigned int keyslot, void *dst, size_t size) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     if (keyslot >= KEYSLOT_AES_MAX) {
         generic_panic();
     }
@@ -777,7 +772,7 @@ void se_generate_random(unsigned int keyslot, void *dst, size_t size) {
 /* SE context save API. */
 void se_set_in_context_save_mode(bool is_context_save_mode) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     uint32_t val = se->SE_SE_SECURITY;
     if (is_context_save_mode) {
         val |= 0x10000;
@@ -791,7 +786,7 @@ void se_set_in_context_save_mode(bool is_context_save_mode) {
 
 void se_generate_random_key(unsigned int dst_keyslot, unsigned int rng_keyslot) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     if (dst_keyslot >= KEYSLOT_AES_MAX || rng_keyslot >= KEYSLOT_AES_MAX) {
         generic_panic();
     }
@@ -801,7 +796,7 @@ void se_generate_random_key(unsigned int dst_keyslot, unsigned int rng_keyslot)
     se->SE_CRYPTO_CONFIG = (rng_keyslot << 24) | 0x108;
     se->SE_RNG_CONFIG = 4;
     se->SE_CRYPTO_LAST_BLOCK = 0;
-    
+
     /* Generate low part of key. */
     se->SE_CRYPTO_KEYTABLE_DST = (dst_keyslot << 8);
     trigger_se_blocking_op(OP_START, NULL, 0, NULL, 0);
@@ -812,7 +807,7 @@ void se_generate_random_key(unsigned int dst_keyslot, unsigned int rng_keyslot)
 
 void se_generate_srk(unsigned int srkgen_keyslot) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     se->SE_CONFIG = (ALG_RNG | DST_SRK);
     se->SE_CRYPTO_CONFIG = (srkgen_keyslot << 24) | 0x108;
     se->SE_RNG_CONFIG = 6;
@@ -847,24 +842,24 @@ void se_save_context(unsigned int srkgen_keyslot, unsigned int rng_keyslot, void
     /* Generate the SRK (context save encryption key). */
     se_generate_random_key(srkgen_keyslot, rng_keyslot);
     se_generate_srk(srkgen_keyslot);
-    
+
     flush_dcache_range(work_buf, work_buf + 0x10);
     se_generate_random(rng_keyslot, work_buf, 0x10);
     flush_dcache_range(work_buf, work_buf + 0x10);
-        
+
     /* Save random initial block. */
     se->SE_CONFIG = (ALG_AES_ENC | DST_MEMORY);
     se->SE_CTX_SAVE_CONFIG = (CTX_SAVE_SRC_MEM);
     se->SE_CRYPTO_LAST_BLOCK = 0;
     se_encrypt_with_srk(dst, 0x10, work_buf, 0x10);
-    
+
     /* Save Sticky Bits. */
     for (unsigned int i = 0; i < 0x2; i++) {
         se->SE_CTX_SAVE_CONFIG = (CTX_SAVE_SRC_STICKY_BITS) | (i << CTX_SAVE_STICKY_BIT_INDEX_SHIFT);
         se->SE_CRYPTO_LAST_BLOCK = 0;
         se_encrypt_with_srk(dst + 0x10 + (i * 0x10), 0x10, NULL, 0);
     }
-    
+
     /* Save AES Key Table. */
     for (unsigned int i = 0; i < KEYSLOT_AES_MAX; i++) {
         se->SE_CTX_SAVE_CONFIG = (CTX_SAVE_SRC_KEYTABLE_AES) | (i << CTX_SAVE_KEY_INDEX_SHIFT) | (CTX_SAVE_KEY_LOW_BITS);
@@ -874,21 +869,21 @@ void se_save_context(unsigned int srkgen_keyslot, unsigned int rng_keyslot, void
         se->SE_CRYPTO_LAST_BLOCK = 0;
         se_encrypt_with_srk(dst + 0x40 + (i * 0x20), 0x10, NULL, 0);
     }
-    
+
     /* Save AES Original IVs. */
     for (unsigned int i = 0; i < KEYSLOT_AES_MAX; i++) {
         se->SE_CTX_SAVE_CONFIG = (CTX_SAVE_SRC_KEYTABLE_AES) | (i << CTX_SAVE_KEY_INDEX_SHIFT) | (CTX_SAVE_KEY_ORIGINAL_IV);
         se->SE_CRYPTO_LAST_BLOCK = 0;
         se_encrypt_with_srk(dst + 0x230 + (i * 0x10), 0x10, NULL, 0);
     }
-    
+
     /* Save AES Updated IVs */
     for (unsigned int i = 0; i < KEYSLOT_AES_MAX; i++) {
         se->SE_CTX_SAVE_CONFIG = (CTX_SAVE_SRC_KEYTABLE_AES) | (i << CTX_SAVE_KEY_INDEX_SHIFT) | (CTX_SAVE_KEY_UPDATED_IV);
         se->SE_CRYPTO_LAST_BLOCK = 0;
         se_encrypt_with_srk(dst + 0x330 + (i * 0x10), 0x10, NULL, 0);
     }
-    
+
     /* Save RSA Keytable. */
     uint8_t *rsa_ctx_out = (uint8_t *)dst + 0x430;
     for (unsigned int rsa_key = 0; rsa_key < KEYSLOT_RSA_MAX; rsa_key++) {
@@ -901,13 +896,13 @@ void se_save_context(unsigned int srkgen_keyslot, unsigned int rng_keyslot, void
             }
         }
     }
-    
+
     /* Save "Known Pattern. " */
     static const uint8_t context_save_known_pattern[0x10] = {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f};
     se->SE_CTX_SAVE_CONFIG = (CTX_SAVE_SRC_MEM);
     se->SE_CRYPTO_LAST_BLOCK = 0;
     se_encrypt_with_srk(dst + 0x830, 0x10, context_save_known_pattern, 0x10);
-    
+
     /* Save SRK into PMC registers. */
     se->SE_CTX_SAVE_CONFIG = (CTX_SAVE_SRC_SRK);
     se->SE_CRYPTO_LAST_BLOCK = 0;

exosphere/src/se.h

@@ -13,7 +13,7 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
- 
+
 #ifndef EXOSPHERE_SE_H
 #define EXOSPHERE_SE_H
 
@@ -213,7 +213,7 @@ void se_aes_256_cbc_encrypt(unsigned int keyslot, void *dst, size_t dst_size, co
 void se_calculate_sha256(void *dst, const void *src, size_t src_size);
 
 /* RSA API */
-void se_exp_mod(unsigned int keyslot, void *buf, size_t size, unsigned int (*callback)(void));
+void se_exp_mod(unsigned int keyslot, const void *buf, size_t size, unsigned int (*callback)(void));
 void se_get_exp_mod_output(void *buf, size_t size);
 void se_synchronous_exp_mod(unsigned int keyslot, void *dst, size_t dst_size, const void *src, size_t src_size);
 bool se_rsa2048_pss_verify(const void *signature, size_t signature_size, const void *modulus, size_t modulus_size, const void *data, size_t data_size);

exosphere/src/smc_api.c

@@ -188,6 +188,7 @@ void set_version_specific_smcs(void) {
         case ATMOSPHERE_TARGET_FIRMWARE_810:
         case ATMOSPHERE_TARGET_FIRMWARE_900:
         case ATMOSPHERE_TARGET_FIRMWARE_910:
+        case ATMOSPHERE_TARGET_FIRMWARE_1000:
             /* No more LoadSecureExpModKey. */
             g_smc_user_table[0xE].handler = NULL;
             g_smc_user_table[0xC].id = 0xC300D60C;
@@ -433,19 +434,18 @@ uint32_t smc_get_result(smc_args_t *args) {
 }
 
 uint32_t smc_exp_mod_get_result(void *buf, uint64_t size) {
-    if (get_exp_mod_done() != 1) {
-        return 3;
+    uint32_t res = get_exp_mod_result();
+    if (res == 0) {
+        if (size == 0x100) {
+            se_get_exp_mod_output(buf, 0x100);
+            /* smc_exp_mod is done now. */
+            clear_user_smc_in_progress();
+            res = 0;
+        } else {
+            res = 2;
+        }
     }
-
-    if (size != 0x100) {
-        return 2;
-    }
-
-    se_get_exp_mod_output(buf, 0x100);
-
-    /* smc_exp_mod is done now. */
-    clear_user_smc_in_progress();
-    return 0;
+    return res;
 }
 
 uint32_t smc_exp_mod(smc_args_t *args) {
@@ -508,30 +508,31 @@ uint32_t smc_unwrap_rsa_oaep_wrapped_titlekey_get_result(void *buf, uint64_t siz
     uint8_t aes_wrapped_titlekey[0x10];
     uint8_t titlekey[0x10];
     uint64_t sealed_titlekey[2];
-    if (get_exp_mod_done() != 1) {
-        return 3;
+    uint32_t res = get_exp_mod_result();
+    if (res == 0) {
+        if (size == 0x10) {
+            se_get_exp_mod_output(rsa_wrapped_titlekey, 0x100);
+            if (tkey_rsa_oaep_unwrap(aes_wrapped_titlekey, 0x10, rsa_wrapped_titlekey, 0x100) == 0x10) {
+                tkey_aes_unwrap(titlekey, 0x10, aes_wrapped_titlekey, 0x10);
+                seal_titlekey(sealed_titlekey, 0x10, titlekey, 0x10);
+
+                p_sealed_key[0] = sealed_titlekey[0];
+                p_sealed_key[1] = sealed_titlekey[1];
+
+                res = 0;
+            } else {
+                /* Failed to extract RSA OAEP wrapped key. */
+                res = 2;
+            }
+
+            /* smc_unwrap_rsa_oaep_wrapped_titlekey is done now. */
+            clear_user_smc_in_progress();
+        } else {
+            res = 2;
+        }
     }
 
-    if (size != 0x10) {
-        return 2;
-    }
-
-    se_get_exp_mod_output(rsa_wrapped_titlekey, 0x100);
-    if (tkey_rsa_oaep_unwrap(aes_wrapped_titlekey, 0x10, rsa_wrapped_titlekey, 0x100) != 0x10) {
-        /* Failed to extract RSA OAEP wrapped key. */
-        clear_user_smc_in_progress();
-        return 2;
-    }
-
-    tkey_aes_unwrap(titlekey, 0x10, aes_wrapped_titlekey, 0x10);
-    seal_titlekey(sealed_titlekey, 0x10, titlekey, 0x10);
-
-    p_sealed_key[0] = sealed_titlekey[0];
-    p_sealed_key[1] = sealed_titlekey[1];
-
-    /* smc_unwrap_rsa_oaep_wrapped_titlekey is done now. */
-    clear_user_smc_in_progress();
-    return 0;
+    return res;
 }
 
 uint32_t smc_unwrap_rsa_oaep_wrapped_titlekey(smc_args_t *args) {

exosphere/src/smc_user.c

@@ -34,12 +34,93 @@
 
 /* Globals. */
 static bool g_crypt_aes_done = false;
-static bool g_exp_mod_done = false;
+static uint32_t g_exp_mod_result = 0;
 
-static uint8_t g_imported_exponents[4][0x100];
+static __attribute__((aligned(4))) uint8_t g_imported_exponents[4][0x100];
+static __attribute__((aligned(4))) uint8_t g_imported_moduli[4][0x100];
+static bool g_is_modulus_verified[4];
+
+static __attribute__((aligned(4))) const uint8_t g_rsa_public_key[4] = { 0x00, 0x01, 0x00, 0x01 };
+
+static __attribute__((aligned(4))) const uint8_t g_rsa_test_vector[0x100] = {
+    'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D',
+    'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D',
+    'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D',
+    'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D',
+    'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D',
+    'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D',
+    'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D',
+    'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D',
+    'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D',
+    'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D',
+    'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D',
+    'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D',
+    'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D',
+    'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D',
+    'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D',
+    'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D'
+};
+
+static uint32_t g_test_exp_mod_keyslot = 0;
+static uint32_t g_test_exp_mod_usecase = 0;
+static bool g_test_exp_mod_in_progress = false;
 
 static uint8_t g_rsausecase_to_cryptousecase[5] = {1, 2, 3, 5, 6};
 
+static void import_rsa_exponent(unsigned int which, const uint8_t *exponent, uint64_t size) {
+    g_is_modulus_verified[which] = false;
+    for (unsigned int i = 0; i < 0x100; i++) {
+        g_imported_exponents[which][i] = exponent[i];
+        g_imported_moduli[which][i]    = 0;
+    }
+}
+
+static void import_rsa_modulus(unsigned int which, const uint8_t *modulus, uint64_t size) {
+    uint64_t clamped_size = 0x100;
+    if (size <= 0x100) {
+        clamped_size = size;
+    }
+    if (clamped_size != 0) {
+        /* The official secure monitor implements this via bit-fiddling, */
+        /* and to prevent accidental inaccuracy we will too. */
+        /* It's probably done to prevent errors on negative sizes. */
+        uint64_t remaining = 0x100;
+        if (size != 0x100 && (~size >= ~0xFFFFFFFFFFFFFEFFULL)) {
+            remaining = size;
+        }
+        memcpy(&g_imported_moduli[which][0], modulus, remaining);
+    }
+}
+
+static bool load_imported_rsa_keypair(unsigned int keyslot, unsigned int which) {
+    if (!g_is_modulus_verified[which]) {
+        return false;
+    }
+    set_rsa_keyslot(keyslot, g_imported_moduli[which], 0x100, g_imported_exponents[which], 0x100);
+    return true;
+}
+
+static void test_rsa_modulus_public(unsigned int which, unsigned int keyslot, const uint8_t *modulus, uint64_t modulus_size, unsigned int (*callback)(void)) {
+    import_rsa_modulus(which, modulus, modulus_size);
+    set_rsa_keyslot(keyslot, modulus, modulus_size, g_rsa_public_key, 0x4);
+    se_exp_mod(keyslot, g_rsa_test_vector, 0x100, callback);
+}
+
+static void test_rsa_modulus_private(unsigned int which, unsigned int keyslot, unsigned int (*callback)(void)) {
+    uint8_t exponentiated_data[0x100];
+    se_get_exp_mod_output(exponentiated_data, sizeof(exponentiated_data));
+    set_rsa_keyslot(keyslot, g_imported_moduli[which], 0x100, g_imported_exponents[which], 0x100);
+    se_exp_mod(keyslot, exponentiated_data, 0x100, callback);
+}
+
+static void validate_rsa_result(unsigned int which) {
+    char result[0x100];
+    se_get_exp_mod_output(result, sizeof(result));
+    if (memcmp(result, g_rsa_test_vector, sizeof(result)) == 0) {
+        g_is_modulus_verified[which] = true;
+    }
+}
+
 static bool is_user_keyslot_valid(unsigned int keyslot) {
     switch (exosphere_get_target_firmware()) {
         case ATMOSPHERE_TARGET_FIRMWARE_100:
@@ -55,27 +136,45 @@ static bool is_user_keyslot_valid(unsigned int keyslot) {
         case ATMOSPHERE_TARGET_FIRMWARE_810:
         case ATMOSPHERE_TARGET_FIRMWARE_900:
         case ATMOSPHERE_TARGET_FIRMWARE_910:
+        case ATMOSPHERE_TARGET_FIRMWARE_1000:
         default:
             return keyslot <= 5;
     }
 }
 
-void set_exp_mod_done(bool done) {
-    g_exp_mod_done = done;
+void set_exp_mod_result(uint32_t result) {
+    g_exp_mod_result = result;
 }
 
-bool get_exp_mod_done(void) {
-    return g_exp_mod_done;
+uint32_t get_exp_mod_result(void) {
+    return g_exp_mod_result;
 }
 
 uint32_t exp_mod_done_handler(void) {
-    set_exp_mod_done(true);
+    set_exp_mod_result(0);
 
     se_trigger_interrupt();
 
     return 0;
 }
 
+static uint32_t test_exp_mod_done_handler(void) {
+    if (g_test_exp_mod_in_progress) {
+        g_test_exp_mod_in_progress = false;
+        test_rsa_modulus_private(g_test_exp_mod_usecase, g_test_exp_mod_keyslot, test_exp_mod_done_handler);
+    } else {
+        validate_rsa_result(g_test_exp_mod_usecase);
+        if (load_imported_rsa_keypair(g_test_exp_mod_keyslot, g_test_exp_mod_usecase)) {
+            se_exp_mod(g_test_exp_mod_keyslot, g_rsa_shared_data.storage_exp_mod.user_data, 0x100, exp_mod_done_handler);
+        } else {
+            set_exp_mod_result(2);
+            se_trigger_interrupt();
+        }
+    }
+
+    return 0;
+}
+
 uint32_t user_exp_mod(smc_args_t *args) {
     uint8_t modulus[0x100];
     uint8_t exponent[0x100];
@@ -108,7 +207,8 @@ uint32_t user_exp_mod(smc_args_t *args) {
         return 2;
     }
 
-    set_exp_mod_done(false);
+    set_exp_mod_result(3);
+
     /* Hardcode RSA keyslot 0. */
     set_rsa_keyslot(0, modulus, 0x100, exponent, exponent_size);
     se_exp_mod(0, input, 0x100, exp_mod_done_handler);
@@ -650,10 +750,21 @@ uint32_t user_secure_exp_mod(smc_args_t *args) {
         return 2;
     }
 
-    set_exp_mod_done(false);
+    set_exp_mod_result(3);
+
     /* Hardcode RSA keyslot 0. */
-    set_rsa_keyslot(0, modulus, 0x100, g_imported_exponents[exponent_id], 0x100);
-    se_exp_mod(0, input, 0x100, exp_mod_done_handler);
+    if (exosphere_get_target_firmware() < ATMOSPHERE_TARGET_FIRMWARE_1000) {
+        set_rsa_keyslot(0, modulus, 0x100, g_imported_exponents[exponent_id], 0x100);
+        se_exp_mod(0, input, 0x100, exp_mod_done_handler);
+    } else if (load_imported_rsa_keypair(0, exponent_id)) {
+        se_exp_mod(0, input, 0x100, exp_mod_done_handler);
+    } else {
+        memcpy(g_rsa_shared_data.storage_exp_mod.user_data, input, 0x100);
+        g_test_exp_mod_keyslot     = 0;
+        g_test_exp_mod_usecase     = exponent_id;
+        g_test_exp_mod_in_progress = true;
+        test_rsa_modulus_public(exponent_id, 0, modulus, 0x100, test_exp_mod_done_handler);
+    }
 
     return 0;
 }
@@ -700,7 +811,7 @@ uint32_t user_unwrap_rsa_oaep_wrapped_titlekey(smc_args_t *args) {
         return 2;
     }
 
-    set_exp_mod_done(false);
+    set_exp_mod_result(3);
 
     /* Expected label_hash occupies args->X[3] to args->X[6]. */
     tkey_set_expected_label_hash(&args->X[3]);
@@ -879,6 +990,7 @@ uint32_t user_decrypt_or_import_rsa_key(smc_args_t *args) {
     }
 
     unsigned int exponent_id;
+    bool import_modulus;
 
     switch (usecase) {
         case 0:
@@ -888,22 +1000,33 @@ uint32_t user_decrypt_or_import_rsa_key(smc_args_t *args) {
             return 0;
         case 1:
             exponent_id = 1;
+            import_modulus = false;
             break;
         case 2:
             exponent_id = 0;
+            import_modulus = true;
             break;
         case 3:
             exponent_id = 2;
+            import_modulus = false;
             break;
         case 4:
             exponent_id = 3;
+            import_modulus = true;
             break;
         default:
             generic_panic();
     }
 
-    /* Copy key to global. */
-    memcpy(g_imported_exponents[exponent_id], user_data, 0x100);
+    /* Modulus import isn't implemented on < 10.0.0. */
+    import_modulus &= (exosphere_get_target_firmware() >= ATMOSPHERE_TARGET_FIRMWARE_1000);
+
+    /* Import the key. */
+    import_rsa_exponent(exponent_id, user_data, 0x100);
+    if (import_modulus) {
+        import_rsa_modulus(exponent_id, user_data + 0x100, 0x100);
+        g_is_modulus_verified[exponent_id] = true;
+    }
     return 0;
 
 }

exosphere/src/smc_user.h

@@ -13,7 +13,7 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
- 
+
 #ifndef EXOSPHERE_SMC_USER_H
 #define EXOSPHERE_SMC_USER_H
 
@@ -41,7 +41,7 @@ uint32_t user_decrypt_or_import_rsa_key(smc_args_t *args);
 void set_crypt_aes_done(bool done);
 bool get_crypt_aes_done(void);
 
-void set_exp_mod_done(bool done);
-bool get_exp_mod_done(void);
+void set_exp_mod_result(uint32_t result);
+uint32_t get_exp_mod_result(void);
 
 #endif

exosphere/src/titlekey.c

@@ -13,7 +13,7 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
- 
+
 #include <stdint.h>
 #include <string.h>
 
@@ -25,14 +25,10 @@
 #include "masterkey.h"
 #include "se.h"
 
-static uint64_t g_tkey_expected_label_hash[4];
-static unsigned int g_tkey_master_key_rev = MASTERKEY_REVISION_MAX;
-static unsigned int g_tkey_type = 0;
-
 /* Set the expected db prefix. */
 void tkey_set_expected_label_hash(uint64_t *label_hash) {
     for (unsigned int i = 0; i < 4; i++) {
-        g_tkey_expected_label_hash[i] = label_hash[i];
+        g_rsa_shared_data.unwrap_titlekey.expected_label_hash[i] = label_hash[i];
     }
 }
 
@@ -40,7 +36,7 @@ void tkey_set_master_key_rev(unsigned int master_key_rev) {
     if (master_key_rev >= MASTERKEY_REVISION_MAX) {
         generic_panic();
     }
-    g_tkey_master_key_rev = master_key_rev;
+    g_rsa_shared_data.unwrap_titlekey.master_key_rev = master_key_rev;
 }
 
 static void tkey_validate_type(unsigned int type) {
@@ -51,7 +47,7 @@ static void tkey_validate_type(unsigned int type) {
 
 void tkey_set_type(unsigned int type) {
     tkey_validate_type(type);
-    g_tkey_type = type;
+    g_rsa_shared_data.unwrap_titlekey.type = type;
 }
 
 /* Reference for MGF1 can be found here: https://en.wikipedia.org/wiki/Mask_generation_function#MGF1 */
@@ -116,7 +112,7 @@ size_t tkey_rsa_oaep_unwrap(void *dst, size_t dst_size, void *src, size_t src_si
     uint8_t *db = message + 0x21;
 
     /* This will be passed to smc_unwrap_rsa_oaep_wrapped_titlekey. */
-    uint8_t *expected_label_hash = (uint8_t *)(&g_tkey_expected_label_hash[0]);
+    uint8_t *expected_label_hash = (uint8_t *)(&g_rsa_shared_data.unwrap_titlekey.expected_label_hash[0]);
 
     /* Unmask the salt. */
     calculate_mgf1_and_xor(salt, 0x20, db, 0xDF);
@@ -171,13 +167,13 @@ static const uint8_t titlekek_sources[TITLEKEY_TYPE_MAX+1][0x10] = {
 };
 
 void tkey_aes_unwrap(void *dst, size_t dst_size, const void *src, size_t src_size) {
-    if (g_tkey_master_key_rev >= MASTERKEY_REVISION_MAX || dst_size != 0x10 || src_size != 0x10) {
+    if (g_rsa_shared_data.unwrap_titlekey.master_key_rev >= MASTERKEY_REVISION_MAX || dst_size != 0x10 || src_size != 0x10) {
         generic_panic();
     }
-    
+
     /* Generate the appropriate titlekek into keyslot 9. */
-    unsigned int master_keyslot = mkey_get_keyslot(g_tkey_master_key_rev);
-    decrypt_data_into_keyslot(KEYSLOT_SWITCH_TEMPKEY, master_keyslot, titlekek_sources[g_tkey_type], 0x10);
+    unsigned int master_keyslot = mkey_get_keyslot(g_rsa_shared_data.unwrap_titlekey.master_key_rev);
+    decrypt_data_into_keyslot(KEYSLOT_SWITCH_TEMPKEY, master_keyslot, titlekek_sources[g_rsa_shared_data.unwrap_titlekey.type], 0x10);
 
     /* Unwrap the titlekey using the titlekek. */
     se_aes_ecb_decrypt_block(KEYSLOT_SWITCH_TEMPKEY, dst, 0x10, src, 0x10);

exosphere/src/titlekey.h

@@ -13,11 +13,12 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
- 
+
 #ifndef EXOSPHERE_TITLEKEY_H
 #define EXOSPHERE_TITLEKEY_H
 
 #include <stdint.h>
+#include "rsa_common.h"
 
 #define TITLEKEY_TYPE_MAX 0x1
 

exosphere/src/uart.c

@@ -21,12 +21,12 @@
 
 static inline void uart_wait_cycles(uint32_t baud, uint32_t num)
 {
-    wait((num * 1000000 + 16 * baud - 1) / (16 * baud));
+    udelay((num * 1000000 + 16 * baud - 1) / (16 * baud));
 }
 
 static inline void uart_wait_syms(uint32_t baud, uint32_t num)
 {
-    wait((num * 1000000 + baud - 1) / baud);
+    udelay((num * 1000000 + baud - 1) / baud);
 }
 
 void uart_config(UartDevice dev) {
@@ -34,28 +34,28 @@ void uart_config(UartDevice dev) {
 
     switch (dev) {
         case UART_A:
-            pinmux->uart1_tx = 0;
-            pinmux->uart1_rx = (PINMUX_INPUT | PINMUX_PULL_UP);
-            pinmux->uart1_rts = 0;
-            pinmux->uart1_cts = (PINMUX_INPUT | PINMUX_PULL_DOWN);
+            pinmux->uart1_tx = (0 | 0 | PINMUX_PULL_NONE | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart1_rx = (PINMUX_INPUT | PINMUX_TRISTATE | PINMUX_PULL_UP | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart1_rts = (0 | 0 | PINMUX_PULL_NONE | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart1_cts = (PINMUX_INPUT | PINMUX_TRISTATE | PINMUX_PULL_DOWN | PINMUX_SELECT_FUNCTION0);
             break;
         case UART_B:
-            pinmux->uart2_tx = 0;
-            pinmux->uart2_rx = (PINMUX_INPUT | PINMUX_PULL_UP);
-            pinmux->uart2_rts = 0;
-            pinmux->uart2_cts = (PINMUX_INPUT | PINMUX_PULL_DOWN);
+            pinmux->uart2_tx = (0 | 0 | PINMUX_PULL_NONE | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart2_rx = (PINMUX_INPUT | PINMUX_TRISTATE | PINMUX_PULL_NONE | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart2_rts = (0 | 0 | PINMUX_PULL_DOWN | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart2_cts = (PINMUX_INPUT | PINMUX_TRISTATE | PINMUX_PULL_NONE | PINMUX_SELECT_FUNCTION0);
             break;
         case UART_C:
-            pinmux->uart3_tx = 0;
-            pinmux->uart3_rx = (PINMUX_INPUT | PINMUX_PULL_UP);
-            pinmux->uart3_rts = 0;
-            pinmux->uart3_cts = (PINMUX_INPUT | PINMUX_PULL_DOWN);
+            pinmux->uart3_tx = (0 | 0 | PINMUX_PULL_NONE | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart3_rx = (PINMUX_INPUT | PINMUX_TRISTATE | PINMUX_PULL_NONE | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart3_rts = (0 | 0 | PINMUX_PULL_DOWN | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart3_cts = (PINMUX_INPUT | PINMUX_TRISTATE | PINMUX_PULL_NONE | PINMUX_SELECT_FUNCTION0);
             break;
         case UART_D:
-            pinmux->uart4_tx = 0;
-            pinmux->uart4_rx = (PINMUX_INPUT | PINMUX_PULL_UP);
-            pinmux->uart4_rts = 0;
-            pinmux->uart4_cts = (PINMUX_INPUT | PINMUX_PULL_DOWN);
+            pinmux->uart4_tx = (0 | 0 | PINMUX_PULL_DOWN | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart4_rx = (PINMUX_INPUT | PINMUX_TRISTATE | PINMUX_PULL_DOWN | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart4_rts = (0 | 0 | PINMUX_PULL_DOWN | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart4_cts = (PINMUX_INPUT | PINMUX_TRISTATE | PINMUX_PULL_DOWN | PINMUX_SELECT_FUNCTION0);
             break;
         case UART_E:
             /* Unused. */

fusee/fusee-primary/src/gpio.c

@@ -21,66 +21,58 @@
 #include "gpio.h"
 #include "utils.h"
 
-static volatile tegra_gpio_bank_t *gpio_get_bank(uint32_t pin) {
+/* Set GPIO's value. */
+static void gpio_register_set(uint32_t pin, bool do_set, uint32_t offset) {
     volatile tegra_gpio_t *gpio = gpio_get_regs();
-    uint32_t bank_number = (pin >> GPIO_BANK_SHIFT);
-
-    return &gpio->bank[bank_number];
-}
-
-static volatile uint32_t gpio_get_port(uint32_t pin) {
-    return ((pin >> GPIO_PORT_SHIFT) & GPIO_PORT_MASK);
-}
-
-static volatile uint32_t gpio_get_mask(uint32_t pin) {
-    uint32_t pin_number = (pin & GPIO_PIN_MASK);
-    return (1 << pin_number);
-}
-
-static void gpio_simple_register_set(uint32_t pin, bool should_be_set, uint32_t offset) {
+    
     /* Retrieve the register set that corresponds to the given pin and offset. */
-    uintptr_t cluster_addr = (uintptr_t)gpio_get_bank(pin) + offset;
-    uint32_t *cluster = (uint32_t *)cluster_addr;
+    volatile uint32_t *cluster = (uint32_t *)((uintptr_t)&gpio->bank[(pin >> GPIO_BANK_SHIFT)] + offset);
 
     /* Figure out the offset into the cluster, and the mask to be used. */
-    uint32_t port = gpio_get_port(pin);
-    uint32_t mask = gpio_get_mask(pin);
+    uint32_t port = ((pin >> GPIO_PORT_SHIFT) & GPIO_PORT_MASK);
+    uint32_t mask = (1 << (pin & GPIO_PIN_MASK));
 
     /* Set or clear the bit, as appropriate. */
-    if (should_be_set)
+    if (do_set)
         cluster[port] |= mask;
     else
         cluster[port] &= ~mask;
     
     /* Dummy read. */
-    (void)cluster[port];
+    cluster[port];
 }
 
-static bool gpio_simple_register_get(uint32_t pin, uint32_t offset) {
+/* Get GPIO's value. */
+static bool gpio_register_get(uint32_t pin, uint32_t offset) {
+    volatile tegra_gpio_t *gpio = gpio_get_regs();
+    
     /* Retrieve the register set that corresponds to the given pin and offset. */
-    uintptr_t cluster_addr = (uintptr_t)gpio_get_bank(pin) + offset;
-    uint32_t *cluster = (uint32_t *)cluster_addr;
+    volatile uint32_t *cluster = (uint32_t *)((uintptr_t)&gpio->bank[(pin >> GPIO_BANK_SHIFT)] + offset);
 
     /* Figure out the offset into the cluster, and the mask to be used. */
-    uint32_t port = gpio_get_port(pin);
-    uint32_t mask = gpio_get_mask(pin);
+    uint32_t port = ((pin >> GPIO_PORT_SHIFT) & GPIO_PORT_MASK);
+    uint32_t mask = (1 << (pin & GPIO_PIN_MASK));
 
     /* Convert the given value to a boolean. */
     return !!(cluster[port] & mask);
 }
 
+/* Configure GPIO's mode. */
 void gpio_configure_mode(uint32_t pin, uint32_t mode) {
-    gpio_simple_register_set(pin, mode == GPIO_MODE_GPIO, offsetof(tegra_gpio_bank_t, config));
+    gpio_register_set(pin, mode == GPIO_MODE_GPIO, offsetof(tegra_gpio_bank_t, config));
 }
 
+/* Configure GPIO's direction. */
 void gpio_configure_direction(uint32_t pin, uint32_t dir) {
-    gpio_simple_register_set(pin, dir == GPIO_DIRECTION_OUTPUT, offsetof(tegra_gpio_bank_t, direction)); 
+    gpio_register_set(pin, dir == GPIO_DIRECTION_OUTPUT, offsetof(tegra_gpio_bank_t, direction)); 
 }
 
+/* Write to GPIO. */
 void gpio_write(uint32_t pin, uint32_t value) {
-    gpio_simple_register_set(pin, value == GPIO_LEVEL_HIGH, offsetof(tegra_gpio_bank_t, out)); 
+    gpio_register_set(pin, value == GPIO_LEVEL_HIGH, offsetof(tegra_gpio_bank_t, out)); 
 }
 
+/* Read from GPIO. */
 uint32_t gpio_read(uint32_t pin) {
-    return gpio_simple_register_get(pin, offsetof(tegra_gpio_bank_t, in)); 
+    return gpio_register_get(pin, offsetof(tegra_gpio_bank_t, in)); 
 }

fusee/fusee-primary/src/panic.c

@@ -13,7 +13,7 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
- 
+
 #include <stdio.h>
 #include "panic.h"
 #include "di.h"
@@ -51,8 +51,10 @@ static const char *get_error_desc_str(uint32_t error_desc) {
 
 static void _check_and_display_atmosphere_fatal_error(void) {
     /* Check for valid magic. */
-    if (ATMOSPHERE_FATAL_ERROR_CONTEXT->magic != ATMOSPHERE_REBOOT_TO_FATAL_MAGIC &&
-        ATMOSPHERE_FATAL_ERROR_CONTEXT->magic != ATMOSPHERE_REBOOT_TO_FATAL_MAGIC_0) {
+    if (ATMOSPHERE_FATAL_ERROR_CONTEXT->magic != ATMOSPHERE_REBOOT_TO_FATAL_MAGIC   &&
+        ATMOSPHERE_FATAL_ERROR_CONTEXT->magic != ATMOSPHERE_REBOOT_TO_FATAL_MAGIC_1 &&
+        ATMOSPHERE_FATAL_ERROR_CONTEXT->magic != ATMOSPHERE_REBOOT_TO_FATAL_MAGIC_0)
+    {
         return;
     }
 
@@ -69,10 +71,10 @@ static void _check_and_display_atmosphere_fatal_error(void) {
         /* Turn on the backlight after initializing the lfb */
         /* to avoid flickering. */
         display_backlight(true);
-        
+
         /* Override the global logging level. */
         log_set_log_level(SCREEN_LOG_LEVEL_ERROR);
-    
+
         /* Copy fatal error context to the stack. */
         atmosphere_fatal_error_ctx ctx = *(ATMOSPHERE_FATAL_ERROR_CONTEXT);
 
@@ -103,7 +105,7 @@ static void _check_and_display_atmosphere_fatal_error(void) {
 void check_and_display_panic(void) {
     /* Handle a panic sent via a stratosphere module. */
     _check_and_display_atmosphere_fatal_error();
-    
+
     /* We also handle our own panics. */
     bool has_panic = ((APBDEV_PMC_RST_STATUS_0 != 0) || (g_panic_code != 0));
     uint32_t code = (g_panic_code == 0) ? APBDEV_PMC_SCRATCH200_0 : g_panic_code;
@@ -152,10 +154,10 @@ void check_and_display_panic(void) {
 
         /* Initialize the display. */
         display_init();
-        
+
         /* Fill the screen. */
         display_color_screen(color);
-        
+
         /* Wait for button and reboot. */
         wait_for_button_and_reboot();
     } else {

fusee/fusee-primary/src/panic.h

@@ -13,7 +13,7 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
- 
+
 #ifndef FUSEE_PANIC_H
 #define FUSEE_PANIC_H
 
@@ -30,6 +30,7 @@
 
 #define AMS_FATAL_ERROR_MAX_STACKTRACE 0x20
 #define AMS_FATAL_ERROR_MAX_STACKDUMP 0x100
+#define AMS_FATAL_ERROR_TLS_SIZE      0x100
 
 /* Atmosphere reboot-to-fatal-error. */
 typedef struct {
@@ -57,10 +58,13 @@ typedef struct {
     uint64_t stack_dump_size;
     uint64_t stack_trace[AMS_FATAL_ERROR_MAX_STACKTRACE];
     uint8_t stack_dump[AMS_FATAL_ERROR_MAX_STACKDUMP];
+    uint8_t tls[AMS_FATAL_ERROR_TLS_SIZE];
 } atmosphere_fatal_error_ctx;
 
+/* "AFE2" */
+#define ATMOSPHERE_REBOOT_TO_FATAL_MAGIC   0x32454641
 /* "AFE1" */
-#define ATMOSPHERE_REBOOT_TO_FATAL_MAGIC 0x31454641
+#define ATMOSPHERE_REBOOT_TO_FATAL_MAGIC_1 0x31454641
 /* "AFE0" */
 #define ATMOSPHERE_REBOOT_TO_FATAL_MAGIC_0 0x30454641
 

fusee/fusee-primary/src/sdmmc/sdmmc_core.c

@@ -615,7 +615,7 @@ static void sdmmc_autocal_run(sdmmc_t *sdmmc, SdmmcBusVoltage voltage)
     while ((sdmmc->regs->auto_cal_status & SDMMC_AUTOCAL_ACTIVE)) {
         /* Ensure we haven't timed out. */
         if (get_time_since(timebase) > SDMMC_AUTOCAL_TIMEOUT) {
-            sdmmc_error(sdmmc, "Auto-calibration timed out!");
+            sdmmc_warn(sdmmc, "Auto-calibration timed out!");
             
             /* Force a register read to refresh the clock control value. */
             sdmmc_get_sd_clock_control(sdmmc);
@@ -1314,9 +1314,9 @@ static int sdmmc_wait_busy(sdmmc_t *sdmmc)
 
 static void sdmmc_intr_enable(sdmmc_t *sdmmc)
 {
-    /* Set all error bits and enable the relevant interrupts. */
-    sdmmc->regs->int_enable |= 0x017F0000;
+    /* Enable the relevant interrupts and set all error bits. */
     sdmmc->regs->int_enable |= (TEGRA_MMC_NORINTSTSEN_CMD_COMPLETE | TEGRA_MMC_NORINTSTSEN_XFER_COMPLETE | TEGRA_MMC_NORINTSTSEN_DMA_INTERRUPT);
+    sdmmc->regs->int_enable |= 0x017F0000;
     
     /* Refresh status. */
     sdmmc->regs->int_status = sdmmc->regs->int_status;
@@ -1324,34 +1324,35 @@ static void sdmmc_intr_enable(sdmmc_t *sdmmc)
 
 static void sdmmc_intr_disable(sdmmc_t *sdmmc)
 {
-    /* Clear all error bits and the interrupts. */
+    /* Clear all error bits and disable the relevant interrupts. */
     sdmmc->regs->int_enable &= ~(0x017F0000);
     sdmmc->regs->int_enable &= ~(TEGRA_MMC_NORINTSTSEN_CMD_COMPLETE | TEGRA_MMC_NORINTSTSEN_XFER_COMPLETE | TEGRA_MMC_NORINTSTSEN_DMA_INTERRUPT);
-    
-    /* Refresh status. */
-    sdmmc->regs->int_status = sdmmc->regs->int_status;
 }
 
-static bool sdmmc_intr_check_status(sdmmc_t *sdmmc, uint16_t status_mask)
+static int sdmmc_intr_check(sdmmc_t *sdmmc, uint16_t *status_out, uint16_t status_mask)
 {
-    bool is_masked = (sdmmc->regs->int_status & status_mask);
+    uint32_t int_status = sdmmc->regs->int_status;
     
-    /* Mask status. */
-    if (is_masked)
-        sdmmc->regs->int_status &= status_mask;
+    sdmmc_debug(sdmmc, "INTSTS: %08X", int_status);
     
-    return is_masked;
-}
-   
-static bool sdmmc_intr_check_error(sdmmc_t *sdmmc)
-{
-    bool is_error = (sdmmc->regs->int_status & TEGRA_MMC_NORINTSTS_ERR_INTERRUPT);
+    /* Return the status, if necessary. */
+    if (status_out)
+        *status_out = (int_status & 0xFFFF);
     
-    /* Refresh status. */
-    if (is_error)
-        sdmmc->regs->int_status = sdmmc->regs->int_status;
+    if (int_status & TEGRA_MMC_NORINTSTS_ERR_INTERRUPT)
+    {
+        /* Acknowledge error by refreshing status. */
+        sdmmc->regs->int_status = int_status;
+        return -1;
+    }
+    else if (int_status & status_mask)
+    {
+        /* Mask the status. */
+        sdmmc->regs->int_status = (int_status & status_mask);
+        return 1;
+    }
     
-    return is_error;
+    return 0;
 }
 
 static int sdmmc_dma_init(sdmmc_t *sdmmc, sdmmc_request_t *req)
@@ -1442,15 +1443,23 @@ static int sdmmc_dma_update(sdmmc_t *sdmmc)
         /* Watch over the DMA transfer. */
         while (!is_timeout) 
         {
+            /* Check interrupts. */
+            uint16_t intr_status = 0;
+            int intr_res = sdmmc_intr_check(sdmmc, &intr_status, TEGRA_MMC_NORINTSTS_XFER_COMPLETE | TEGRA_MMC_NORINTSTS_DMA_INTERRUPT);
+            
             /* An error has been raised. Reset. */
-            if (sdmmc_intr_check_error(sdmmc))
+            if (intr_res < 0)
             {
                 sdmmc_do_sw_reset(sdmmc);
                 return 0;
             }
             
+            /* Transfer is over. */
+            if (intr_status & TEGRA_MMC_NORINTSTS_XFER_COMPLETE)
+                return 1;
+            
             /* We have a DMA interrupt. Restart the transfer where it was interrupted. */
-            if (sdmmc_intr_check_status(sdmmc, TEGRA_MMC_NORINTSTS_DMA_INTERRUPT))
+            if (intr_status & TEGRA_MMC_NORINTSTS_DMA_INTERRUPT)
             {
                 if (sdmmc->use_adma)
                 {
@@ -1467,10 +1476,6 @@ static int sdmmc_dma_update(sdmmc_t *sdmmc)
                 sdmmc->next_dma_addr += 0x80000;
             }
             
-            /* Transfer is over. */
-            if (sdmmc_intr_check_status(sdmmc, TEGRA_MMC_NORINTSTS_XFER_COMPLETE))
-                return 1;
-            
             /* Keep checking if timeout expired. */
             is_timeout = (get_time_since(timebase) > 2000000);            
         }
@@ -1526,12 +1531,15 @@ static int sdmmc_wait_for_cmd(sdmmc_t *sdmmc)
 
     /* Wait for CMD to finish. */
     while (!is_err && !is_timeout) {
+        /* Check interrupts. */
+        int intr_res = sdmmc_intr_check(sdmmc, 0, TEGRA_MMC_NORINTSTS_CMD_COMPLETE);
+        
         /* Command is done. */
-        if (sdmmc_intr_check_status(sdmmc, TEGRA_MMC_NORINTSTS_CMD_COMPLETE))
+        if (intr_res > 0)
             return 1;
         
         /* Check for any raised errors. */
-        is_err = sdmmc_intr_check_error(sdmmc);
+        is_err = (intr_res < 0);
         
         /* Keep checking if timeout expired. */
         is_timeout = (get_time_since(timebase) > 2000000);
@@ -1642,6 +1650,7 @@ int sdmmc_send_cmd(sdmmc_t *sdmmc, sdmmc_command_t *cmd, sdmmc_request_t *req, u
         is_dma = true;
         dma_blkcnt = sdmmc_dma_init(sdmmc, req);
         
+        /* Abort in case initialization failed. */
         if (!dma_blkcnt)
         {
             sdmmc_error(sdmmc, "Failed to initialize the DMA transfer!");
@@ -1669,12 +1678,14 @@ int sdmmc_send_cmd(sdmmc_t *sdmmc, sdmmc_command_t *cmd, sdmmc_request_t *req, u
         /* Save response, if necessary. */
         sdmmc_save_response(sdmmc, cmd->flags);
         
-        /* Process the DMA request. */
+        /* Update the DMA request. */
         if (req)
         {
+            /* Disable interrupts and abort in case updating failed. */
             if (!sdmmc_dma_update(sdmmc))
             {
-                sdmmc_error(sdmmc, "Failed to process the DMA transfer!");
+                sdmmc_warn(sdmmc, "Failed to update the DMA transfer!");
+                sdmmc_intr_disable(sdmmc);
                 return 0;
             }
             
@@ -1840,7 +1851,7 @@ static int sdmmc_send_tuning(sdmmc_t *sdmmc, uint32_t opcode)
     while (!is_timeout)
     {
         /* Buffer Read Ready was asserted. */
-        if (sdmmc_intr_check_status(sdmmc, TEGRA_MMC_NORINTSTSEN_BUFFER_READ_READY))
+        if (sdmmc_intr_check(sdmmc, 0, TEGRA_MMC_NORINTSTSEN_BUFFER_READ_READY) > 0)
         {
             /* Manually disable the Buffer Read Ready interrupt. */
             sdmmc->regs->int_enable &= ~(TEGRA_MMC_NORINTSTSEN_BUFFER_READ_READY);

fusee/fusee-primary/src/uart.c

@@ -34,28 +34,28 @@ void uart_config(UartDevice dev) {
 
     switch (dev) {
         case UART_A:
-            pinmux->uart1_tx = 0;
-            pinmux->uart1_rx = (PINMUX_INPUT | PINMUX_PULL_UP);
-            pinmux->uart1_rts = 0;
-            pinmux->uart1_cts = (PINMUX_INPUT | PINMUX_PULL_DOWN);
+            pinmux->uart1_tx = (0 | 0 | PINMUX_PULL_NONE | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart1_rx = (PINMUX_INPUT | PINMUX_TRISTATE | PINMUX_PULL_UP | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart1_rts = (0 | 0 | PINMUX_PULL_NONE | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart1_cts = (PINMUX_INPUT | PINMUX_TRISTATE | PINMUX_PULL_DOWN | PINMUX_SELECT_FUNCTION0);
             break;
         case UART_B:
-            pinmux->uart2_tx = 0;
-            pinmux->uart2_rx = (PINMUX_INPUT | PINMUX_PULL_UP);
-            pinmux->uart2_rts = 0;
-            pinmux->uart2_cts = (PINMUX_INPUT | PINMUX_PULL_DOWN);
+            pinmux->uart2_tx = (0 | 0 | PINMUX_PULL_NONE | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart2_rx = (PINMUX_INPUT | PINMUX_TRISTATE | PINMUX_PULL_NONE | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart2_rts = (0 | 0 | PINMUX_PULL_DOWN | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart2_cts = (PINMUX_INPUT | PINMUX_TRISTATE | PINMUX_PULL_NONE | PINMUX_SELECT_FUNCTION0);
             break;
         case UART_C:
-            pinmux->uart3_tx = 0;
-            pinmux->uart3_rx = (PINMUX_INPUT | PINMUX_PULL_UP);
-            pinmux->uart3_rts = 0;
-            pinmux->uart3_cts = (PINMUX_INPUT | PINMUX_PULL_DOWN);
+            pinmux->uart3_tx = (0 | 0 | PINMUX_PULL_NONE | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart3_rx = (PINMUX_INPUT | PINMUX_TRISTATE | PINMUX_PULL_NONE | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart3_rts = (0 | 0 | PINMUX_PULL_DOWN | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart3_cts = (PINMUX_INPUT | PINMUX_TRISTATE | PINMUX_PULL_NONE | PINMUX_SELECT_FUNCTION0);
             break;
         case UART_D:
-            pinmux->uart4_tx = 0;
-            pinmux->uart4_rx = (PINMUX_INPUT | PINMUX_PULL_UP);
-            pinmux->uart4_rts = 0;
-            pinmux->uart4_cts = (PINMUX_INPUT | PINMUX_PULL_DOWN);
+            pinmux->uart4_tx = (0 | 0 | PINMUX_PULL_DOWN | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart4_rx = (PINMUX_INPUT | PINMUX_TRISTATE | PINMUX_PULL_DOWN | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart4_rts = (0 | 0 | PINMUX_PULL_DOWN | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart4_cts = (PINMUX_INPUT | PINMUX_TRISTATE | PINMUX_PULL_DOWN | PINMUX_SELECT_FUNCTION0);
             break;
         case UART_E:
             /* Unused. */

fusee/fusee-secondary/Makefile

@@ -84,7 +84,7 @@ ifneq ($(BUILD),$(notdir $(CURDIR)))
 export OUTPUT	:=	$(CURDIR)/$(TARGET)
 export TOPDIR	:=	$(CURDIR)
 
-export KIPDIRS	:= $(AMS)/stratosphere/loader $(AMS)/stratosphere/pm $(AMS)/stratosphere/sm $(AMS)/stratosphere/boot $(AMS)/stratosphere/spl $(AMS)/stratosphere/ams_mitm
+export KIPDIRS	:= $(AMS)/stratosphere/loader $(AMS)/stratosphere/ncm $(AMS)/stratosphere/pm $(AMS)/stratosphere/sm $(AMS)/stratosphere/boot $(AMS)/stratosphere/spl $(AMS)/stratosphere/ams_mitm
 export VPATH	:=	$(foreach dir,$(SOURCES),$(CURDIR)/$(dir)) \
 			$(foreach dir,$(DATA),$(CURDIR)/$(dir)) \
 			$(AMS)/exosphere $(AMS)/exosphere/lp0fw $(AMS)/exosphere/rebootstub \
@@ -96,7 +96,7 @@ export DEPSDIR	:=	$(CURDIR)/$(BUILD)
 CFILES		:=	$(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.c)))
 CPPFILES	:=	$(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.cpp)))
 SFILES		:=	$(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.s)))
-KIPFILES 	:=	loader.kip pm.kip sm.kip ams_mitm.kip spl.kip boot.kip
+KIPFILES 	:=	loader.kip ncm.kip pm.kip sm.kip ams_mitm.kip spl.kip boot.kip
 BINFILES	:=	$(foreach dir,$(DATA),$(notdir $(wildcard $(dir)/*.*))) fusee-primary.bin \
 				exosphere.bin lp0fw.bin rebootstub.bin thermosphere.bin splash_screen.bmp \
 				sept-primary.bin sept-secondary_00.enc sept-secondary_01.enc emummc.kip \
@@ -221,12 +221,12 @@ sept_secondary_01.enc.o	sept_secondary_01_enc.h:	sept-secondary_01.enc
 #---------------------------------------------------------------------------------
 	@echo $(notdir $<)
 	@$(_bin2o)
-	
+
 sept_secondary_dev_00.enc.o	sept_secondary_dev_00_enc.h:	sept-secondary_dev_00.enc
 #---------------------------------------------------------------------------------
 	@echo $(notdir $<)
 	@$(_bin2o)
-	
+
 sept_secondary_dev_01.enc.o	sept_secondary_dev_01_enc.h:	sept-secondary_dev_01.enc
 #---------------------------------------------------------------------------------
 	@echo $(notdir $<)

fusee/fusee-secondary/linker.ld

@@ -230,6 +230,8 @@ SECTIONS
     PROVIDE(__loader_kip_size__ = loader_kip_end - loader_kip);
     PROVIDE(__lp0fw_bin_start__ = lp0fw_bin - __start__);
     PROVIDE(__lp0fw_bin_size__ = lp0fw_bin_end - lp0fw_bin);
+    PROVIDE(__ncm_kip_start__ = ncm_kip - __start__);
+    PROVIDE(__ncm_kip_size__ = ncm_kip_end - ncm_kip);
     PROVIDE(__pm_kip_start__ = pm_kip - __start__);
     PROVIDE(__pm_kip_size__ = pm_kip_end - pm_kip);
     PROVIDE(__rebootstub_bin_start__ = rebootstub_bin - __start__);

fusee/fusee-secondary/src/emummc_cfg.h

@@ -82,6 +82,9 @@ typedef enum {
     FS_VER_9_1_0,
     FS_VER_9_1_0_EXFAT,
 
+    FS_VER_10_0_0,
+    FS_VER_10_0_0_EXFAT,
+
     FS_VER_MAX,
 } emummc_fs_ver_t;
 

fusee/fusee-secondary/src/gpio.c

@@ -21,66 +21,58 @@
 #include "gpio.h"
 #include "utils.h"
 
-static volatile tegra_gpio_bank_t *gpio_get_bank(uint32_t pin) {
+/* Set GPIO's value. */
+static void gpio_register_set(uint32_t pin, bool do_set, uint32_t offset) {
     volatile tegra_gpio_t *gpio = gpio_get_regs();
-    uint32_t bank_number = (pin >> GPIO_BANK_SHIFT);
-
-    return &gpio->bank[bank_number];
-}
-
-static volatile uint32_t gpio_get_port(uint32_t pin) {
-    return ((pin >> GPIO_PORT_SHIFT) & GPIO_PORT_MASK);
-}
-
-static volatile uint32_t gpio_get_mask(uint32_t pin) {
-    uint32_t pin_number = (pin & GPIO_PIN_MASK);
-    return (1 << pin_number);
-}
-
-static void gpio_simple_register_set(uint32_t pin, bool should_be_set, uint32_t offset) {
+    
     /* Retrieve the register set that corresponds to the given pin and offset. */
-    uintptr_t cluster_addr = (uintptr_t)gpio_get_bank(pin) + offset;
-    uint32_t *cluster = (uint32_t *)cluster_addr;
+    volatile uint32_t *cluster = (uint32_t *)((uintptr_t)&gpio->bank[(pin >> GPIO_BANK_SHIFT)] + offset);
 
     /* Figure out the offset into the cluster, and the mask to be used. */
-    uint32_t port = gpio_get_port(pin);
-    uint32_t mask = gpio_get_mask(pin);
+    uint32_t port = ((pin >> GPIO_PORT_SHIFT) & GPIO_PORT_MASK);
+    uint32_t mask = (1 << (pin & GPIO_PIN_MASK));
 
     /* Set or clear the bit, as appropriate. */
-    if (should_be_set)
+    if (do_set)
         cluster[port] |= mask;
     else
         cluster[port] &= ~mask;
     
     /* Dummy read. */
-    (void)cluster[port];
+    cluster[port];
 }
 
-static bool gpio_simple_register_get(uint32_t pin, uint32_t offset) {
+/* Get GPIO's value. */
+static bool gpio_register_get(uint32_t pin, uint32_t offset) {
+    volatile tegra_gpio_t *gpio = gpio_get_regs();
+    
     /* Retrieve the register set that corresponds to the given pin and offset. */
-    uintptr_t cluster_addr = (uintptr_t)gpio_get_bank(pin) + offset;
-    uint32_t *cluster = (uint32_t *)cluster_addr;
+    volatile uint32_t *cluster = (uint32_t *)((uintptr_t)&gpio->bank[(pin >> GPIO_BANK_SHIFT)] + offset);
 
     /* Figure out the offset into the cluster, and the mask to be used. */
-    uint32_t port = gpio_get_port(pin);
-    uint32_t mask = gpio_get_mask(pin);
+    uint32_t port = ((pin >> GPIO_PORT_SHIFT) & GPIO_PORT_MASK);
+    uint32_t mask = (1 << (pin & GPIO_PIN_MASK));
 
     /* Convert the given value to a boolean. */
     return !!(cluster[port] & mask);
 }
 
+/* Configure GPIO's mode. */
 void gpio_configure_mode(uint32_t pin, uint32_t mode) {
-    gpio_simple_register_set(pin, mode == GPIO_MODE_GPIO, offsetof(tegra_gpio_bank_t, config));
+    gpio_register_set(pin, mode == GPIO_MODE_GPIO, offsetof(tegra_gpio_bank_t, config));
 }
 
+/* Configure GPIO's direction. */
 void gpio_configure_direction(uint32_t pin, uint32_t dir) {
-    gpio_simple_register_set(pin, dir == GPIO_DIRECTION_OUTPUT, offsetof(tegra_gpio_bank_t, direction)); 
+    gpio_register_set(pin, dir == GPIO_DIRECTION_OUTPUT, offsetof(tegra_gpio_bank_t, direction)); 
 }
 
+/* Write to GPIO. */
 void gpio_write(uint32_t pin, uint32_t value) {
-    gpio_simple_register_set(pin, value == GPIO_LEVEL_HIGH, offsetof(tegra_gpio_bank_t, out)); 
+    gpio_register_set(pin, value == GPIO_LEVEL_HIGH, offsetof(tegra_gpio_bank_t, out)); 
 }
 
+/* Read from GPIO. */
 uint32_t gpio_read(uint32_t pin) {
-    return gpio_simple_register_get(pin, offsetof(tegra_gpio_bank_t, in)); 
+    return gpio_register_get(pin, offsetof(tegra_gpio_bank_t, in)); 
 }

fusee/fusee-secondary/src/ips.c

@@ -417,6 +417,9 @@ static const uint8_t g_fs_hashes[FS_VER_MAX][0x8] = {
 
     "\xB5\xE7\xA6\x4C\x6F\x5C\x4F\xE3", /* FS_VER_9_1_0 */
     "\xF1\x96\xD1\x44\xD0\x44\x45\xB6", /* FS_VER_9_1_0_EXFAT */
+
+    "\x3E\xEB\xD9\xB7\xBC\xD1\xB5\xE0", /* FS_VER_10_0_0 */
+    "\x81\x7E\xA2\xB0\xB7\x02\xC1\xF3", /* FS_VER_10_0_0_EXFAT */
 };
 
 kip1_header_t *apply_kip_ips_patches(kip1_header_t *kip, size_t kip_size, emummc_fs_ver_t *out_fs_ver) {

fusee/fusee-secondary/src/kernel_patches.c

@@ -486,20 +486,77 @@ static const instruction_t MAKE_KERNEL_PATCH_NAME(900, proc_id_send)[] = {0xA9BF
 static const uint8_t MAKE_KERNEL_PATTERN_NAME(900, proc_id_recv)[] = {0x68, 0x03, 0x40, 0xF9, 0x08, 0x1D, 0x40, 0xF9, 0xE0, 0x03, 0x1B, 0xAA, 0x00, 0x01, 0x3F, 0xD6, 0xE8, 0x03, 0x17, 0x2A, 0xF7, 0x0A, 0x00, 0x11, 0x08, 0xF5, 0x7E, 0xD3};
 static const instruction_t MAKE_KERNEL_PATCH_NAME(900, proc_id_recv)[] = {0xA9BF2FEA, 0xF9404BEB, 0x2A1703EA, 0xD37EF54A, 0xF86A696A, 0x92FFFFE9, 0x8A090148, 0xD2FFFFE9, 0x8A09014A, 0xD2FFFFC9, 0xEB09015F, 0x54000100, 0xA9BF27E8, 0xF9400368, 0xF9401D08, 0xAA1B03E0, 0xD63F0100, 0xA8C127E8, 0xAA0003E8, 0xA8C12FEA, 0xAA0803E0};
 
+/*
+    stp x10, x11, [sp, #-0x10]!
+    ldr x11, [sp, #0xC0]
+    mov w10, w22
+    lsl x10, x10, #2
+    ldr x10, [x11, x10]
+    mov x9, #0x0000ffffffffffff
+    and x8, x10, x9
+    mov x9, #0xffff000000000000
+    and x10, x10, x9
+    mov x9, #0xfffe000000000000
+    cmp x10, x9
+    beq #0x20
+
+    stp x8, x9, [sp, #-0x10]!
+    ldr x8, [x23]
+    ldr x8, [x8, #0x38]
+    mov x0, x23
+    blr x8
+    ldp x8, x9, [sp],#0x10
+    mov x8, x0
+
+    ldp x10, x11, [sp],#0x10
+    mov x0, x8
+*/
+static const uint8_t MAKE_KERNEL_PATTERN_NAME(1000, proc_id_send)[] = {0xE8, 0x02, 0x40, 0xF9, 0x08, 0x1D, 0x40, 0xF9, 0xE0, 0x03, 0x17, 0xAA, 0x00, 0x01, 0x3F, 0xD6, 0x08, 0x4B, 0x36, 0x8B, 0x09, 0xFC, 0x60, 0xD3, 0x00, 0x25, 0x00, 0x29};
+static const instruction_t MAKE_KERNEL_PATCH_NAME(1000, proc_id_send)[] = {0xA9BF2FEA, 0xF94063EB, 0x2A1603EA, 0xD37EF54A, 0xF86A696A, 0x92FFFFE9, 0x8A090148, 0xD2FFFFE9, 0x8A09014A, 0xD2FFFFC9, 0xEB09015F, 0x54000100, 0xA9BF27E8, 0xF94002E8, 0xF9401D08, 0xAA1703E0, 0xD63F0100, 0xA8C127E8, 0xAA0003E8, 0xA8C12FEA, 0xAA0803E0};
+/*
+    stp x10, x11, [sp, #-0x10]!
+    ldr x11, [sp, #0xC8]
+    mov w10, w26
+    lsl x10, x10, #2
+    ldr x10, [x11, x10]
+    mov x9, #0x0000ffffffffffff
+    and x8, x10, x9
+    mov x9, #0xffff000000000000
+    and x10, x10, x9
+    mov x9, #0xfffe000000000000
+    cmp x10, x9
+    beq #0x20
+
+    stp x8, x9, [sp, #-0x10]!
+    ldr x8, [x28]
+    ldr x8, [x8, #0x38]
+    mov x0, x28
+    blr x8
+    ldp x8, x9, [sp],#0x10
+    mov x8, x0
+
+    ldp x10, x11, [sp],#0x10
+    mov x0, x8
+*/
+static const uint8_t MAKE_KERNEL_PATTERN_NAME(1000, proc_id_recv)[] = {0x88, 0x03, 0x40, 0xF9, 0x08, 0x1D, 0x40, 0xF9, 0xE0, 0x03, 0x1C, 0xAA, 0x00, 0x01, 0x3F, 0xD6, 0xE8, 0x87, 0x40, 0xF9, 0x08, 0x49, 0x3A, 0x8B, 0x09, 0xFC, 0x60, 0xD3};
+static const instruction_t MAKE_KERNEL_PATCH_NAME(1000, proc_id_recv)[] = {0xA9BF2FEA, 0xF94067EB, 0x2A1A03EA, 0xD37EF54A, 0xF86A696A, 0x92FFFFE9, 0x8A090148, 0xD2FFFFE9, 0x8A09014A, 0xD2FFFFC9, 0xEB09015F, 0x54000100, 0xA9BF27E8, 0xF9400388, 0xF9401D08, 0xAA1C03E0, 0xD63F0100, 0xA8C127E8, 0xAA0003E8, 0xA8C12FEA, 0xAA0803E0};
+
+
 /* svcControlCodeMemory Patches */
 /* b.eq -> nop */
-static const instruction_t MAKE_KERNEL_PATCH_NAME(500, svc_control_codememory)[] = {MAKE_NOP};
-static const instruction_t MAKE_KERNEL_PATCH_NAME(600, svc_control_codememory)[] = {MAKE_NOP};
-static const instruction_t MAKE_KERNEL_PATCH_NAME(700, svc_control_codememory)[] = {MAKE_NOP};
-static const instruction_t MAKE_KERNEL_PATCH_NAME(800, svc_control_codememory)[] = {MAKE_NOP};
-static const instruction_t MAKE_KERNEL_PATCH_NAME(900, svc_control_codememory)[] = {MAKE_NOP};
+static const instruction_t MAKE_KERNEL_PATCH_NAME(500,  svc_control_codememory)[] = {MAKE_NOP};
+static const instruction_t MAKE_KERNEL_PATCH_NAME(600,  svc_control_codememory)[] = {MAKE_NOP};
+static const instruction_t MAKE_KERNEL_PATCH_NAME(700,  svc_control_codememory)[] = {MAKE_NOP};
+static const instruction_t MAKE_KERNEL_PATCH_NAME(800,  svc_control_codememory)[] = {MAKE_NOP};
+static const instruction_t MAKE_KERNEL_PATCH_NAME(900,  svc_control_codememory)[] = {MAKE_NOP};
+static const instruction_t MAKE_KERNEL_PATCH_NAME(1000, svc_control_codememory)[] = {MAKE_NOP};
 
-
-static const instruction_t MAKE_KERNEL_PATCH_NAME(500, system_memory_increase)[] = {0x52A3C008}; /* MOV W8,   #0x1E000000 */
-static const instruction_t MAKE_KERNEL_PATCH_NAME(600, system_memory_increase)[] = {0x52A3B008}; /* MOV W8,   #0x1D800000 */
-static const instruction_t MAKE_KERNEL_PATCH_NAME(700, system_memory_increase)[] = {0x52A3B008}; /* MOV W8,   #0x1D800000 */
-static const instruction_t MAKE_KERNEL_PATCH_NAME(800, system_memory_increase)[] = {0x52A3B013}; /* MOV W19,  #0x1D800000 */
-static const instruction_t MAKE_KERNEL_PATCH_NAME(900, system_memory_increase)[] = {0x52A3B013}; /* MOV W19,  #0x1D800000 */
+static const instruction_t MAKE_KERNEL_PATCH_NAME(500,  system_memory_increase)[] = {0x52A3C008}; /* MOV W8,   #0x1E000000 */
+static const instruction_t MAKE_KERNEL_PATCH_NAME(600,  system_memory_increase)[] = {0x52A3B008}; /* MOV W8,   #0x1D800000 */
+static const instruction_t MAKE_KERNEL_PATCH_NAME(700,  system_memory_increase)[] = {0x52A3B008}; /* MOV W8,   #0x1D800000 */
+static const instruction_t MAKE_KERNEL_PATCH_NAME(800,  system_memory_increase)[] = {0x52A3B013}; /* MOV W19,  #0x1D800000 */
+static const instruction_t MAKE_KERNEL_PATCH_NAME(900,  system_memory_increase)[] = {0x52A3B013}; /* MOV W19,  #0x1D800000 */
+static const instruction_t MAKE_KERNEL_PATCH_NAME(1000, system_memory_increase)[] = {0x52A3B013}; /* MOV W19,  #0x1D800000 */
 
 /* Hook Definitions. */
 static const kernel_patch_t g_kernel_patches_100[] = {
@@ -735,6 +792,35 @@ static const kernel_patch_t g_kernel_patches_900[] = {
     }
 };
 
+static const kernel_patch_t g_kernel_patches_1000[] = {
+    {   /* Send Message Process ID Patch. */
+        .pattern_size = 0x1C,
+        .pattern = MAKE_KERNEL_PATTERN_NAME(1000, proc_id_send),
+        .pattern_hook_offset = 0x0,
+        .payload_num_instructions = sizeof(MAKE_KERNEL_PATCH_NAME(1000, proc_id_send))/sizeof(instruction_t),
+        .branch_back_offset = 0x10,
+        .payload = MAKE_KERNEL_PATCH_NAME(1000, proc_id_send)
+    },
+    {   /* Receive Message Process ID Patch. */
+        .pattern_size = 0x1C,
+        .pattern = MAKE_KERNEL_PATTERN_NAME(1000, proc_id_recv),
+        .pattern_hook_offset = 0x0,
+        .payload_num_instructions = sizeof(MAKE_KERNEL_PATCH_NAME(1000, proc_id_recv))/sizeof(instruction_t),
+        .branch_back_offset = 0x10,
+        .payload = MAKE_KERNEL_PATCH_NAME(1000, proc_id_recv)
+    },
+    {   /* svcControlCodeMemory Patch. */
+        .payload_num_instructions = sizeof(MAKE_KERNEL_PATCH_NAME(1000, svc_control_codememory))/sizeof(instruction_t),
+        .payload = MAKE_KERNEL_PATCH_NAME(1000, svc_control_codememory),
+        .patch_offset = 0x45DAC,
+    },
+    {   /* System Memory Increase Patch. */
+        .payload_num_instructions = sizeof(MAKE_KERNEL_PATCH_NAME(1000, system_memory_increase))/sizeof(instruction_t),
+        .payload = MAKE_KERNEL_PATCH_NAME(1000, system_memory_increase),
+        .patch_offset = 0x66950,
+    }
+};
+
 #define KERNEL_PATCHES(vers) .num_patches = sizeof(g_kernel_patches_##vers)/sizeof(kernel_patch_t), .patches = g_kernel_patches_##vers,
 
 /* Kernel Infos. */
@@ -801,6 +887,25 @@ static const kernel_info_t g_kernel_infos[] = {
         .embedded_ini_ptr = 0x180,
         .free_code_space_offset = 0x65780,
         KERNEL_PATCHES(900)
+    },
+    {   /* 9.2.0. */
+        /* NOTE: 9.2.0 has identical kernel layout to 9.0.0, so patches may be reused. */
+        .hash = {0x66, 0xE7, 0x73, 0xE7, 0xF5, 0xCB, 0x9B, 0xB8, 0x66, 0xB7, 0xB1, 0x26, 0x23, 0x02, 0x76, 0xF2, 0xD1, 0x60, 0x3E, 0x09, 0x14, 0x19, 0xC2, 0x84, 0xFA, 0x5D, 0x0F, 0x16, 0xA3, 0x65, 0xFA, 0x17},
+        .hash_offset = 0x1C0,
+        .hash_size = 0x90000 - 0x1C0,
+        .embedded_ini_offset = 0x90000,
+        .embedded_ini_ptr = 0x180,
+        .free_code_space_offset = 0x65780,
+        KERNEL_PATCHES(900)
+    },
+    {   /* 10.0.0. */
+        .hash = {0x59, 0x31, 0xE6, 0x46, 0xF7, 0xAA, 0x15, 0x59, 0x78, 0xC7, 0xB3, 0xA5, 0xFA, 0x80, 0xE2, 0xC0, 0xCA, 0x6F, 0x31, 0x97, 0x3D, 0x86, 0x8A, 0x69, 0xF3, 0xBF, 0xE6, 0xE5, 0x61, 0xA7, 0x1B, 0x5B, },
+        .hash_offset = 0x1B8,
+        .hash_size = 0x93000 - 0x1B8,
+        .embedded_ini_offset = 0x93000,
+        .embedded_ini_ptr = 0x178,
+        .free_code_space_offset = 0x67790,
+        KERNEL_PATCHES(1000)
     }
 };
 
@@ -845,7 +950,7 @@ const kernel_info_t *get_kernel_info(void *kernel, size_t size) {
     return NULL;
 }
 
-void package2_patch_kernel(void *_kernel, size_t *kernel_size, bool is_sd_kernel, void **out_ini1) {
+void package2_patch_kernel(void *_kernel, size_t *kernel_size, bool is_sd_kernel, void **out_ini1, uint32_t target_firmware) {
     const kernel_info_t *kernel_info = get_kernel_info(_kernel, *kernel_size);
     *out_ini1 = NULL;
 
@@ -866,6 +971,12 @@ void package2_patch_kernel(void *_kernel, size_t *kernel_size, bool is_sd_kernel
         const uint32_t kernel_ldr_offset = *((volatile uint64_t *)((uintptr_t)_kernel + kernel_info->embedded_ini_ptr + 8));
         memcpy((void *)((uintptr_t)_kernel + kernel_ldr_offset), kernel_ldr_bin, kernel_ldr_bin_size);
 
+        /* Set target firmware for our kernel loader. */
+        uint32_t *kldr_u32 = (uint32_t *)((uintptr_t)_kernel + kernel_ldr_offset);
+        if (kldr_u32[1] == 0x30444C4D) {
+            kldr_u32[2] = target_firmware;
+        }
+
         /* Update size. */
         *kernel_size = kernel_ldr_offset + kernel_ldr_bin_size;
 

fusee/fusee-secondary/src/kernel_patches.h

@@ -19,6 +19,6 @@
 
 #include "utils.h"
 
-void package2_patch_kernel(void *kernel, size_t *kernel_size, bool is_sd_kernel, void **out_ini1);
+void package2_patch_kernel(void *kernel, size_t *kernel_size, bool is_sd_kernel, void **out_ini1, uint32_t target_firmware);
 
 #endif

fusee/fusee-secondary/src/key_derivation.c

@@ -160,6 +160,7 @@ int derive_nx_keydata(uint32_t target_firmware, const nx_keyblob_t *keyblobs, ui
                 desired_keyblob = MASTERKEY_REVISION_900;
                 /* Fallthrough */
             case ATMOSPHERE_TARGET_FIRMWARE_910:
+            case ATMOSPHERE_TARGET_FIRMWARE_1000:
                 desired_keyblob = MASTERKEY_REVISION_910_CURRENT;
                 break;
             default:

fusee/fusee-secondary/src/nxboot.c

@@ -184,6 +184,12 @@ static int stratosphere_ini_handler(void *user, const char *section, const char
             strat_cfg->has_nogc_config = true;
             sscanf(value, "%d", &tmp);
             strat_cfg->enable_nogc = tmp != 0;
+        } else if (strcmp(name, STRATOSPHERE_ENABLE_NCM_KEY) == 0) {
+            sscanf(value, "%d", &tmp);
+            strat_cfg->ncm_enabled = tmp != 0;
+            if (strat_cfg->ncm_enabled) {
+                stratosphere_enable_ncm();
+            }
         } else {
             return 0;
         }
@@ -226,6 +232,8 @@ static uint32_t nxboot_get_target_firmware(const void *package1loader) {
                 return ATMOSPHERE_TARGET_FIRMWARE_900;
             } else if (memcmp(package1loader_header->build_timestamp, "20191021", 8) == 0) {
                 return ATMOSPHERE_TARGET_FIRMWARE_910;
+            } else if (memcmp(package1loader_header->build_timestamp, "20200303", 8) == 0) {
+                return ATMOSPHERE_TARGET_FIRMWARE_1000;
             } else {
                 fatal_error("[NXBOOT] Unable to identify package1!\n");
             }

fusee/fusee-secondary/src/package2.c

@@ -35,10 +35,9 @@
 
 static void package2_decrypt(package2_header_t *package2);
 static size_t package2_get_src_section(void **section, package2_header_t *package2, unsigned int id);
-static size_t package2_get_thermosphere(const void **thermosphere);
+static size_t package2_get_thermosphere(void **thermosphere);
 static ini1_header_t *package2_rebuild_ini1(ini1_header_t *ini1, uint32_t target_firmware, void *emummc, size_t emummc_size);
-static void package2_append_section(unsigned int id, package2_header_t *package2, const void *data, size_t size);
-static void package2_fixup_thermosphere_and_entrypoint(package2_header_t *package2);
+static void package2_append_section(unsigned int id, package2_header_t *package2, void *data, size_t size);
 static void package2_fixup_header_and_section_hashes(package2_header_t *package2, size_t size);
 
 static inline size_t align_to_4(size_t s) {
@@ -51,7 +50,7 @@ void package2_rebuild_and_copy(package2_header_t *package2, uint32_t target_firm
     void *kernel;
     size_t kernel_size;
     bool is_sd_kernel = false;
-    const void *thermosphere;
+    void *thermosphere;
     size_t thermosphere_size;
     ini1_header_t *orig_ini1, *rebuilt_ini1;
 
@@ -68,8 +67,6 @@ void package2_rebuild_and_copy(package2_header_t *package2, uint32_t target_firm
         fatal_error(u8"Error: Package2 has no unused section for Thermosphère!\n");
     }
 
-    package2->metadata.section_offsets[PACKAGE2_SECTION_UNUSED] = 0; /* base of DRAM */
-
     /* Load Kernel from SD, if possible. */
     {
         size_t sd_kernel_size = get_file_size("atmosphere/kernel.bin");
@@ -90,7 +87,7 @@ void package2_rebuild_and_copy(package2_header_t *package2, uint32_t target_firm
     }
 
     /* Perform any patches we want to the NX kernel. */
-    package2_patch_kernel(kernel, &kernel_size, is_sd_kernel, (void *)&orig_ini1);
+    package2_patch_kernel(kernel, &kernel_size, is_sd_kernel, (void *)&orig_ini1, target_firmware);
 
     /* Ensure we know where embedded INI is if present, and we don't if not. */
     if ((target_firmware < ATMOSPHERE_TARGET_FIRMWARE_800 && orig_ini1 != NULL) ||
@@ -128,9 +125,6 @@ void package2_rebuild_and_copy(package2_header_t *package2, uint32_t target_firm
     package2_append_section(PACKAGE2_SECTION_INI1, rebuilt_package2, rebuilt_ini1, rebuilt_ini1->size);
     package2_append_section(PACKAGE2_SECTION_UNUSED, rebuilt_package2, thermosphere, thermosphere_size);
 
-    /* Swap entrypoint if Thermosphère is present */
-    package2_fixup_thermosphere_and_entrypoint(rebuilt_package2);
-
     /* Fix all necessary data in the header to accomodate for the new patches. */
     package2_fixup_header_and_section_hashes(rebuilt_package2, rebuilt_package2_size);
 
@@ -238,7 +232,7 @@ static bool package2_validate_metadata(package2_meta_t *metadata, uint8_t data[]
 
     /* Perform version checks. */
     /* We will be compatible with all package2s released before current, but not newer ones. */
-    if (metadata->version_max >= PACKAGE2_MINVER_THEORETICAL && metadata->version_min < PACKAGE2_MAXVER_910_CURRENT) {
+    if (metadata->version_max >= PACKAGE2_MINVER_THEORETICAL && metadata->version_min < PACKAGE2_MAXVER_1000_CURRENT) {
         return true;
     }
 
@@ -315,9 +309,12 @@ static size_t package2_get_src_section(void **section, package2_header_t *packag
     return package2->metadata.section_sizes[id];
 }
 
-static size_t package2_get_thermosphere(const void **thermosphere) {
-    (*thermosphere) = thermosphere_bin;
-    return thermosphere_bin_size;
+static size_t package2_get_thermosphere(void **thermosphere) {
+    /*extern const uint8_t thermosphere_bin[];
+    extern const uint32_t thermosphere_bin_size;*/
+    /* TODO: enable when tested. */
+    (*thermosphere) = NULL;
+    return 0;
 }
 
 static ini1_header_t *package2_rebuild_ini1(ini1_header_t *ini1, uint32_t target_firmware, void *emummc, size_t emummc_size) {
@@ -338,7 +335,7 @@ static ini1_header_t *package2_rebuild_ini1(ini1_header_t *ini1, uint32_t target
     return merged;
 }
 
-static void package2_append_section(unsigned int id, package2_header_t *package2, const void *data, size_t size) {
+static void package2_append_section(unsigned int id, package2_header_t *package2, void *data, size_t size) {
     /* This function must be called in ascending order of id. */
     /* We assume that the loading address doesn't need to be changed. */
     uint8_t *dst = package2->data;
@@ -350,22 +347,6 @@ static void package2_append_section(unsigned int id, package2_header_t *package2
     package2->metadata.section_sizes[id] = align_to_4(size);
 }
 
-static void package2_fixup_thermosphere_and_entrypoint(package2_header_t *package2) {
-    /* Return if Thermosphère is not present */
-    if (package2->metadata.section_sizes[PACKAGE2_SECTION_UNUSED] == 0) {
-        return;
-    }
-
-    uint8_t *dst = package2->data;
-    for (unsigned int i = 0; i < PACKAGE2_SECTION_UNUSED; i++) {
-        dst += package2->metadata.section_sizes[i];
-    }
-
-    /* Swap kernel entrypoint with Thermosphère */
-    *(uint64_t *)(dst + 8) = DRAM_BASE_PHYSICAL + package2->metadata.entrypoint;
-    package2->metadata.entrypoint = 0;
-}
-
 static void package2_fixup_header_and_section_hashes(package2_header_t *package2, size_t size) {
     uint8_t *data = package2->data;
 

fusee/fusee-secondary/src/package2.h

@@ -39,7 +39,8 @@
 #define PACKAGE2_MAXVER_700_800 0xA
 #define PACKAGE2_MAXVER_810 0xB
 #define PACKAGE2_MAXVER_900 0xC
-#define PACKAGE2_MAXVER_910_CURRENT 0xD
+#define PACKAGE2_MAXVER_910_920 0xD
+#define PACKAGE2_MAXVER_1000_CURRENT 0xE
 
 #define PACKAGE2_MINVER_100 0x3
 #define PACKAGE2_MINVER_200 0x4
@@ -52,10 +53,10 @@
 #define PACKAGE2_MINVER_700_800 0xB
 #define PACKAGE2_MINVER_810 0xC
 #define PACKAGE2_MINVER_900 0xD
-#define PACKAGE2_MINVER_910_CURRENT 0xE
+#define PACKAGE2_MINVER_910_920 0xE
+#define PACKAGE2_MINVER_1000_CURRENT 0xF
 
 #define NX_BOOTLOADER_PACKAGE2_LOAD_ADDRESS ((void *)(0xA9800000ull))
-#define DRAM_BASE_PHYSICAL (0x80000000)
 
 typedef struct {
     union {

fusee/fusee-secondary/src/sdmmc/sdmmc_core.c

@@ -615,7 +615,7 @@ static void sdmmc_autocal_run(sdmmc_t *sdmmc, SdmmcBusVoltage voltage)
     while ((sdmmc->regs->auto_cal_status & SDMMC_AUTOCAL_ACTIVE)) {
         /* Ensure we haven't timed out. */
         if (get_time_since(timebase) > SDMMC_AUTOCAL_TIMEOUT) {
-            sdmmc_error(sdmmc, "Auto-calibration timed out!");
+            sdmmc_warn(sdmmc, "Auto-calibration timed out!");
             
             /* Force a register read to refresh the clock control value. */
             sdmmc_get_sd_clock_control(sdmmc);
@@ -1314,9 +1314,9 @@ static int sdmmc_wait_busy(sdmmc_t *sdmmc)
 
 static void sdmmc_intr_enable(sdmmc_t *sdmmc)
 {
-    /* Set all error bits and enable the relevant interrupts. */
-    sdmmc->regs->int_enable |= 0x017F0000;
+    /* Enable the relevant interrupts and set all error bits. */
     sdmmc->regs->int_enable |= (TEGRA_MMC_NORINTSTSEN_CMD_COMPLETE | TEGRA_MMC_NORINTSTSEN_XFER_COMPLETE | TEGRA_MMC_NORINTSTSEN_DMA_INTERRUPT);
+    sdmmc->regs->int_enable |= 0x017F0000;
     
     /* Refresh status. */
     sdmmc->regs->int_status = sdmmc->regs->int_status;
@@ -1324,34 +1324,35 @@ static void sdmmc_intr_enable(sdmmc_t *sdmmc)
 
 static void sdmmc_intr_disable(sdmmc_t *sdmmc)
 {
-    /* Clear all error bits and the interrupts. */
+    /* Clear all error bits and disable the relevant interrupts. */
     sdmmc->regs->int_enable &= ~(0x017F0000);
     sdmmc->regs->int_enable &= ~(TEGRA_MMC_NORINTSTSEN_CMD_COMPLETE | TEGRA_MMC_NORINTSTSEN_XFER_COMPLETE | TEGRA_MMC_NORINTSTSEN_DMA_INTERRUPT);
-    
-    /* Refresh status. */
-    sdmmc->regs->int_status = sdmmc->regs->int_status;
 }
 
-static bool sdmmc_intr_check_status(sdmmc_t *sdmmc, uint16_t status_mask)
+static int sdmmc_intr_check(sdmmc_t *sdmmc, uint16_t *status_out, uint16_t status_mask)
 {
-    bool is_masked = (sdmmc->regs->int_status & status_mask);
+    uint32_t int_status = sdmmc->regs->int_status;
     
-    /* Mask status. */
-    if (is_masked)
-        sdmmc->regs->int_status &= status_mask;
+    sdmmc_debug(sdmmc, "INTSTS: %08X", int_status);
     
-    return is_masked;
-}
-   
-static bool sdmmc_intr_check_error(sdmmc_t *sdmmc)
-{
-    bool is_error = (sdmmc->regs->int_status & TEGRA_MMC_NORINTSTS_ERR_INTERRUPT);
+    /* Return the status, if necessary. */
+    if (status_out)
+        *status_out = (int_status & 0xFFFF);
     
-    /* Refresh status. */
-    if (is_error)
-        sdmmc->regs->int_status = sdmmc->regs->int_status;
+    if (int_status & TEGRA_MMC_NORINTSTS_ERR_INTERRUPT)
+    {
+        /* Acknowledge error by refreshing status. */
+        sdmmc->regs->int_status = int_status;
+        return -1;
+    }
+    else if (int_status & status_mask)
+    {
+        /* Mask the status. */
+        sdmmc->regs->int_status = (int_status & status_mask);
+        return 1;
+    }
     
-    return is_error;
+    return 0;
 }
 
 static int sdmmc_dma_init(sdmmc_t *sdmmc, sdmmc_request_t *req)
@@ -1442,15 +1443,23 @@ static int sdmmc_dma_update(sdmmc_t *sdmmc)
         /* Watch over the DMA transfer. */
         while (!is_timeout) 
         {
+            /* Check interrupts. */
+            uint16_t intr_status = 0;
+            int intr_res = sdmmc_intr_check(sdmmc, &intr_status, TEGRA_MMC_NORINTSTS_XFER_COMPLETE | TEGRA_MMC_NORINTSTS_DMA_INTERRUPT);
+            
             /* An error has been raised. Reset. */
-            if (sdmmc_intr_check_error(sdmmc))
+            if (intr_res < 0)
             {
                 sdmmc_do_sw_reset(sdmmc);
                 return 0;
             }
             
+            /* Transfer is over. */
+            if (intr_status & TEGRA_MMC_NORINTSTS_XFER_COMPLETE)
+                return 1;
+            
             /* We have a DMA interrupt. Restart the transfer where it was interrupted. */
-            if (sdmmc_intr_check_status(sdmmc, TEGRA_MMC_NORINTSTS_DMA_INTERRUPT))
+            if (intr_status & TEGRA_MMC_NORINTSTS_DMA_INTERRUPT)
             {
                 if (sdmmc->use_adma)
                 {
@@ -1467,10 +1476,6 @@ static int sdmmc_dma_update(sdmmc_t *sdmmc)
                 sdmmc->next_dma_addr += 0x80000;
             }
             
-            /* Transfer is over. */
-            if (sdmmc_intr_check_status(sdmmc, TEGRA_MMC_NORINTSTS_XFER_COMPLETE))
-                return 1;
-            
             /* Keep checking if timeout expired. */
             is_timeout = (get_time_since(timebase) > 2000000);            
         }
@@ -1526,12 +1531,15 @@ static int sdmmc_wait_for_cmd(sdmmc_t *sdmmc)
 
     /* Wait for CMD to finish. */
     while (!is_err && !is_timeout) {
+        /* Check interrupts. */
+        int intr_res = sdmmc_intr_check(sdmmc, 0, TEGRA_MMC_NORINTSTS_CMD_COMPLETE);
+        
         /* Command is done. */
-        if (sdmmc_intr_check_status(sdmmc, TEGRA_MMC_NORINTSTS_CMD_COMPLETE))
+        if (intr_res > 0)
             return 1;
         
         /* Check for any raised errors. */
-        is_err = sdmmc_intr_check_error(sdmmc);
+        is_err = (intr_res < 0);
         
         /* Keep checking if timeout expired. */
         is_timeout = (get_time_since(timebase) > 2000000);
@@ -1642,6 +1650,7 @@ int sdmmc_send_cmd(sdmmc_t *sdmmc, sdmmc_command_t *cmd, sdmmc_request_t *req, u
         is_dma = true;
         dma_blkcnt = sdmmc_dma_init(sdmmc, req);
         
+        /* Abort in case initialization failed. */
         if (!dma_blkcnt)
         {
             sdmmc_error(sdmmc, "Failed to initialize the DMA transfer!");
@@ -1669,12 +1678,14 @@ int sdmmc_send_cmd(sdmmc_t *sdmmc, sdmmc_command_t *cmd, sdmmc_request_t *req, u
         /* Save response, if necessary. */
         sdmmc_save_response(sdmmc, cmd->flags);
         
-        /* Process the DMA request. */
+        /* Update the DMA request. */
         if (req)
         {
+            /* Disable interrupts and abort in case updating failed. */
             if (!sdmmc_dma_update(sdmmc))
             {
-                sdmmc_error(sdmmc, "Failed to process the DMA transfer!");
+                sdmmc_warn(sdmmc, "Failed to update the DMA transfer!");
+                sdmmc_intr_disable(sdmmc);
                 return 0;
             }
             
@@ -1840,7 +1851,7 @@ static int sdmmc_send_tuning(sdmmc_t *sdmmc, uint32_t opcode)
     while (!is_timeout)
     {
         /* Buffer Read Ready was asserted. */
-        if (sdmmc_intr_check_status(sdmmc, TEGRA_MMC_NORINTSTSEN_BUFFER_READ_READY))
+        if (sdmmc_intr_check(sdmmc, 0, TEGRA_MMC_NORINTSTSEN_BUFFER_READ_READY) > 0)
         {
             /* Manually disable the Buffer Read Ready interrupt. */
             sdmmc->regs->int_enable &= ~(TEGRA_MMC_NORINTSTSEN_BUFFER_READ_READY);

fusee/fusee-secondary/src/start.s

@@ -97,11 +97,18 @@ _metadata:
 #define CONTENT_TYPE_KLD 9
 #define CONTENT_TYPE_KRN 10
 
+#define CONTENT_FLAG_NONE          (0 << 0)
+
+#define CONTENT_FLAG0_EXPERIMENTAL (1 << 0)
+
 _content_headers:
 /* ams_mitm content header */
 .word __ams_mitm_kip_start__
 .word __ams_mitm_kip_size__
-.word CONTENT_TYPE_KIP
+.byte CONTENT_TYPE_KIP
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
 .word 0xCCCCCCCC
 .asciz "ams_mitm"
 .align 5
@@ -109,7 +116,10 @@ _content_headers:
 /* boot content header */
 .word __boot_kip_start__
 .word __boot_kip_size__
-.word CONTENT_TYPE_KIP
+.byte CONTENT_TYPE_KIP
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
 .word 0xCCCCCCCC
 .asciz "boot"
 .align 5
@@ -117,7 +127,10 @@ _content_headers:
 /* exosphere content header */
 .word __exosphere_bin_start__
 .word __exosphere_bin_size__
-.word CONTENT_TYPE_EXO
+.byte CONTENT_TYPE_EXO
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
 .word 0xCCCCCCCC
 .asciz "exosphere"
 .align 5
@@ -125,7 +138,10 @@ _content_headers:
 /* fusee_primary content header */
 .word __fusee_primary_bin_start__
 .word __fusee_primary_bin_size__
-.word CONTENT_TYPE_FSP
+.byte CONTENT_TYPE_FSP
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
 .word 0xCCCCCCCC
 .asciz "fusee_primary"
 .align 5
@@ -133,15 +149,21 @@ _content_headers:
 /* loader content header */
 .word __loader_kip_start__
 .word __loader_kip_size__
-.word CONTENT_TYPE_KIP
+.byte CONTENT_TYPE_KIP
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
 .word 0xCCCCCCCC
-.asciz "loader"
+.asciz "Loader"
 .align 5
 
 /* lp0fw content header */
 .word __lp0fw_bin_start__
 .word __lp0fw_bin_size__
-.word CONTENT_TYPE_WBT
+.byte CONTENT_TYPE_WBT
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
 .word 0xCCCCCCCC
 .asciz "lp0fw"
 .align 5
@@ -149,15 +171,21 @@ _content_headers:
 /* pm content header */
 .word __pm_kip_start__
 .word __pm_kip_size__
-.word CONTENT_TYPE_KIP
+.byte CONTENT_TYPE_KIP
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
 .word 0xCCCCCCCC
-.asciz "pm"
+.asciz "ProcessManager"
 .align 5
 
 /* rebootstub content header */
 .word __rebootstub_bin_start__
 .word __rebootstub_bin_size__
-.word CONTENT_TYPE_RBT
+.byte CONTENT_TYPE_RBT
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
 .word 0xCCCCCCCC
 .asciz "rebootstub"
 .align 5
@@ -165,7 +193,10 @@ _content_headers:
 /* sept_primary content header */
 .word __sept_primary_bin_start__
 .word __sept_primary_bin_size__
-.word CONTENT_TYPE_SP1
+.byte CONTENT_TYPE_SP1
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
 .word 0xCCCCCCCC
 .asciz "sept_primary"
 .align 5
@@ -173,7 +204,10 @@ _content_headers:
 /* sept_secondary 00 content header */
 .word __sept_secondary_00_enc_start__
 .word __sept_secondary_00_enc_size__
-.word CONTENT_TYPE_SP2
+.byte CONTENT_TYPE_SP2
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
 .word 0xCCCCCCCC
 .asciz "septsecondary00"
 .align 5
@@ -181,7 +215,10 @@ _content_headers:
 /* sept_secondary 01 content header */
 .word __sept_secondary_01_enc_start__
 .word __sept_secondary_01_enc_size__
-.word CONTENT_TYPE_SP2
+.byte CONTENT_TYPE_SP2
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
 .word 0xCCCCCCCC
 .asciz "septsecondary01"
 .align 5
@@ -189,7 +226,10 @@ _content_headers:
 /* sm content header */
 .word __sm_kip_start__
 .word __sm_kip_size__
-.word CONTENT_TYPE_KIP
+.byte CONTENT_TYPE_KIP
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
 .word 0xCCCCCCCC
 .asciz "sm"
 .align 5
@@ -197,15 +237,32 @@ _content_headers:
 /* spl content header */
 .word __spl_kip_start__
 .word __spl_kip_size__
-.word CONTENT_TYPE_KIP
+.byte CONTENT_TYPE_KIP
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
 .word 0xCCCCCCCC
 .asciz "spl"
 .align 5
 
+/* ncm content header */
+.word __ncm_kip_start__
+.word __ncm_kip_size__
+.byte CONTENT_TYPE_KIP
+.byte CONTENT_FLAG0_EXPERIMENTAL
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
+.word 0xCCCCCCCC
+.asciz "NCM"
+.align 5
+
 /* emummc content header */
 .word __emummc_kip_start__
 .word __emummc_kip_size__
-.word CONTENT_TYPE_EMC
+.byte CONTENT_TYPE_EMC
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
 .word 0xCCCCCCCC
 .asciz "emummc"
 .align 5
@@ -213,7 +270,10 @@ _content_headers:
 /* kernel_ldr content header */
 .word __kernel_ldr_bin_start__
 .word __kernel_ldr_bin_size__
-.word CONTENT_TYPE_KLD
+.byte CONTENT_TYPE_KLD
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
 .word 0xCCCCCCCC
 .asciz "kernel_ldr"
 .align 5
@@ -221,7 +281,10 @@ _content_headers:
 /* splash_screen content header */
 .word __splash_screen_bmp_start__
 .word __splash_screen_bmp_size__
-.word CONTENT_TYPE_BMP
+.byte CONTENT_TYPE_BMP
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
 .word 0xCCCCCCCC
 .asciz "splash_screen"
 .align 5

fusee/fusee-secondary/src/stratosphere.c

@@ -13,7 +13,7 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
- 
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <stdint.h>
@@ -30,6 +30,7 @@
 #define u8 uint8_t
 #define u32 uint32_t
 #include "loader_kip.h"
+#include "ncm_kip.h"
 #include "pm_kip.h"
 #include "sm_kip.h"
 #include "ams_mitm_kip.h"
@@ -47,9 +48,10 @@ static bool g_stratosphere_pm_enabled = true;
 static bool g_stratosphere_ams_mitm_enabled = true;
 static bool g_stratosphere_spl_enabled = true;
 static bool g_stratosphere_boot_enabled = true;
+static bool g_stratosphere_ncm_enabled = false;
 
-extern const uint8_t loader_kip[], pm_kip[], sm_kip[], spl_kip[], boot_kip[], ams_mitm_kip[];
-extern const uint32_t loader_kip_size, pm_kip_size, sm_kip_size, spl_kip_size, boot_kip_size, ams_mitm_kip_size;
+extern const uint8_t loader_kip[], pm_kip[], sm_kip[], spl_kip[], boot_kip[], ncm_kip[], ams_mitm_kip[];
+extern const uint32_t loader_kip_size, pm_kip_size, sm_kip_size, spl_kip_size, boot_kip_size, ncm_kip_size, ams_mitm_kip_size;
 
 static emummc_fs_ver_t g_fs_ver = FS_VER_1_0_0;
 
@@ -57,6 +59,19 @@ emummc_fs_ver_t stratosphere_get_fs_version(void) {
     return g_fs_ver;
 }
 
+void stratosphere_enable_ncm(void) {
+    /* The Atmosphere team believes our implementation of NCM to be extremely accurate, */
+    /* and does not think it likely there is any possibility of undesirable behavior */
+    /* when using the NCM reimplementation. However, because NCM manages critical save games */
+    /* the implementation will default to off for some time, until the code has been thoroughly */
+    /* tested in practice. */
+
+    /* PLEASE NOTE: The default behavior will be NCM on in a future atmosphere release, */
+    /* and this opt-in functionality will be removed at that time. */
+
+    g_stratosphere_ncm_enabled = true;
+}
+
 /* GCC doesn't consider the size as const... we have to write it ourselves. */
 
 ini1_header_t *stratosphere_get_ini1(uint32_t target_firmware) {
@@ -68,38 +83,43 @@ ini1_header_t *stratosphere_get_ini1(uint32_t target_firmware) {
     }
 
     size_t size = sizeof(ini1_header_t);
-    
+
     /* Calculate our processes' sizes. */
     if (g_stratosphere_loader_enabled) {
         size += loader_kip_size;
         num_processes++;
     }
-    
+
     if (g_stratosphere_pm_enabled) {
         size += pm_kip_size;
         num_processes++;
     }
-    
+
     if (g_stratosphere_sm_enabled) {
         size += sm_kip_size;
         num_processes++;
     }
-    
+
     if (g_stratosphere_spl_enabled) {
         size += spl_kip_size;
         num_processes++;
     }
-    
+
     if (g_stratosphere_ams_mitm_enabled) {
         size += ams_mitm_kip_size;
         num_processes++;
     }
-    
+
+    if (g_stratosphere_ncm_enabled) {
+        size += ncm_kip_size;
+        num_processes++;
+    }
+
     if (g_stratosphere_boot_enabled) {
         size += boot_kip_size;
         num_processes++;
     }
-    
+
     g_stratosphere_ini1 = (ini1_header_t *)malloc(size);
 
     if (g_stratosphere_ini1 == NULL) {
@@ -134,6 +154,11 @@ ini1_header_t *stratosphere_get_ini1(uint32_t target_firmware) {
         data += spl_kip_size;
     }
 
+    if (g_stratosphere_ncm_enabled) {
+        memcpy(data, ncm_kip, ncm_kip_size);
+        data += ncm_kip_size;
+    }
+
     if (g_stratosphere_ams_mitm_enabled) {
         memcpy(data, ams_mitm_kip, ams_mitm_kip_size);
         data += ams_mitm_kip_size;
@@ -143,7 +168,7 @@ ini1_header_t *stratosphere_get_ini1(uint32_t target_firmware) {
         memcpy(data, boot_kip, boot_kip_size);
         data += boot_kip_size;
     }
-    
+
     return g_stratosphere_ini1;
 }
 
@@ -160,26 +185,26 @@ void stratosphere_free_ini1(void) {
 
 static void try_add_sd_kip(ini1_header_t *ini1, const char *kip_path) {
     size_t file_size = get_file_size(kip_path);
-    
+
     if (ini1->size + file_size > PACKAGE2_SIZE_MAX) {
         fatal_error("Failed to load %s: INI1 would be too large!\n", kip_path);
     }
-    
+
     kip1_header_t kip_header;
     if (read_from_file(&kip_header, sizeof(kip_header), kip_path) != sizeof(kip_header) || kip_header.magic != MAGIC_KIP1) {
         return;
     }
-    
+
     size_t kip_size = kip1_get_size_from_header(&kip_header);
     if (kip_size > file_size) {
         fatal_error("Failed to load %s: KIP size is corrupt!\n", kip_path);
     }
-    
+
     if (read_from_file(ini1->kip_data + ini1->size - sizeof(ini1_header_t), kip_size, kip_path) != kip_size) {
         /* TODO: is this error fatal? */
         return;
     }
-    
+
     ini1->size += kip_size;
     ini1->num_processes++;
 }
@@ -189,24 +214,24 @@ static kip1_header_t *inject_emummc_kip(kip1_header_t *fs_kip, kip1_header_t *em
     size_t fs_kip_size, emummc_kip_size;
     fs_kip = kip1_uncompress(fs_kip, &fs_kip_size);
     emummc_kip = kip1_uncompress(emummc_kip, &emummc_kip_size);
-    
+
     /* Allocate kip. */
     kip1_header_t *injected_kip = calloc(1, fs_kip_size + emummc_kip_size);
     if (injected_kip == NULL) {
         fatal_error("Failed to allocate memory for emummc kip injection!");
     }
     memcpy(injected_kip, fs_kip, sizeof(*fs_kip));
-    
+
     const size_t fs_contents_size = kip1_get_size_from_header(fs_kip) - sizeof(kip1_header_t);
-    
+
     const size_t emummc_data_size = emummc_kip->section_headers[3].out_offset + emummc_kip->section_headers[3].out_size;
     if (emummc_data_size & 0xFFF) {
         fatal_error("Invalid emummc kip!");
     }
-    
+
     /* Copy over capabilities. */
     memcpy(injected_kip->capabilities, emummc_kip->capabilities, sizeof(emummc_kip->capabilities));
-    
+
     /* Add extra cap for 1.0.0 */
     if (stratosphere_get_fs_version() == FS_VER_1_0_0) {
         for (size_t i = 0; i < 0x20; i++) {
@@ -217,14 +242,14 @@ static kip1_header_t *inject_emummc_kip(kip1_header_t *fs_kip, kip1_header_t *em
             }
         }
     }
-    
+
     /* Update sections. */
     injected_kip->section_headers[0].out_size += emummc_data_size;
     injected_kip->section_headers[0].compressed_size += emummc_data_size;
     for (size_t i = 1; i < 4; i++) {
         injected_kip->section_headers[i].out_offset += emummc_data_size;
     }
-    
+
     /* Copy data. */
     {
         size_t ofs = 0;
@@ -234,22 +259,22 @@ static kip1_header_t *inject_emummc_kip(kip1_header_t *fs_kip, kip1_header_t *em
         }
     }
     memcpy(injected_kip->data + emummc_data_size, fs_kip->data, fs_contents_size);
-    
+
     return injected_kip;
 }
 
-ini1_header_t *stratosphere_get_sd_files_ini1(void) {    
+ini1_header_t *stratosphere_get_sd_files_ini1(void) {
     if (g_sd_files_ini1 != NULL) {
         return g_sd_files_ini1;
     }
-    
+
     /* Allocate space. */
     g_sd_files_ini1 = (ini1_header_t *)malloc(PACKAGE2_SIZE_MAX);
     g_sd_files_ini1->magic = MAGIC_INI1;
     g_sd_files_ini1->size = sizeof(ini1_header_t);
     g_sd_files_ini1->num_processes = 0;
     g_sd_files_ini1->_0xC = 0;
-    
+
     /* Load all kips from /atmosphere/kips/<*>.kip or /atmosphere/kips/<*>/<*>.kip. */
     DIR *kips_dir = opendir("atmosphere/kips");
     struct dirent *ent;
@@ -258,15 +283,15 @@ ini1_header_t *stratosphere_get_sd_files_ini1(void) {
             if (strcmp(ent->d_name, ".") == 0 || strcmp(ent->d_name, "..") == 0) {
                 continue;
             }
-            
+
             char kip_path[0x301] = {0};
             snprintf(kip_path, 0x300, "atmosphere/kips/%s", ent->d_name);
-            
+
             struct stat kip_stat;
             if (stat(kip_path, &kip_stat) == -1) {
                 continue;
             }
-            
+
             if ((kip_stat.st_mode & S_IFMT) == S_IFREG) {
                 /* If file, add to ini1. */
                 try_add_sd_kip(g_sd_files_ini1, kip_path);
@@ -279,15 +304,15 @@ ini1_header_t *stratosphere_get_sd_files_ini1(void) {
                         if (strcmp(sub_ent->d_name, ".") == 0 || strcmp(sub_ent->d_name, "..") == 0) {
                             continue;
                         }
-                        
+
                         /* Reuse kip path variable. */
                         memset(kip_path, 0, sizeof(kip_path));
                         snprintf(kip_path, 0x300, "atmosphere/kips/%s/%s", ent->d_name, sub_ent->d_name);
-                        
+
                         if (stat(kip_path, &kip_stat) == -1) {
                             continue;
                         }
-                        
+
                         if ((kip_stat.st_mode & S_IFMT) == S_IFREG) {
                             /* If file, add to ini1. */
                             try_add_sd_kip(g_sd_files_ini1, kip_path);
@@ -299,7 +324,7 @@ ini1_header_t *stratosphere_get_sd_files_ini1(void) {
         }
         closedir(kips_dir);
     }
-    
+
     return g_sd_files_ini1;
 }
 
@@ -344,7 +369,7 @@ ini1_header_t *stratosphere_merge_inis(ini1_header_t **inis, size_t num_inis, vo
             }
 
             offset += kip1_get_size_from_header(current_kip);
-            
+
             bool already_loaded = false;
             for (uint32_t j = 0; j < merged->num_processes; j++) {
                 if (process_list[j] == current_kip->title_id) {
@@ -355,20 +380,20 @@ ini1_header_t *stratosphere_merge_inis(ini1_header_t **inis, size_t num_inis, vo
             if (already_loaded) {
                 continue;
             }
-            
+
             print(SCREEN_LOG_LEVEL_MANDATORY, "[NXBOOT]: Loading KIP %08x%08x...\n", (uint32_t)(current_kip->title_id >> 32), (uint32_t)current_kip->title_id);
 
             size_t current_kip_size = kip1_get_size_from_header(current_kip);
             if (current_kip_size > remaining_size) {
                 fatal_error("Not enough space for all the KIP1s!\n");
             }
-            
+
             kip1_header_t *patched_kip = apply_kip_ips_patches(current_kip, current_kip_size, &g_fs_ver);
-            
+
             if (current_kip->title_id == FS_TITLE_ID && emummc != NULL) {
                 patched_kip = inject_emummc_kip(patched_kip != NULL ? patched_kip : current_kip, (kip1_header_t *)emummc);
             }
-            
+
             if (patched_kip != NULL) {
                 size_t patched_kip_size = kip1_get_size_from_header(patched_kip);
                 if (patched_kip_size > remaining_size) {
@@ -377,9 +402,9 @@ ini1_header_t *stratosphere_merge_inis(ini1_header_t **inis, size_t num_inis, vo
                 memcpy(current_dst_kip, patched_kip, patched_kip_size);
                 remaining_size -= patched_kip_size;
                 current_dst_kip += patched_kip_size;
-                
+
                 free(patched_kip);
-            } else {   
+            } else {
                 memcpy(current_dst_kip, current_kip, current_kip_size);
                 remaining_size -= current_kip_size;
                 current_dst_kip += current_kip_size;

fusee/fusee-secondary/src/stratosphere.h

@@ -13,7 +13,7 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
- 
+
 #ifndef FUSEE_STRATOSPHERE_H
 #define FUSEE_STRATOSPHERE_H
 
@@ -30,6 +30,8 @@ ini1_header_t *stratosphere_get_ini1(uint32_t target_firmware);
 ini1_header_t *stratosphere_get_sd_files_ini1(void);
 void stratosphere_free_ini1(void);
 
+void stratosphere_enable_ncm(void);
+
 emummc_fs_ver_t stratosphere_get_fs_version(void);
 
 ini1_header_t *stratosphere_merge_inis(ini1_header_t **inis, unsigned int num_inis, void *emummc, size_t emummc_size);
@@ -37,8 +39,10 @@ ini1_header_t *stratosphere_merge_inis(ini1_header_t **inis, unsigned int num_in
 typedef struct {
     bool has_nogc_config;
     bool enable_nogc;
+    bool ncm_enabled;
 } stratosphere_cfg_t;
 
 #define STRATOSPHERE_NOGC_KEY "nogc"
+#define STRATOSPHERE_ENABLE_NCM_KEY "enable_ncm"
 
 #endif

libraries/.gitrepo

@@ -6,7 +6,7 @@
 [subrepo]
 	remote = https://github.com/Atmosphere-NX/Atmosphere-libs
 	branch = master
-	commit = 08c9b3cbf85471fe6adb5f42ba9f03357a5f633a
-	parent = 814c9d1cfb54c192bc1deb140ab476c9f3dd0edf
+	commit = da6eac986d7f67ca3dcc3a8df0c191ffbea4686f
+	parent = eb48e7cc5943bd594b3166cb888d3dadb0474107
 	method = merge
-	cmdver = 0.4.0
+	cmdver = 0.4.1

libraries/README.md

@@ -11,6 +11,7 @@ You can find a copy of the license in the [LICENSE file](LICENSE).
 
 Exemptions:
 * The [yuzu emulator project](https://github.com/yuzu-emu/yuzu) is exempt from GPLv2 licensing and may (at its option) instead license any source code authored for the atmosphere-libs project as GPLv2 or later.
+* [Nintendo](https://github.com/Nintendo) is exempt from GPLv2 licensing and may (at its option) instead license any source code authored for the atmosphere-libs project under the Zero-Clause BSD license.
 
 Credits
 =====

libraries/config/arch/arm64/arch.mk

@@ -5,7 +5,7 @@ endif
 include $(DEVKITPRO)/devkitA64/base_rules
 
 export ATMOSPHERE_DEFINES  += -DATMOSPHERE_ARCH_ARM64
-export ATMOSPHERE_SETTINGS += -march=armv8-a -mtp=soft
+export ATMOSPHERE_SETTINGS += -march=armv8-a+crc+crypto -mtp=soft
 export ATMOSPHERE_CFLAGS   +=
 export ATMOSPHERE_CXXFLAGS +=
 export ATMOSPHERE_ASFLAGS  +=

libraries/config/board/nintendo/nx/board.mk

@@ -1,4 +1,4 @@
-export ATMOSPHERE_DEFINES  += -DATMOSPHERE_BOARD_NINTENDO_SWITCH -D__SWITCH__
+export ATMOSPHERE_DEFINES  += -DATMOSPHERE_BOARD_NINTENDO_NX -D__SWITCH__
 export ATMOSPHERE_SETTINGS +=
 export ATMOSPHERE_CFLAGS   +=
 export ATMOSPHERE_CXXFLAGS +=

libraries/config/common.mk

@@ -24,11 +24,11 @@ export ATMOSPHERE_ASFLAGS  :=
 
 ifeq ($(ATMOSPHERE_BOARD),nx-hac-001)
 export ATMOSPHERE_ARCH_DIR   := arch/arm64
-export ATMOSPHERE_BOARD_DIR  := board/nintendo/switch
+export ATMOSPHERE_BOARD_DIR  := board/nintendo/nx
 export ATMOSPHERE_OS_DIR     := os/horizon
 
 export ATMOSPHERE_ARCH_NAME  := arm64
-export ATMOSPHERE_BOARD_NAME := nintendo_switch
+export ATMOSPHERE_BOARD_NAME := nintendo_nx
 export ATMOSPHERE_OS_NAME    := horizon
 endif
 
@@ -76,14 +76,17 @@ TARGET       := $(notdir $(CURDIR))
 BUILD        := build
 DATA         := data
 INCLUDES     := include
-SOURCES      ?= $(foreach d,$(filter-out source/arch source/board,$(wildcard source)),$(call DIR_WILDCARD,$d) $d)
 
-ifneq ($(strip $(wildcard source/$(ATMOSPHERE_ARCH_DIR)/.*)),)
-SOURCES += $(call DIR_WILDCARD,source/$(ATMOSPHERE_ARCH_DIR))
-endif
-ifneq ($(strip $(wildcard source/$(ATMOSPHERE_BOARD_DIR)/.*)),)
-SOURCES += $(call DIR_WILDCARD,source/$(ATMOSPHERE_BOARD_DIR))
-endif
-ifneq ($(strip $(wildcard source/$(ATMOSPHERE_OS_DIR)/.*)),)
-SOURCES += $(call DIR_WILDCARD,source/$(ATMOSPHERE_OS_DIR))
-endif
+GENERAL_SOURCE_DIRS=$1 $(foreach d,$(filter-out $1/arch $1/board $1,$(wildcard $1/*)),$(if $(wildcard $d/.),$(call DIR_WILDCARD,$d) $d,))
+SPECIFIC_SOURCE_DIRS=$(if $(wildcard $1/$2/.*),$1/$2 $(call DIR_WILDCARD,$1/$2),)
+ALL_SOURCE_DIRS=$(call GENERAL_SOURCE_DIRS,$1) $(call SPECIFIC_SOURCE_DIRS,$1,$(ATMOSPHERE_ARCH_DIR)) $(call SPECIFIC_SOURCE_DIRS,$1,$(ATMOSPHERE_BOARD_DIR)) $(call SPECIFIC_SOURCE_DIRS,$1,$(ATMOSPHERE_OS_DIR))
+
+SOURCES      ?= $(call ALL_SOURCE_DIRS,source)
+
+#---------------------------------------------------------------------------------
+# Rules for compiling pre-compiled headers
+#---------------------------------------------------------------------------------
+%.gch: %.hpp
+	@echo $<
+	$(CXX) -w -x c++-header -MMD -MP -MF $(DEPSDIR)/$*.d $(CXXFLAGS) -c $< -o $@ $(ERROR_FILTER)
+	@cp $@ $(<).gch

libraries/config/templates/mesosphere.mk

@@ -7,12 +7,12 @@ include  $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/../common.mk
 # options for code generation
 #---------------------------------------------------------------------------------
 export DEFINES     := $(ATMOSPHERE_DEFINES) -DATMOSPHERE_IS_MESOSPHERE
-export SETTINGS    := $(ATMOSPHERE_SETTINGS) -O2 -mgeneral-regs-only -ffixed-x18 -Werror
+export SETTINGS    := $(ATMOSPHERE_SETTINGS) -O2 -mgeneral-regs-only -ffixed-x18 -Werror -fno-non-call-exceptions
 export CFLAGS      := $(ATMOSPHERE_CFLAGS) $(SETTINGS) $(DEFINES) $(INCLUDE)
-export CXXFLAGS    := $(CFLAGS) $(ATMOSPHERE_CXXFLAGS)
+export CXXFLAGS    := $(CFLAGS) $(ATMOSPHERE_CXXFLAGS) -fno-use-cxa-atexit
 export ASFLAGS     := $(ATMOSPHERE_ASFLAGS) $(SETTINGS) $(DEFINES)
 
-export LDFLAGS	=	-specs=$(TOPDIR)/kernel_ldr.specs -nostdlib -nostartfiles -g $(SETTINGS) -Wl,-Map,$(notdir $*.map)
+export LDFLAGS	=	-specs=$(TOPDIR)/$(notdir $(TOPDIR)).specs -nostdlib -nostartfiles -g $(SETTINGS) -Wl,-Map,$(notdir $*.map) -Wl,-z,relro,-z,now
 
 export CXXWRAPS := -Wl,--wrap,__cxa_pure_virtual \
 			-Wl,--wrap,__cxa_throw \
@@ -25,7 +25,6 @@ export CXXWRAPS := -Wl,--wrap,__cxa_pure_virtual \
 			-Wl,--wrap,__cxa_call_terminate \
 			-Wl,--wrap,__gxx_personality_v0 \
 			-Wl,--wrap,_Unwind_Resume \
-			-Wl,--wrap,_Unwind_Resume \
 			-Wl,--wrap,_ZSt19__throw_logic_errorPKc \
 			-Wl,--wrap,_ZSt20__throw_length_errorPKc \
 			-Wl,--wrap,_ZNSt11logic_errorC2EPKc

libraries/config/templates/stratosphere.mk

@@ -15,11 +15,11 @@ include $(DEVKITPRO)/libnx/switch_rules
 #---------------------------------------------------------------------------------
 # options for code generation
 #---------------------------------------------------------------------------------
-export DEFINES     := $(ATMOSPHERE_DEFINES) -DATMOSPHERE_IS_STRATOSPHERE
-export SETTINGS    := $(ATMOSPHERE_SETTINGS) -O2
-export CFLAGS      := $(ATMOSPHERE_CFLAGS) $(SETTINGS) $(DEFINES) $(INCLUDE)
-export CXXFLAGS    := $(CFLAGS) $(ATMOSPHERE_CXXFLAGS)
-export ASFLAGS     := $(ATMOSPHERE_ASFLAGS) $(SETTINGS) $(DEFINES)
+export DEFINES     = $(ATMOSPHERE_DEFINES) -DATMOSPHERE_IS_STRATOSPHERE -D_GNU_SOURCE
+export SETTINGS    = $(ATMOSPHERE_SETTINGS) -O2
+export CFLAGS      = $(ATMOSPHERE_CFLAGS) $(SETTINGS) $(DEFINES) $(INCLUDE)
+export CXXFLAGS    = $(CFLAGS) $(ATMOSPHERE_CXXFLAGS)
+export ASFLAGS     = $(ATMOSPHERE_ASFLAGS) $(SETTINGS) $(DEFINES)
 
 export CXXWRAPS := -Wl,--wrap,__cxa_pure_virtual \
 			-Wl,--wrap,__cxa_throw \
@@ -36,17 +36,17 @@ export CXXWRAPS := -Wl,--wrap,__cxa_pure_virtual \
 			-Wl,--wrap,_ZSt20__throw_length_errorPKc \
 			-Wl,--wrap,_ZNSt11logic_errorC2EPKc
 
-export LDFLAGS     := -specs=$(DEVKITPRO)/libnx/switch.specs $(SETTINGS) $(CXXWRAPS) -Wl,-Map,$(notdir $*.map)
+export LDFLAGS     = -specs=$(DEVKITPRO)/libnx/switch.specs $(SETTINGS) $(CXXWRAPS) -Wl,-Map,$(notdir $*.map)
 
-export LIBS	:= -lstratosphere -lnx
+export LIBS	= -lstratosphere -lnx
 
 #---------------------------------------------------------------------------------
 # list of directories containing libraries, this must be the top level containing
 # include and lib
 #---------------------------------------------------------------------------------
-export LIBDIRS	:= $(PORTLIBS) $(LIBNX) $(ATMOSPHERE_LIBRARIES_DIR)/libvapours $(ATMOSPHERE_LIBRARIES_DIR)/libstratosphere
+export LIBDIRS	= $(PORTLIBS) $(LIBNX) $(ATMOSPHERE_LIBRARIES_DIR)/libvapours $(ATMOSPHERE_LIBRARIES_DIR)/libstratosphere
 
 #---------------------------------------------------------------------------------
 # stratosphere sysmodules may (but usually do not) have an exefs source dir
 #---------------------------------------------------------------------------------
-export EXEFS_SRC := exefs_src
+export EXEFS_SRC = exefs_src

libraries/libmesosphere/Makefile

@@ -6,12 +6,17 @@ include  $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/../config/common.mk
 #---------------------------------------------------------------------------------
 # options for code generation
 #---------------------------------------------------------------------------------
+#PRECOMPILED_HEADERS := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/include/mesosphere.hpp
+PRECOMPILED_HEADERS :=
+
 DEFINES     := $(ATMOSPHERE_DEFINES) -DATMOSPHERE_IS_MESOSPHERE
-SETTINGS    := $(ATMOSPHERE_SETTINGS) -O2 -mgeneral-regs-only -ffixed-x18 -Werror
+SETTINGS    := $(ATMOSPHERE_SETTINGS) -O2 -mgeneral-regs-only -ffixed-x18 -Werror -fno-non-call-exceptions
 CFLAGS      := $(ATMOSPHERE_CFLAGS) $(SETTINGS) $(DEFINES) $(INCLUDE)
-CXXFLAGS    := $(CFLAGS) $(ATMOSPHERE_CXXFLAGS) -flto
+CXXFLAGS    := $(CFLAGS) $(ATMOSPHERE_CXXFLAGS) -fno-use-cxa-atexit -flto
 ASFLAGS     := $(ATMOSPHERE_ASFLAGS) $(SETTINGS)
 
+SOURCES     += $(call ALL_SOURCE_DIRS,../libvapours/source)
+
 LIBS        :=
 
 #---------------------------------------------------------------------------------
@@ -27,7 +32,7 @@ LIBDIRS	:= $(ATMOSPHERE_LIBRARIES_DIR)/libvapours
 ifneq ($(BUILD),$(notdir $(CURDIR)))
 #---------------------------------------------------------------------------------
 
-export VPATH	:=	$(foreach dir,$(SOURCES),$(CURDIR)/$(dir)) \
+export VPATH	:=	$(foreach dir,$(SOURCES),$(CURDIR)/$(dir)) $(CURDIR)/include \
 			$(foreach dir,$(DATA),$(CURDIR)/$(dir))
 
 CFILES      :=	$(foreach dir,$(SOURCES),$(filter-out $(notdir $(wildcard $(dir)/*.arch.*.c)) $(notdir $(wildcard $(dir)/*.board.*.c)) $(notdir $(wildcard $(dir)/*.os.*.c)), \
@@ -64,6 +69,7 @@ endif
 
 export OFILES_BIN	:=	$(addsuffix .o,$(BINFILES))
 export OFILES_SRC	:=	$(CPPFILES:.cpp=.o) $(CFILES:.c=.o) $(SFILES:.s=.o)
+export GCH_FILES    :=	$(foreach hdr,$(PRECOMPILED_HEADERS:.hpp=.gch),$(notdir $(hdr)))
 export OFILES 	:=	$(OFILES_BIN) $(OFILES_SRC)
 export HFILES_BIN	:=	$(addsuffix .h,$(subst .,_,$(BINFILES)))
 
@@ -105,13 +111,15 @@ clean:
 #---------------------------------------------------------------------------------
 else
 
-DEPENDS	:=	$(OFILES:.o=.d)
+DEPENDS	:=	$(OFILES:.o=.d) $(GCH_FILES:.gch=.d)
 
 #---------------------------------------------------------------------------------
 # main targets
 #---------------------------------------------------------------------------------
 $(OUTPUT)	:	$(OFILES)
 
+$(filter-out kern_svc_tables.o, $(OFILES))	:	$(GCH_FILES)
+
 $(OFILES_SRC)	: $(HFILES_BIN)
 
 #---------------------------------------------------------------------------------

libraries/libmesosphere/README.md

@@ -11,6 +11,7 @@ You can find a copy of the license in the [LICENSE file](LICENSE).
 
 Exemptions:
 * The [yuzu emulator project](https://github.com/yuzu-emu/yuzu) is exempt from GPLv2 licensing and may (at its option) instead license any source code authored for the libmesosphere project as GPLv2 or later.
+* [Nintendo](https://github.com/Nintendo) is exempt from GPLv2 licensing and may (at its option) instead license any source code authored for the libmesosphere project under the Zero-Clause BSD license.
 
 Credits
 =====

libraries/libmesosphere/include/mesosphere.hpp

@@ -19,20 +19,70 @@
 #include <vapours.hpp>
 
 /* First, pull in core macros (panic, etc). */
-#include "mesosphere/kern_panic.hpp"
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_panic.hpp>
 
 /* Primitive types. */
-#include "mesosphere/kern_k_typed_address.hpp"
-#include "mesosphere/kern_initial_process.hpp"
+#include <mesosphere/kern_k_typed_address.hpp>
+#include <mesosphere/kern_initial_process.hpp>
+#include <mesosphere/kern_k_exception_context.hpp>
 
 /* Core pre-initialization includes. */
-#include "mesosphere/kern_select_cpu.hpp"
+#include <mesosphere/kern_select_cpu.hpp>
+#include <mesosphere/kern_select_system_control.hpp>
+#include <mesosphere/kern_k_target_system.hpp>
 
 /* Initialization headers. */
-#include "mesosphere/init/kern_init_elf.hpp"
-#include "mesosphere/init/kern_init_layout.hpp"
-#include "mesosphere/init/kern_init_page_table_select.hpp"
+#include <mesosphere/init/kern_init_elf.hpp>
+#include <mesosphere/init/kern_init_layout.hpp>
+#include <mesosphere/init/kern_init_slab_setup.hpp>
+#include <mesosphere/init/kern_init_page_table_select.hpp>
+#include <mesosphere/init/kern_init_arguments_select.hpp>
+#include <mesosphere/kern_k_memory_layout.hpp>
 
 /* Core functionality. */
-#include "mesosphere/kern_select_interrupts.hpp"
-#include "mesosphere/kern_select_k_system_control.hpp"
+#include <mesosphere/kern_select_interrupt_manager.hpp>
+#include <mesosphere/kern_k_spin_lock.hpp>
+#include <mesosphere/kern_k_memory_manager.hpp>
+#include <mesosphere/kern_k_interrupt_task_manager.hpp>
+#include <mesosphere/kern_k_core_local_region.hpp>
+#include <mesosphere/kern_k_slab_heap.hpp>
+#include <mesosphere/kern_k_light_lock.hpp>
+#include <mesosphere/kern_k_dpc_manager.hpp>
+#include <mesosphere/kern_kernel.hpp>
+#include <mesosphere/kern_k_page_table_manager.hpp>
+#include <mesosphere/kern_select_page_table.hpp>
+
+/* Miscellaneous objects. */
+#include <mesosphere/kern_k_shared_memory_info.hpp>
+#include <mesosphere/kern_k_event_info.hpp>
+
+/* Auto Objects. */
+#include <mesosphere/kern_k_auto_object.hpp>
+#include <mesosphere/kern_k_synchronization_object.hpp>
+#include <mesosphere/kern_k_readable_event.hpp>
+#include <mesosphere/kern_k_handle_table.hpp>
+#include <mesosphere/kern_k_event.hpp>
+#include <mesosphere/kern_k_interrupt_event.hpp>
+#include <mesosphere/kern_k_light_session.hpp>
+#include <mesosphere/kern_k_session.hpp>
+#include <mesosphere/kern_k_session_request.hpp>
+#include <mesosphere/kern_k_port.hpp>
+#include <mesosphere/kern_k_shared_memory.hpp>
+#include <mesosphere/kern_k_transfer_memory.hpp>
+#include <mesosphere/kern_k_code_memory.hpp>
+#include <mesosphere/kern_k_device_address_space.hpp>
+#include <mesosphere/kern_select_debug.hpp>
+#include <mesosphere/kern_k_process.hpp>
+#include <mesosphere/kern_k_resource_limit.hpp>
+#include <mesosphere/kern_k_synchronization.hpp>
+
+/* More Miscellaneous objects. */
+#include <mesosphere/kern_k_object_name.hpp>
+#include <mesosphere/kern_k_scoped_resource_reservation.hpp>
+
+/* Supervisor Calls. */
+#include <mesosphere/kern_svc.hpp>
+
+/* Main functionality. */
+#include <mesosphere/kern_main.hpp>

libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_arguments.hpp

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+
+namespace ams::kern::init {
+
+    struct KInitArguments {
+        u64 ttbr0;
+        u64 ttbr1;
+        u64 tcr;
+        u64 mair;
+        u64 cpuactlr;
+        u64 cpuectlr;
+        u64 sctlr;
+        u64 sp;
+        u64 entrypoint;
+        u64 argument;
+        u64 setup_function;
+    };
+
+}

libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp

@@ -15,173 +15,13 @@
  */
 #pragma once
 #include <vapours.hpp>
-#include <mesosphere/kern_panic.hpp>
+#include <mesosphere/kern_common.hpp>
 #include <mesosphere/kern_k_typed_address.hpp>
-#include "../kern_cpu.hpp"
-
-namespace ams::kern::init {
-
-    constexpr size_t PageSize              = 0x1000;
-    constexpr size_t L1BlockSize           = 0x40000000;
-    constexpr size_t L2BlockSize           = 0x200000;
-    constexpr size_t L2ContiguousBlockSize = 0x10 * L2BlockSize;
-    constexpr size_t L3BlockSize           = 0x1000;
-    constexpr size_t L3ContiguousBlockSize = 0x10 * L3BlockSize;
-
-    class PageTableEntry {
-        public:
-            enum Permission : u64 {
-                Permission_KernelRWX = ((0ul << 53) | (1ul << 54) | (0ul << 6)),
-                Permission_KernelRX  = ((0ul << 53) | (1ul << 54) | (2ul << 6)),
-                Permission_KernelR   = ((1ul << 53) | (1ul << 54) | (2ul << 6)),
-                Permission_KernelRW  = ((1ul << 53) | (1ul << 54) | (0ul << 6)),
-
-                Permission_UserRX    = ((1ul << 53) | (0ul << 54) | (3ul << 6)),
-                Permission_UserR     = ((1ul << 53) | (1ul << 54) | (3ul << 6)),
-                Permission_UserRW    = ((1ul << 53) | (1ul << 54) | (1ul << 6)),
-            };
-
-            enum Shareable : u64 {
-                Shareable_NonShareable   = (0 << 8),
-                Shareable_OuterShareable = (2 << 8),
-                Shareable_InnerShareable = (3 << 8),
-            };
-
-            /* Official attributes are: */
-            /*    0x00, 0x04, 0xFF, 0x44. 4-7 are unused. */
-            enum PageAttribute : u64 {
-                PageAttribute_Device_nGnRnE            = (0 << 2),
-                PageAttribute_Device_nGnRE             = (1 << 2),
-                PageAttribute_NormalMemory             = (2 << 2),
-                PageAttribute_NormalMemoryNotCacheable = (3 << 2),
-            };
-
-            enum AccessFlag : u64 {
-                AccessFlag_NotAccessed = (0 << 10),
-                AccessFlag_Accessed    = (1 << 10),
-            };
-        protected:
-            u64 attributes;
-        public:
-            /* Take in a raw attribute. */
-            constexpr ALWAYS_INLINE PageTableEntry(u64 attr) : attributes(attr) { /* ... */ }
-
-            /* Extend a previous attribute. */
-            constexpr ALWAYS_INLINE PageTableEntry(const PageTableEntry &rhs, u64 new_attr) : attributes(rhs.attributes | new_attr) { /* ... */ }
-
-            /* Construct a new attribute. */
-            constexpr ALWAYS_INLINE PageTableEntry(Permission perm, PageAttribute p_a, Shareable share)
-                : attributes(static_cast<u64>(perm) | static_cast<u64>(AccessFlag_Accessed) | static_cast<u64>(p_a) | static_cast<u64>(share))
-            {
-                /* ... */
-            }
-        protected:
-            constexpr ALWAYS_INLINE u64 GetBits(size_t offset, size_t count) const {
-                return (this->attributes >> offset) & ((1ul << count) - 1);
-            }
-
-            constexpr ALWAYS_INLINE u64 SelectBits(size_t offset, size_t count) const {
-                return this->attributes & (((1ul << count) - 1) << offset);
-            }
-        public:
-            constexpr ALWAYS_INLINE bool IsUserExecuteNever()        const { return this->GetBits(54, 1) != 0; }
-            constexpr ALWAYS_INLINE bool IsPrivilegedExecuteNever()  const { return this->GetBits(53, 1) != 0; }
-            constexpr ALWAYS_INLINE bool IsContiguous()              const { return this->GetBits(52, 1) != 0; }
-            constexpr ALWAYS_INLINE AccessFlag GetAccessFlag()       const { return static_cast<AccessFlag>(this->GetBits(10, 1)); }
-            constexpr ALWAYS_INLINE Shareable GetShareable()         const { return static_cast<Shareable>(this->GetBits(8, 2)); }
-            constexpr ALWAYS_INLINE PageAttribute GetPageAttribute() const { return static_cast<PageAttribute>(this->GetBits(2, 3)); }
-            constexpr ALWAYS_INLINE bool IsNonSecure()               const { return this->GetBits(5, 1) != 0; }
-            constexpr ALWAYS_INLINE bool IsBlock()                   const { return this->GetBits(0, 2) == 0x1; }
-            constexpr ALWAYS_INLINE bool IsTable()                   const { return this->GetBits(0, 2) == 0x3; }
-
-            /* Should not be called except by derived classes. */
-            constexpr ALWAYS_INLINE u64 GetRawAttributes() const {
-                return this->attributes;
-            }
-    };
-
-    static_assert(sizeof(PageTableEntry) == sizeof(u64));
-
-    constexpr PageTableEntry InvalidPageTableEntry = PageTableEntry(0);
-
-    constexpr size_t MaxPageTableEntries = PageSize / sizeof(PageTableEntry);
-
-    class L1PageTableEntry : public PageTableEntry {
-        public:
-            constexpr ALWAYS_INLINE L1PageTableEntry(KPhysicalAddress phys_addr, bool pxn)
-                : PageTableEntry((0x3ul << 60) | (static_cast<u64>(pxn) << 59) | GetInteger(phys_addr) | 0x3)
-            {
-                /* ... */
-            }
-
-            constexpr ALWAYS_INLINE L1PageTableEntry(KPhysicalAddress phys_addr, const PageTableEntry &attr, bool contig)
-                : PageTableEntry(attr, (static_cast<u64>(contig) << 52) | GetInteger(phys_addr) | 0x1)
-            {
-                /* ... */
-            }
-
-            constexpr ALWAYS_INLINE KPhysicalAddress GetBlock() const {
-                return this->SelectBits(30, 18);
-            }
-
-            constexpr ALWAYS_INLINE KPhysicalAddress GetTable() const {
-                return this->SelectBits(12, 36);
-            }
-
-            constexpr ALWAYS_INLINE bool IsCompatibleWithAttribute(const PageTableEntry &rhs, bool contig) const {
-                /* Check whether this has the same permission/etc as the desired attributes. */
-                return L1PageTableEntry(this->GetBlock(), rhs, contig).GetRawAttributes() == this->GetRawAttributes();
-            }
-    };
-
-    class L2PageTableEntry : public PageTableEntry {
-        public:
-            constexpr ALWAYS_INLINE L2PageTableEntry(KPhysicalAddress phys_addr, bool pxn)
-                : PageTableEntry((0x3ul << 60) | (static_cast<u64>(pxn) << 59) | GetInteger(phys_addr) | 0x3)
-            {
-                /* ... */
-            }
-
-            constexpr ALWAYS_INLINE L2PageTableEntry(KPhysicalAddress phys_addr, const PageTableEntry &attr, bool contig)
-                : PageTableEntry(attr, (static_cast<u64>(contig) << 52) | GetInteger(phys_addr) | 0x1)
-            {
-                /* ... */
-            }
-
-            constexpr ALWAYS_INLINE KPhysicalAddress GetBlock() const {
-                return this->SelectBits(21, 27);
-            }
-
-            constexpr ALWAYS_INLINE KPhysicalAddress GetTable() const {
-                return this->SelectBits(12, 36);
-            }
-
-            constexpr ALWAYS_INLINE bool IsCompatibleWithAttribute(const PageTableEntry &rhs, bool contig) const {
-                /* Check whether this has the same permission/etc as the desired attributes. */
-                return L2PageTableEntry(this->GetBlock(), rhs, contig).GetRawAttributes() == this->GetRawAttributes();
-            }
-    };
-
-    class L3PageTableEntry : public PageTableEntry {
-        public:
-            constexpr ALWAYS_INLINE L3PageTableEntry(KPhysicalAddress phys_addr, const PageTableEntry &attr, bool contig)
-                : PageTableEntry(attr, (static_cast<u64>(contig) << 52) | GetInteger(phys_addr) | 0x3)
-            {
-                /* ... */
-            }
-
-            constexpr ALWAYS_INLINE bool IsBlock() const { return this->GetBits(0, 2) == 0x3; }
-
-            constexpr ALWAYS_INLINE KPhysicalAddress GetBlock() const {
-                return this->SelectBits(12, 36);
-            }
-
-            constexpr ALWAYS_INLINE bool IsCompatibleWithAttribute(const PageTableEntry &rhs, bool contig) const {
-                /* Check whether this has the same permission/etc as the desired attributes. */
-                return L3PageTableEntry(this->GetBlock(), rhs, contig).GetRawAttributes() == this->GetRawAttributes();
-            }
-    };
+#include <mesosphere/kern_select_cpu.hpp>
+#include <mesosphere/arch/arm64/kern_k_page_table_entry.hpp>
+#include <mesosphere/kern_select_system_control.hpp>
 
+namespace ams::kern::arch::arm64::init {
 
     class KInitialPageTable {
         public:
@@ -190,10 +30,14 @@ namespace ams::kern::init {
                     virtual KPhysicalAddress Allocate()           { return Null<KPhysicalAddress>; }
                     virtual void Free(KPhysicalAddress phys_addr) { /* Nothing to do here. */ (void)(phys_addr); }
             };
+
+            struct NoClear{};
         private:
             KPhysicalAddress l1_table;
         public:
-            constexpr ALWAYS_INLINE KInitialPageTable(KPhysicalAddress l1) : l1_table(l1) {
+            constexpr ALWAYS_INLINE KInitialPageTable(KPhysicalAddress l1, NoClear) : l1_table(l1) { /* ... */ }
+
+            constexpr ALWAYS_INLINE KInitialPageTable(KPhysicalAddress l1) : KInitialPageTable(l1, NoClear{}) {
                 ClearNewPageTable(this->l1_table);
             }
 
@@ -221,12 +65,226 @@ namespace ams::kern::init {
                 /* The MMU is necessarily not yet turned on, if we are creating an initial page table. */
                 std::memset(reinterpret_cast<void *>(GetInteger(address)), 0, PageSize);
             }
+        private:
+            size_t NOINLINE GetBlockCount(KVirtualAddress virt_addr, size_t size, size_t block_size) {
+                const KVirtualAddress end_virt_addr = virt_addr + size;
+                size_t count = 0;
+                while (virt_addr < end_virt_addr) {
+                    L1PageTableEntry *l1_entry = GetL1Entry(this->l1_table, virt_addr);
+
+                    /* If an L1 block is mapped or we're empty, advance by L1BlockSize. */
+                    if (l1_entry->IsBlock() || l1_entry->IsEmpty()) {
+                        MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), L1BlockSize));
+                        MESOSPHERE_INIT_ABORT_UNLESS(static_cast<size_t>(end_virt_addr - virt_addr) >= L1BlockSize);
+                        virt_addr += L1BlockSize;
+                        if (l1_entry->IsBlock() && block_size == L1BlockSize) {
+                            count++;
+                        }
+                        continue;
+                    }
+
+                    /* Non empty and non-block must be table. */
+                    MESOSPHERE_INIT_ABORT_UNLESS(l1_entry->IsTable());
+
+                    /* Table, so check if we're mapped in L2. */
+                    L2PageTableEntry *l2_entry = GetL2Entry(l1_entry, virt_addr);
+
+                    if (l2_entry->IsBlock() || l2_entry->IsEmpty()) {
+                        const size_t advance_size = (l2_entry->IsBlock() && l2_entry->IsContiguous()) ? L2ContiguousBlockSize : L2BlockSize;
+                        MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), advance_size));
+                        MESOSPHERE_INIT_ABORT_UNLESS(static_cast<size_t>(end_virt_addr - virt_addr) >= advance_size);
+                        virt_addr += advance_size;
+                        if (l2_entry->IsBlock() && block_size == advance_size) {
+                            count++;
+                        }
+                        continue;
+                    }
+
+                    /* Non empty and non-block must be table. */
+                    MESOSPHERE_INIT_ABORT_UNLESS(l2_entry->IsTable());
+
+                    /* Table, so check if we're mapped in L3. */
+                    L3PageTableEntry *l3_entry = GetL3Entry(l2_entry, virt_addr);
+
+                    /* L3 must be block or empty. */
+                    MESOSPHERE_INIT_ABORT_UNLESS(l3_entry->IsBlock() || l3_entry->IsEmpty());
+
+                    const size_t advance_size = (l3_entry->IsBlock() && l3_entry->IsContiguous()) ? L3ContiguousBlockSize : L3BlockSize;
+                    MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), advance_size));
+                    MESOSPHERE_INIT_ABORT_UNLESS(static_cast<size_t>(end_virt_addr - virt_addr) >= advance_size);
+                    virt_addr += advance_size;
+                    if (l3_entry->IsBlock() && block_size == advance_size) {
+                        count++;
+                    }
+                }
+                return count;
+            }
+
+            KVirtualAddress NOINLINE GetBlockByIndex(KVirtualAddress virt_addr, size_t size, size_t block_size, size_t index) {
+                const KVirtualAddress end_virt_addr = virt_addr + size;
+                size_t count = 0;
+                while (virt_addr < end_virt_addr) {
+                    L1PageTableEntry *l1_entry = GetL1Entry(this->l1_table, virt_addr);
+
+                    /* If an L1 block is mapped or we're empty, advance by L1BlockSize. */
+                    if (l1_entry->IsBlock() || l1_entry->IsEmpty()) {
+                        MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), L1BlockSize));
+                        MESOSPHERE_INIT_ABORT_UNLESS(static_cast<size_t>(end_virt_addr - virt_addr) >= L1BlockSize);
+                        if (l1_entry->IsBlock() && block_size == L1BlockSize) {
+                            if ((count++) == index) {
+                                return virt_addr;
+                            }
+                        }
+                        virt_addr += L1BlockSize;
+                        continue;
+                    }
+
+                    /* Non empty and non-block must be table. */
+                    MESOSPHERE_INIT_ABORT_UNLESS(l1_entry->IsTable());
+
+                    /* Table, so check if we're mapped in L2. */
+                    L2PageTableEntry *l2_entry = GetL2Entry(l1_entry, virt_addr);
+
+                    if (l2_entry->IsBlock() || l2_entry->IsEmpty()) {
+                        const size_t advance_size = (l2_entry->IsBlock() && l2_entry->IsContiguous()) ? L2ContiguousBlockSize : L2BlockSize;
+                        MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), advance_size));
+                        MESOSPHERE_INIT_ABORT_UNLESS(static_cast<size_t>(end_virt_addr - virt_addr) >= advance_size);
+                        if (l2_entry->IsBlock() && block_size == advance_size) {
+                            if ((count++) == index) {
+                                return virt_addr;
+                            }
+                        }
+                        virt_addr += advance_size;
+                        continue;
+                    }
+
+                    /* Non empty and non-block must be table. */
+                    MESOSPHERE_INIT_ABORT_UNLESS(l2_entry->IsTable());
+
+                    /* Table, so check if we're mapped in L3. */
+                    L3PageTableEntry *l3_entry = GetL3Entry(l2_entry, virt_addr);
+
+                    /* L3 must be block or empty. */
+                    MESOSPHERE_INIT_ABORT_UNLESS(l3_entry->IsBlock() || l3_entry->IsEmpty());
+
+                    const size_t advance_size = (l3_entry->IsBlock() && l3_entry->IsContiguous()) ? L3ContiguousBlockSize : L3BlockSize;
+                    MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), advance_size));
+                    MESOSPHERE_INIT_ABORT_UNLESS(static_cast<size_t>(end_virt_addr - virt_addr) >= advance_size);
+                    if (l3_entry->IsBlock() && block_size == advance_size) {
+                        if ((count++) == index) {
+                            return virt_addr;
+                        }
+                    }
+                    virt_addr += advance_size;
+                }
+                return Null<KVirtualAddress>;
+            }
+
+            PageTableEntry *GetMappingEntry(KVirtualAddress virt_addr, size_t block_size) {
+                L1PageTableEntry *l1_entry = GetL1Entry(this->l1_table, virt_addr);
+
+                if (l1_entry->IsBlock()) {
+                    MESOSPHERE_INIT_ABORT_UNLESS(block_size == L1BlockSize);
+                    return l1_entry;
+                }
+
+                MESOSPHERE_INIT_ABORT_UNLESS(l1_entry->IsTable());
+
+                /* Table, so check if we're mapped in L2. */
+                L2PageTableEntry *l2_entry = GetL2Entry(l1_entry, virt_addr);
+
+                if (l2_entry->IsBlock()) {
+                    const size_t real_size = (l2_entry->IsContiguous()) ? L2ContiguousBlockSize : L2BlockSize;
+                    MESOSPHERE_INIT_ABORT_UNLESS(real_size == block_size);
+                    return l2_entry;
+                }
+
+                MESOSPHERE_INIT_ABORT_UNLESS(l2_entry->IsTable());
+
+                /* Table, so check if we're mapped in L3. */
+                L3PageTableEntry *l3_entry = GetL3Entry(l2_entry, virt_addr);
+
+                /* L3 must be block. */
+                MESOSPHERE_INIT_ABORT_UNLESS(l3_entry->IsBlock());
+
+                const size_t real_size = (l3_entry->IsContiguous()) ? L3ContiguousBlockSize : L3BlockSize;
+                MESOSPHERE_INIT_ABORT_UNLESS(real_size == block_size);
+                return l3_entry;
+            }
+
+            void NOINLINE SwapBlocks(KVirtualAddress src_virt_addr, KVirtualAddress dst_virt_addr, size_t block_size, bool do_copy) {
+                static_assert(L2ContiguousBlockSize / L2BlockSize == L3ContiguousBlockSize / L3BlockSize);
+                const bool contig = (block_size == L2ContiguousBlockSize || block_size == L3ContiguousBlockSize);
+                const size_t num_mappings = contig ? L2ContiguousBlockSize / L2BlockSize : 1;
+
+                /* Unmap the source. */
+                PageTableEntry *src_entry = this->GetMappingEntry(src_virt_addr, block_size);
+                const auto src_saved = *src_entry;
+                for (size_t i = 0; i < num_mappings; i++) {
+                    *src_entry = InvalidPageTableEntry;
+                }
+
+                /* Unmap the target. */
+                PageTableEntry *dst_entry = this->GetMappingEntry(dst_virt_addr, block_size);
+                const auto dst_saved = *dst_entry;
+                for (size_t i = 0; i < num_mappings; i++) {
+                    *dst_entry = InvalidPageTableEntry;
+                }
+
+                /* Invalidate the entire tlb. */
+                cpu::DataSynchronizationBarrierInnerShareable();
+                cpu::InvalidateEntireTlb();
+
+                /* Copy data, if we should. */
+                const u64 negative_block_size_for_mask = static_cast<u64>(-static_cast<s64>(block_size));
+                const u64 offset_mask                  = negative_block_size_for_mask & ((1ul << 36) - 1);
+                const KVirtualAddress copy_src_addr = KVirtualAddress(src_saved.GetRawAttributesUnsafeForSwap() & offset_mask);
+                const KVirtualAddress copy_dst_addr = KVirtualAddress(dst_saved.GetRawAttributesUnsafeForSwap() & offset_mask);
+                if (block_size && do_copy) {
+                    u8 tmp[0x100];
+                    for (size_t ofs = 0; ofs < block_size; ofs += sizeof(tmp)) {
+                        std::memcpy(tmp, GetVoidPointer(copy_src_addr + ofs), sizeof(tmp));
+                        std::memcpy(GetVoidPointer(copy_src_addr + ofs), GetVoidPointer(copy_dst_addr + ofs), sizeof(tmp));
+                        std::memcpy(GetVoidPointer(copy_dst_addr + ofs), tmp, sizeof(tmp));
+                    }
+                }
+
+                /* Swap the mappings. */
+                const u64 attr_preserve_mask  = (negative_block_size_for_mask | 0xFFFF000000000000ul) ^ ((1ul << 36) - 1);
+                const size_t shift_for_contig = contig ? 4 : 0;
+                size_t advanced_size = 0;
+                const u64 src_attr_val = src_saved.GetRawAttributesUnsafeForSwap() & attr_preserve_mask;
+                const u64 dst_attr_val = dst_saved.GetRawAttributesUnsafeForSwap() & attr_preserve_mask;
+                for (size_t i = 0; i < num_mappings; i++) {
+                    reinterpret_cast<u64 *>(src_entry)[i] = GetInteger(copy_dst_addr + (advanced_size >> shift_for_contig)) | src_attr_val;
+                    reinterpret_cast<u64 *>(dst_entry)[i] = GetInteger(copy_src_addr + (advanced_size >> shift_for_contig)) | dst_attr_val;
+                    advanced_size += block_size;
+                }
+
+                cpu::DataSynchronizationBarrierInnerShareable();
+            }
+
+            void NOINLINE PhysicallyRandomize(KVirtualAddress virt_addr, size_t size, size_t block_size, bool do_copy) {
+                const size_t block_count = this->GetBlockCount(virt_addr, size, block_size);
+                if (block_count > 1) {
+                    for (size_t cur_block = 0; cur_block < block_count; cur_block++) {
+                        const size_t target_block = KSystemControl::Init::GenerateRandomRange(cur_block, block_count - 1);
+                        if (cur_block != target_block) {
+                            const KVirtualAddress cur_virt_addr    = this->GetBlockByIndex(virt_addr, size, block_size, cur_block);
+                            const KVirtualAddress target_virt_addr = this->GetBlockByIndex(virt_addr, size, block_size, target_block);
+                            MESOSPHERE_INIT_ABORT_UNLESS(cur_virt_addr    != Null<KVirtualAddress>);
+                            MESOSPHERE_INIT_ABORT_UNLESS(target_virt_addr != Null<KVirtualAddress>);
+                            this->SwapBlocks(cur_virt_addr, target_virt_addr, block_size, do_copy);
+                        }
+                    }
+                }
+            }
         public:
             void NOINLINE Map(KVirtualAddress virt_addr, size_t size, KPhysicalAddress phys_addr, const PageTableEntry &attr, IPageAllocator &allocator) {
                 /* Ensure that addresses and sizes are page aligned. */
-                MESOSPHERE_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr),    PageSize));
-                MESOSPHERE_ABORT_UNLESS(util::IsAligned(GetInteger(phys_addr),    PageSize));
-                MESOSPHERE_ABORT_UNLESS(util::IsAligned(size,                     PageSize));
+                MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr),    PageSize));
+                MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(phys_addr),    PageSize));
+                MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(size,                     PageSize));
 
                 /* Iteratively map pages until the requested region is mapped. */
                 while (size > 0) {
@@ -309,10 +367,37 @@ namespace ams::kern::init {
                 }
             }
 
+            KPhysicalAddress GetPhysicalAddress(KVirtualAddress virt_addr) const {
+                /* Get the L1 entry. */
+                const L1PageTableEntry *l1_entry = GetL1Entry(this->l1_table, virt_addr);
+
+                if (l1_entry->IsBlock()) {
+                    return l1_entry->GetBlock() + (GetInteger(virt_addr) & (L1BlockSize - 1));
+                }
+
+                MESOSPHERE_INIT_ABORT_UNLESS(l1_entry->IsTable());
+
+                /* Get the L2 entry. */
+                const L2PageTableEntry *l2_entry = GetL2Entry(l1_entry, virt_addr);
+
+                if (l2_entry->IsBlock()) {
+                    return l2_entry->GetBlock() + (GetInteger(virt_addr) & (L2BlockSize - 1));
+                }
+
+                MESOSPHERE_INIT_ABORT_UNLESS(l2_entry->IsTable());
+
+                /* Get the L3 entry. */
+                const L3PageTableEntry *l3_entry = GetL3Entry(l2_entry, virt_addr);
+
+                MESOSPHERE_INIT_ABORT_UNLESS(l3_entry->IsBlock());
+
+                return l3_entry->GetBlock() + (GetInteger(virt_addr) & (L3BlockSize - 1));
+            }
+
             bool IsFree(KVirtualAddress virt_addr, size_t size) {
                 /* Ensure that addresses and sizes are page aligned. */
-                MESOSPHERE_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr),    PageSize));
-                MESOSPHERE_ABORT_UNLESS(util::IsAligned(size,                     PageSize));
+                MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr),    PageSize));
+                MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(size,                     PageSize));
 
                 const KVirtualAddress end_virt_addr = virt_addr + size;
                 while (virt_addr < end_virt_addr) {
@@ -360,8 +445,8 @@ namespace ams::kern::init {
                 cpu::DataSynchronizationBarrierInnerShareable();
 
                 /* Ensure that addresses and sizes are page aligned. */
-                MESOSPHERE_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr),    PageSize));
-                MESOSPHERE_ABORT_UNLESS(util::IsAligned(size,                     PageSize));
+                MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr),    PageSize));
+                MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(size,                     PageSize));
 
                 /* Iteratively reprotect pages until the requested region is reprotected. */
                 while (size > 0) {
@@ -371,9 +456,9 @@ namespace ams::kern::init {
                     if (l1_entry->IsBlock()) {
                         /* Ensure that we are allowed to have an L1 block here. */
                         const KPhysicalAddress block = l1_entry->GetBlock();
-                        MESOSPHERE_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), L1BlockSize));
-                        MESOSPHERE_ABORT_UNLESS(util::IsAligned(size, L1BlockSize));
-                        MESOSPHERE_ABORT_UNLESS(l1_entry->IsCompatibleWithAttribute(attr_before, false));
+                        MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), L1BlockSize));
+                        MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(size, L1BlockSize));
+                        MESOSPHERE_INIT_ABORT_UNLESS(l1_entry->IsCompatibleWithAttribute(attr_before, false));
 
                         /* Invalidate the existing L1 block. */
                         *static_cast<PageTableEntry *>(l1_entry) = InvalidPageTableEntry;
@@ -389,7 +474,7 @@ namespace ams::kern::init {
                     }
 
                     /* Not a block, so we must be a table. */
-                    MESOSPHERE_ABORT_UNLESS(l1_entry->IsTable());
+                    MESOSPHERE_INIT_ABORT_UNLESS(l1_entry->IsTable());
 
                     L2PageTableEntry *l2_entry = GetL2Entry(l1_entry, virt_addr);
                     if (l2_entry->IsBlock()) {
@@ -397,14 +482,14 @@ namespace ams::kern::init {
 
                         if (l2_entry->IsContiguous()) {
                             /* Ensure that we are allowed to have a contiguous L2 block here. */
-                            MESOSPHERE_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), L2ContiguousBlockSize));
-                            MESOSPHERE_ABORT_UNLESS(util::IsAligned(GetInteger(block),     L2ContiguousBlockSize));
-                            MESOSPHERE_ABORT_UNLESS(util::IsAligned(size,                  L2ContiguousBlockSize));
+                            MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), L2ContiguousBlockSize));
+                            MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(block),     L2ContiguousBlockSize));
+                            MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(size,                  L2ContiguousBlockSize));
 
                             /* Invalidate the existing contiguous L2 block. */
                             for (size_t i = 0; i < L2ContiguousBlockSize / L2BlockSize; i++) {
                                 /* Ensure that the entry is valid. */
-                                MESOSPHERE_ABORT_UNLESS(l2_entry[i].IsCompatibleWithAttribute(attr_before, true));
+                                MESOSPHERE_INIT_ABORT_UNLESS(l2_entry[i].IsCompatibleWithAttribute(attr_before, true));
                                 static_cast<PageTableEntry *>(l2_entry)[i] = InvalidPageTableEntry;
                             }
                             cpu::DataSynchronizationBarrierInnerShareable();
@@ -419,10 +504,10 @@ namespace ams::kern::init {
                             size      -= L2ContiguousBlockSize;
                         } else {
                             /* Ensure that we are allowed to have an L2 block here. */
-                            MESOSPHERE_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), L2BlockSize));
-                            MESOSPHERE_ABORT_UNLESS(util::IsAligned(GetInteger(block),     L2BlockSize));
-                            MESOSPHERE_ABORT_UNLESS(util::IsAligned(size,                  L2BlockSize));
-                            MESOSPHERE_ABORT_UNLESS(l2_entry->IsCompatibleWithAttribute(attr_before, false));
+                            MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), L2BlockSize));
+                            MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(block),     L2BlockSize));
+                            MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(size,                  L2BlockSize));
+                            MESOSPHERE_INIT_ABORT_UNLESS(l2_entry->IsCompatibleWithAttribute(attr_before, false));
 
                             /* Invalidate the existing L2 block. */
                             *static_cast<PageTableEntry *>(l2_entry) = InvalidPageTableEntry;
@@ -440,23 +525,23 @@ namespace ams::kern::init {
                     }
 
                     /* Not a block, so we must be a table. */
-                    MESOSPHERE_ABORT_UNLESS(l2_entry->IsTable());
+                    MESOSPHERE_INIT_ABORT_UNLESS(l2_entry->IsTable());
 
                     /* We must have a mapped l3 entry to reprotect. */
                     L3PageTableEntry *l3_entry = GetL3Entry(l2_entry, virt_addr);
-                    MESOSPHERE_ABORT_UNLESS(l3_entry->IsBlock());
+                    MESOSPHERE_INIT_ABORT_UNLESS(l3_entry->IsBlock());
                     const KPhysicalAddress block = l3_entry->GetBlock();
 
                     if (l3_entry->IsContiguous()) {
                         /* Ensure that we are allowed to have a contiguous L3 block here. */
-                        MESOSPHERE_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), L3ContiguousBlockSize));
-                        MESOSPHERE_ABORT_UNLESS(util::IsAligned(GetInteger(block),     L3ContiguousBlockSize));
-                        MESOSPHERE_ABORT_UNLESS(util::IsAligned(size,                  L3ContiguousBlockSize));
+                        MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), L3ContiguousBlockSize));
+                        MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(block),     L3ContiguousBlockSize));
+                        MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(size,                  L3ContiguousBlockSize));
 
                         /* Invalidate the existing contiguous L3 block. */
                         for (size_t i = 0; i < L3ContiguousBlockSize / L3BlockSize; i++) {
                             /* Ensure that the entry is valid. */
-                            MESOSPHERE_ABORT_UNLESS(l3_entry[i].IsCompatibleWithAttribute(attr_before, true));
+                            MESOSPHERE_INIT_ABORT_UNLESS(l3_entry[i].IsCompatibleWithAttribute(attr_before, true));
                             static_cast<PageTableEntry *>(l3_entry)[i] = InvalidPageTableEntry;
                         }
                         cpu::DataSynchronizationBarrierInnerShareable();
@@ -471,10 +556,10 @@ namespace ams::kern::init {
                         size      -= L3ContiguousBlockSize;
                     } else {
                         /* Ensure that we are allowed to have an L3 block here. */
-                        MESOSPHERE_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), L3BlockSize));
-                        MESOSPHERE_ABORT_UNLESS(util::IsAligned(GetInteger(block),     L3BlockSize));
-                        MESOSPHERE_ABORT_UNLESS(util::IsAligned(size,                  L3BlockSize));
-                        MESOSPHERE_ABORT_UNLESS(l3_entry->IsCompatibleWithAttribute(attr_before, false));
+                        MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), L3BlockSize));
+                        MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(block),     L3BlockSize));
+                        MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(size,                  L3BlockSize));
+                        MESOSPHERE_INIT_ABORT_UNLESS(l3_entry->IsCompatibleWithAttribute(attr_before, false));
 
                         /* Invalidate the existing L3 block. */
                         *static_cast<PageTableEntry *>(l3_entry) = InvalidPageTableEntry;
@@ -493,6 +578,67 @@ namespace ams::kern::init {
                 cpu::DataSynchronizationBarrierInnerShareable();
             }
 
+            void PhysicallyRandomize(KVirtualAddress virt_addr, size_t size, bool do_copy) {
+                this->PhysicallyRandomize(virt_addr, size, L1BlockSize, do_copy);
+                this->PhysicallyRandomize(virt_addr, size, L2ContiguousBlockSize, do_copy);
+                this->PhysicallyRandomize(virt_addr, size, L2BlockSize, do_copy);
+                this->PhysicallyRandomize(virt_addr, size, L3ContiguousBlockSize, do_copy);
+                this->PhysicallyRandomize(virt_addr, size, L3BlockSize, do_copy);
+            }
+
+    };
+
+    class KInitialPageAllocator : public KInitialPageTable::IPageAllocator {
+        public:
+            struct State {
+                uintptr_t next_address;
+                uintptr_t free_bitmap;
+            };
+        private:
+            State state;
+        public:
+            constexpr ALWAYS_INLINE KInitialPageAllocator() : state{} { /* ... */ }
+
+            ALWAYS_INLINE void Initialize(uintptr_t address) {
+                this->state.next_address = address + BITSIZEOF(this->state.free_bitmap) * PageSize;
+                this->state.free_bitmap  = ~uintptr_t();
+            }
+
+            ALWAYS_INLINE void InitializeFromState(uintptr_t state_val) {
+                if (kern::GetTargetFirmware() >= kern::TargetFirmware_10_0_0) {
+                    this->state = *reinterpret_cast<State *>(state_val);
+                } else {
+                    this->state.next_address = state_val;
+                    this->state.free_bitmap  = 0;
+                }
+            }
+
+            ALWAYS_INLINE void GetFinalState(State *out) {
+                *out = this->state;
+                this->state = {};
+            }
+        public:
+            virtual KPhysicalAddress Allocate() override {
+                MESOSPHERE_INIT_ABORT_UNLESS(this->state.next_address != Null<uintptr_t>);
+                uintptr_t allocated = this->state.next_address;
+                if (this->state.free_bitmap != 0) {
+                    u64 index;
+                    uintptr_t mask;
+                    do {
+                        index = KSystemControl::Init::GenerateRandomRange(0, BITSIZEOF(this->state.free_bitmap) - 1);
+                        mask  = (static_cast<uintptr_t>(1) << index);
+                    } while ((this->state.free_bitmap & mask) == 0);
+                    this->state.free_bitmap &= ~mask;
+                    allocated = this->state.next_address - ((BITSIZEOF(this->state.free_bitmap) - index) * PageSize);
+                } else {
+                    this->state.next_address += PageSize;
+                }
+
+                std::memset(reinterpret_cast<void *>(allocated), 0, PageSize);
+                return allocated;
+            }
+
+            /* No need to override free. The default does nothing, and so would we. */
     };
 
 }

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu.hpp

@@ -15,9 +15,27 @@
  */
 #pragma once
 #include <vapours.hpp>
-#include "kern_cpu_system_registers.hpp"
+#include <mesosphere/arch/arm64/kern_cpu_system_registers.hpp>
+#include <mesosphere/kern_select_userspace_memory_access.hpp>
 
-namespace ams::kern::arm64::cpu {
+namespace ams::kern::arch::arm64::cpu {
+
+#if defined(ATMOSPHERE_CPU_ARM_CORTEX_A57) || defined(ATMOSPHERE_CPU_ARM_CORTEX_A53)
+    constexpr inline size_t InstructionCacheLineSize = 0x40;
+    constexpr inline size_t DataCacheLineSize        = 0x40;
+    constexpr inline size_t NumPerformanceCounters   = 6;
+#else
+    #error "Unknown CPU for cache line sizes"
+#endif
+
+#if defined(ATMOSPHERE_BOARD_NINTENDO_NX)
+    constexpr inline size_t NumCores = 4;
+#else
+    #error "Unknown Board for cpu::NumCores"
+#endif
+
+    /* Initialization. */
+    NOINLINE void InitializeInterruptThreads(s32 core_id);
 
     /* Helpers for managing memory state. */
     ALWAYS_INLINE void DataSynchronizationBarrier() {
@@ -46,13 +64,154 @@ namespace ams::kern::arm64::cpu {
         EnsureInstructionConsistency();
     }
 
+    ALWAYS_INLINE void SwitchProcess(u64 ttbr, u32 proc_id) {
+        SetTtbr0El1(ttbr);
+        ContextIdRegisterAccessor(0).SetProcId(proc_id).Store();
+        InstructionMemoryBarrier();
+    }
+
+    /* Performance counter helpers. */
+    ALWAYS_INLINE u64 GetCycleCounter() {
+        return cpu::GetPmcCntrEl0();
+    }
+
+    ALWAYS_INLINE u32 GetPerformanceCounter(s32 n) {
+        u64 counter = 0;
+        if (n < static_cast<s32>(NumPerformanceCounters)) {
+            switch (n) {
+                case 0:
+                    counter = cpu::GetPmevCntr0El0();
+                    break;
+                case 1:
+                    counter = cpu::GetPmevCntr1El0();
+                    break;
+                case 2:
+                    counter = cpu::GetPmevCntr2El0();
+                    break;
+                case 3:
+                    counter = cpu::GetPmevCntr3El0();
+                    break;
+                case 4:
+                    counter = cpu::GetPmevCntr4El0();
+                    break;
+                case 5:
+                    counter = cpu::GetPmevCntr5El0();
+                    break;
+                default:
+                    break;
+            }
+        }
+        return static_cast<u32>(counter);
+    }
+
+    /* Helper for address access. */
+    ALWAYS_INLINE bool GetPhysicalAddressWritable(KPhysicalAddress *out, KVirtualAddress addr, bool privileged = false) {
+        const uintptr_t va = GetInteger(addr);
+
+        if (privileged) {
+            __asm__ __volatile__("at s1e1w, %[va]" :: [va]"r"(va) : "memory");
+        } else {
+            __asm__ __volatile__("at s1e0w, %[va]" :: [va]"r"(va) : "memory");
+        }
+        InstructionMemoryBarrier();
+
+        u64 par = GetParEl1();
+
+        if (par & 0x1) {
+            return false;
+        }
+
+        if (out) {
+            *out = KPhysicalAddress((par & 0xFFFFFFFFF000ull) | (va & 0xFFFull));
+        }
+        return true;
+    }
+
+    ALWAYS_INLINE bool GetPhysicalAddressReadable(KPhysicalAddress *out, KVirtualAddress addr, bool privileged = false) {
+        const uintptr_t va = GetInteger(addr);
+
+        if (privileged) {
+            __asm__ __volatile__("at s1e1r, %[va]" :: [va]"r"(va) : "memory");
+        } else {
+            __asm__ __volatile__("at s1e0r, %[va]" :: [va]"r"(va) : "memory");
+        }
+        InstructionMemoryBarrier();
+
+        u64 par = GetParEl1();
+
+        if (par & 0x1) {
+            return false;
+        }
+
+        if (out) {
+            *out = KPhysicalAddress((par & 0xFFFFFFFFF000ull) | (va & 0xFFFull));
+        }
+        return true;
+    }
+
+    /* Synchronization helpers. */
+    NOINLINE void SynchronizeAllCores();
+
     /* Cache management helpers. */
-    void FlushEntireDataCacheShared();
-    void FlushEntireDataCacheLocal();
+    void ClearPageToZeroImpl(void *);
+    void FlushEntireDataCacheSharedForInit();
+    void FlushEntireDataCacheLocalForInit();
+
+    void FlushEntireDataCache();
+
+    Result InvalidateDataCache(void *addr, size_t size);
+    Result StoreDataCache(const void *addr, size_t size);
+    Result FlushDataCache(const void *addr, size_t size);
+    Result InvalidateInstructionCache(void *addr, size_t size);
+
+    ALWAYS_INLINE void ClearPageToZero(void *page) {
+        MESOSPHERE_ASSERT(util::IsAligned(reinterpret_cast<uintptr_t>(page), PageSize));
+        MESOSPHERE_ASSERT(page != nullptr);
+        ClearPageToZeroImpl(page);
+    }
+
+    ALWAYS_INLINE void InvalidateTlbByAsid(u32 asid) {
+        const u64 value = (static_cast<u64>(asid) << 48);
+        __asm__ __volatile__("tlbi aside1is, %[value]" :: [value]"r"(static_cast<u64>(value) << 48) : "memory");
+        EnsureInstructionConsistency();
+    }
+
+    ALWAYS_INLINE void InvalidateTlbByAsidAndVa(u32 asid, KProcessAddress virt_addr) {
+        const u64 value = (static_cast<u64>(asid) << 48) | ((GetInteger(virt_addr) >> 12) & 0xFFFFFFFFFFFul);
+        __asm__ __volatile__("tlbi aside1is, %[value]" :: [value]"r"(value) : "memory");
+        EnsureInstructionConsistency();
+    }
 
     ALWAYS_INLINE void InvalidateEntireTlb() {
         __asm__ __volatile__("tlbi vmalle1is" ::: "memory");
         EnsureInstructionConsistency();
     }
 
+    ALWAYS_INLINE void InvalidateEntireTlbDataOnly() {
+        __asm__ __volatile__("tlbi vmalle1is" ::: "memory");
+        DataSynchronizationBarrier();
+    }
+
+    ALWAYS_INLINE void InvalidateTlbByVaDataOnly(KProcessAddress virt_addr) {
+        const u64 value = ((GetInteger(virt_addr) >> 12) & 0xFFFFFFFFFFFul);
+        __asm__ __volatile__("tlbi vaae1is, %[value]" :: [value]"r"(value) : "memory");
+        DataSynchronizationBarrier();
+    }
+
+    ALWAYS_INLINE uintptr_t GetCoreLocalRegionAddress() {
+        register uintptr_t x18 asm("x18");
+        __asm__ __volatile__("" : [x18]"=r"(x18));
+        return x18;
+    }
+
+    ALWAYS_INLINE void SetCoreLocalRegionAddress(uintptr_t value) {
+        register uintptr_t x18 asm("x18") = value;
+        __asm__ __volatile__("":: [x18]"r"(x18));
+        SetTpidrEl1(value);
+    }
+
+    ALWAYS_INLINE void SwitchThreadLocalRegion(uintptr_t tlr) {
+        cpu::SetTpidrRoEl0(tlr);
+    }
+
 }

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu_system_registers.hpp

@@ -16,7 +16,7 @@
 #pragma once
 #include <vapours.hpp>
 
-namespace ams::kern::arm64::cpu {
+namespace ams::kern::arch::arm64::cpu {
 
     #define MESOSPHERE_CPU_GET_SYSREG(name)                                            \
         ({                                                                             \
@@ -37,8 +37,14 @@ namespace ams::kern::arm64::cpu {
     MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(Ttbr0El1, ttbr0_el1)
     MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(Ttbr1El1, ttbr1_el1)
 
-    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(MairEl1, mair_el1)
     MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(TcrEl1, tcr_el1)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(MairEl1, mair_el1)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(TpidrEl1, tpidr_el1)
+
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(VbarEl1, vbar_el1)
+
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(FarEl1, far_el1)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(ParEl1, par_el1)
 
     MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(SctlrEl1, sctlr_el1)
 
@@ -46,21 +52,215 @@ namespace ams::kern::arm64::cpu {
     MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(CpuEctlrEl1, s3_1_c15_c2_1)
 
     MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(CsselrEl1, csselr_el1)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(CcsidrEl1, ccsidr_el1)
+
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(OslarEl1, oslar_el1)
+
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(TpidrRoEl0, tpidrro_el0)
+
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(EsrEl1, esr_el1)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(Afsr0El1, afsr0_el1)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(Afsr1El1, afsr1_el1)
+
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(PmUserEnrEl0, pmuserenr_el0)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(PmcCntrEl0,   pmccntr_el0)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(PmevCntr0El0, pmevcntr0_el0)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(PmevCntr1El0, pmevcntr1_el0)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(PmevCntr2El0, pmevcntr2_el0)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(PmevCntr3El0, pmevcntr3_el0)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(PmevCntr4El0, pmevcntr4_el0)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(PmevCntr5El0, pmevcntr5_el0)
+
+    #define FOR_I_IN_0_TO_15(HANDLER, ...)                                                                              \
+        HANDLER(0,  ## __VA_ARGS__) HANDLER(1,  ## __VA_ARGS__) HANDLER(2,  ## __VA_ARGS__) HANDLER(3,  ## __VA_ARGS__) \
+        HANDLER(4,  ## __VA_ARGS__) HANDLER(5,  ## __VA_ARGS__) HANDLER(6,  ## __VA_ARGS__) HANDLER(7,  ## __VA_ARGS__) \
+        HANDLER(8,  ## __VA_ARGS__) HANDLER(9,  ## __VA_ARGS__) HANDLER(10, ## __VA_ARGS__) HANDLER(11, ## __VA_ARGS__) \
+        HANDLER(12, ## __VA_ARGS__) HANDLER(13, ## __VA_ARGS__) HANDLER(14, ## __VA_ARGS__) HANDLER(15, ## __VA_ARGS__) \
+
+    #define MESOSPHERE_CPU_DEFINE_DBG_SYSREG_ACCESSORS(ID, ...)                     \
+        MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(DbgWcr##ID##El1, dbgwcr##ID##_el1)   \
+        MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(DbgWvr##ID##El1, dbgwvr##ID##_el1)   \
+        MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(DbgBcr##ID##El1, dbgbcr##ID##_el1)   \
+        MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(DbgBvr##ID##El1, dbgbvr##ID##_el1)
+
+    FOR_I_IN_0_TO_15(MESOSPHERE_CPU_DEFINE_DBG_SYSREG_ACCESSORS)
+
+    #undef MESOSPHERE_CPU_DEFINE_DBG_SYSREG_ACCESSORS
 
     /* Base class for register accessors. */
-    class GenericRegisterAccessor {
+    class GenericRegisterAccessorBase {
+        NON_COPYABLE(GenericRegisterAccessorBase);
+        NON_MOVEABLE(GenericRegisterAccessorBase);
         private:
             u64 value;
         public:
-            ALWAYS_INLINE GenericRegisterAccessor(u64 v) : value(v) { /* ... */ }
+            constexpr ALWAYS_INLINE GenericRegisterAccessorBase(u64 v) : value(v) { /* ... */ }
         protected:
+            constexpr ALWAYS_INLINE u64 GetValue() const {
+                return this->value;
+            }
+
             constexpr ALWAYS_INLINE u64 GetBits(size_t offset, size_t count) const {
                 return (this->value >> offset) & ((1ul << count) - 1);
             }
+
+            constexpr ALWAYS_INLINE void SetBits(size_t offset, size_t count, u64 value) {
+                const u64 mask = ((1ul << count) - 1) << offset;
+                this->value &= ~mask;
+                this->value |= (value & (mask >> offset)) << offset;
+            }
+
+            constexpr ALWAYS_INLINE void SetBitsDirect(size_t offset, size_t count, u64 value) {
+                const u64 mask = ((1ul << count) - 1) << offset;
+                this->value &= ~mask;
+                this->value |= (value & mask);
+            }
+
+            constexpr ALWAYS_INLINE void SetBit(size_t offset, bool enabled) {
+                const u64 mask = 1ul << offset;
+                if (enabled) {
+                    this->value |= mask;
+                } else {
+                    this->value &= ~mask;
+                }
+            }
     };
 
-    /* Special code for main id register. */
-    class MainIdRegisterAccessor : public GenericRegisterAccessor {
+    template<typename Derived>
+    class GenericRegisterAccessor : public GenericRegisterAccessorBase {
+        public:
+            constexpr ALWAYS_INLINE GenericRegisterAccessor(u64 v) : GenericRegisterAccessorBase(v) { /* ... */ }
+        protected:
+            ALWAYS_INLINE void Store() const {
+                static_cast<const Derived *>(this)->Store();
+            }
+    };
+
+    #define MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(name) class name##RegisterAccessor : public GenericRegisterAccessor<name##RegisterAccessor>
+
+    #define MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(accessor, reg_name)                                                  \
+        ALWAYS_INLINE accessor##RegisterAccessor() : GenericRegisterAccessor(MESOSPHERE_CPU_GET_SYSREG(reg_name)) { /* ... */ } \
+        constexpr ALWAYS_INLINE accessor##RegisterAccessor(u64 v) : GenericRegisterAccessor(v) { /* ... */ }                    \
+                                                                                                                                \
+        ALWAYS_INLINE void Store() { const u64 v = this->GetValue(); MESOSPHERE_CPU_SET_SYSREG(reg_name, v); }
+
+    /* Accessors. */
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(MemoryAccessIndirection) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(MemoryAccessIndirection, mair_el1)
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(TranslationControl) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(TranslationControl, tcr_el1)
+
+            constexpr ALWAYS_INLINE size_t GetT1Size() const {
+                const size_t shift_value = this->GetBits(16, 6);
+                return size_t(1) << (size_t(64) - shift_value);
+            }
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(ArchitecturalFeatureAccessControl) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(ArchitecturalFeatureAccessControl, cpacr_el1)
+
+            constexpr ALWAYS_INLINE decltype(auto) SetFpEnabled(bool en) {
+                if (en) {
+                    this->SetBits(20, 2, 0x3);
+                } else {
+                    this->SetBits(20, 2, 0x0);
+                }
+                return *this;
+            }
+
+            constexpr ALWAYS_INLINE bool IsFpEnabled() {
+                return this->GetBits(20, 2) != 0;
+            }
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(DebugFeature) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(DebugFeature, id_aa64dfr0_el1)
+
+            constexpr ALWAYS_INLINE size_t GetNumWatchpoints() const {
+                return this->GetBits(20, 4);
+            }
+
+            constexpr ALWAYS_INLINE size_t GetNumBreakpoints() const {
+                return this->GetBits(12, 4);
+            }
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(MonitorDebugSystemControl) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(MonitorDebugSystemControl, mdscr_el1)
+
+            constexpr ALWAYS_INLINE bool GetMde() const {
+                return this->GetBits(15, 1) != 0;
+            }
+
+            constexpr ALWAYS_INLINE size_t GetTdcc() const {
+                return this->GetBits(12, 1) != 0;
+            }
+
+            constexpr ALWAYS_INLINE decltype(auto) SetMde(bool set) {
+                this->SetBit(15, set);
+                return *this;
+            }
+
+            constexpr ALWAYS_INLINE decltype(auto) SetTdcc(bool set) {
+                this->SetBit(12, set);
+                return *this;
+            }
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(MultiprocessorAffinity) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(MultiprocessorAffinity, mpidr_el1)
+
+            constexpr ALWAYS_INLINE u64 GetAff0() const {
+                return this->GetBits(0, 8);
+            }
+
+            constexpr ALWAYS_INLINE u64 GetAff1() const {
+                return this->GetBits(8, 8);
+            }
+
+            constexpr ALWAYS_INLINE u64 GetAff2() const {
+                return this->GetBits(16, 8);
+            }
+
+            constexpr ALWAYS_INLINE u64 GetAff3() const {
+                return this->GetBits(32, 8);
+            }
+
+            constexpr ALWAYS_INLINE u64 GetCpuOnArgument() const {
+                constexpr u64 Mask = 0x000000FF00FFFF00ul;
+                return this->GetValue() & Mask;
+            }
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(ThreadId) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(ThreadId, tpidr_el1)
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(OsLockAccess) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(OsLockAccess, oslar_el1)
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(ContextId) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(ContextId, contextidr_el1)
+
+            constexpr ALWAYS_INLINE decltype(auto) SetProcId(u32 proc_id) {
+                this->SetBits(0, BITSIZEOF(proc_id), proc_id);
+                return *this;
+            }
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(MainId) {
         public:
             enum class Implementer {
                 ArmLimited = 0x41,
@@ -70,7 +270,7 @@ namespace ams::kern::arm64::cpu {
                 CortexA57  = 0xD07,
             };
         public:
-            ALWAYS_INLINE MainIdRegisterAccessor() : GenericRegisterAccessor(MESOSPHERE_CPU_GET_SYSREG(midr_el1)) { /* ... */ }
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(MainId, midr_el1)
         public:
             constexpr ALWAYS_INLINE Implementer GetImplementer() const {
                 return static_cast<Implementer>(this->GetBits(24, 8));
@@ -93,10 +293,69 @@ namespace ams::kern::arm64::cpu {
             }
     };
 
-    /* Accessors for cache registers. */
-    class CacheLineIdAccessor : public GenericRegisterAccessor {
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(SystemControl) {
         public:
-            ALWAYS_INLINE CacheLineIdAccessor() : GenericRegisterAccessor(MESOSPHERE_CPU_GET_SYSREG(clidr_el1)) { /* ... */ }
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(SystemControl, sctlr_el1)
+
+            constexpr ALWAYS_INLINE decltype(auto) SetWxn(bool en) {
+                this->SetBit(19, en);
+                return *this;
+            }
+    };
+
+    /* Accessors for timer registers. */
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(CounterTimerKernelControl) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(CounterTimerKernelControl, cntkctl_el1)
+
+            constexpr ALWAYS_INLINE decltype(auto) SetEl0PctEn(bool en) {
+                this->SetBit(0, en);
+                return *this;
+            }
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(CounterTimerPhysicalTimerControl) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(CounterTimerPhysicalTimerControl, cntp_ctl_el0)
+
+            constexpr ALWAYS_INLINE decltype(auto) SetEnable(bool en) {
+                this->SetBit(0, en);
+                return *this;
+            }
+
+            constexpr ALWAYS_INLINE decltype(auto) SetIMask(bool en) {
+                this->SetBit(1, en);
+                return *this;
+            }
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(CounterTimerPhysicalTimerCompareValue) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(CounterTimerPhysicalTimerCompareValue, cntp_cval_el0)
+
+            constexpr ALWAYS_INLINE u64 GetCompareValue() {
+                return this->GetValue();
+            }
+
+            constexpr ALWAYS_INLINE decltype(auto) SetCompareValue(u64 value) {
+                this->SetBits(0, BITSIZEOF(value), value);
+                return *this;
+            }
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(CounterTimerPhysicalCountValue) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(CounterTimerPhysicalCountValue, cntpct_el0)
+
+            constexpr ALWAYS_INLINE u64 GetCount() {
+                return this->GetValue();
+            }
+    };
+
+    /* Accessors for cache registers. */
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(CacheLineId) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(CacheLineId, clidr_el1)
         public:
             constexpr ALWAYS_INLINE int GetLevelsOfCoherency() const {
                 return static_cast<int>(this->GetBits(24, 3));
@@ -109,9 +368,9 @@ namespace ams::kern::arm64::cpu {
             /* TODO: Other bitfield accessors? */
     };
 
-    class CacheSizeIdAccessor : public GenericRegisterAccessor {
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(CacheSizeId) {
         public:
-            ALWAYS_INLINE CacheSizeIdAccessor() : GenericRegisterAccessor(MESOSPHERE_CPU_GET_SYSREG(ccsidr_el1)) { /* ... */ }
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(CacheSizeId, ccsidr_el1)
         public:
             constexpr ALWAYS_INLINE int GetNumberOfSets() const {
                 return static_cast<int>(this->GetBits(13, 15));
@@ -128,6 +387,9 @@ namespace ams::kern::arm64::cpu {
             /* TODO: Other bitfield accessors? */
     };
 
+    #undef  FOR_I_IN_0_TO_15
+    #undef  MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS
+    #undef  MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS
     #undef  MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS
     #undef  MESOSPHERE_CPU_GET_SYSREG
     #undef  MESOSPHERE_CPU_SET_SYSREG

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_debug.hpp

@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_select_cpu.hpp>
+#include <mesosphere/kern_k_debug_base.hpp>
+
+namespace ams::kern {
+
+    class KThread;
+    class KProcess;
+
+}
+
+namespace ams::kern::arch::arm64 {
+
+    class KDebug final : public KAutoObjectWithSlabHeapAndContainer<KDebug, KDebugBase> {
+        MESOSPHERE_AUTOOBJECT_TRAITS(KDebug, KSynchronizationObject);
+        public:
+            /* TODO: This is a placeholder definition. */
+    };
+
+}

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_exception_context.hpp

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+
+namespace ams::kern::arch::arm64 {
+
+    struct KExceptionContext {
+        u64 x[(30 - 0) + 1];
+        u64 sp;
+        u64 pc;
+        u64 psr;
+        u64 tpidr;
+        u64 reserved;
+    };
+    static_assert(sizeof(KExceptionContext) == 0x120);
+
+}

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_hardware_timer.hpp

@@ -0,0 +1,94 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_select_cpu.hpp>
+#include <mesosphere/kern_k_hardware_timer_base.hpp>
+
+namespace ams::kern::arch::arm64 {
+
+    namespace impl {
+
+        class KHardwareTimerInterruptTask;
+
+    }
+
+    class KHardwareTimer : public KHardwareTimerBase {
+        public:
+            constexpr KHardwareTimer() : KHardwareTimerBase() { /* ... */ }
+        public:
+            /* Public API. */
+            NOINLINE void Initialize(s32 core_id);
+            NOINLINE void Finalize();
+
+            static s64 GetTick() {
+                return GetCount();
+            }
+
+            void RegisterAbsoluteTask(KTimerTask *task, s64 task_time) {
+                KScopedDisableDispatch dd;
+                KScopedSpinLock lk(this->GetLock());
+
+                if (this->RegisterAbsoluteTaskImpl(task, task_time)) {
+                    SetCompareValue(task_time);
+                    EnableInterrupt();
+                }
+            }
+        private:
+            friend class impl::KHardwareTimerInterruptTask;
+            NOINLINE void DoInterruptTask();
+        private:
+            /* Hardware register accessors. */
+            static ALWAYS_INLINE void InitializeGlobalTimer() {
+                /* Set kernel control. */
+                cpu::CounterTimerKernelControlRegisterAccessor(0).SetEl0PctEn(true).Store();
+
+                /* Disable the physical timer. */
+                cpu::CounterTimerPhysicalTimerControlRegisterAccessor(0).SetEnable(false).SetIMask(false).Store();
+
+                /* Set the compare value to the maximum. */
+                cpu::CounterTimerPhysicalTimerCompareValueRegisterAccessor(0).SetCompareValue(std::numeric_limits<u64>::max()).Store();
+
+                /* Enable the physical timer, with interrupt masked. */
+                cpu::CounterTimerPhysicalTimerControlRegisterAccessor(0).SetEnable(true).SetIMask(true).Store();
+            }
+
+            static ALWAYS_INLINE void EnableInterrupt() {
+                cpu::CounterTimerPhysicalTimerControlRegisterAccessor(0).SetEnable(true).SetIMask(false).Store();
+            }
+
+            static ALWAYS_INLINE void DisableInterrupt() {
+                cpu::CounterTimerPhysicalTimerControlRegisterAccessor(0).SetEnable(true).SetIMask(true).Store();
+            }
+
+            static ALWAYS_INLINE void StopTimer() {
+                /* Set the compare value to the maximum. */
+                cpu::CounterTimerPhysicalTimerCompareValueRegisterAccessor(0).SetCompareValue(std::numeric_limits<u64>::max()).Store();
+
+                /* Disable the physical timer. */
+                cpu::CounterTimerPhysicalTimerControlRegisterAccessor(0).SetEnable(false).SetIMask(false).Store();
+            }
+
+            static ALWAYS_INLINE s64 GetCount() {
+                return cpu::CounterTimerPhysicalCountValueRegisterAccessor().GetCount();
+            }
+
+            static ALWAYS_INLINE void SetCompareValue(s64 value) {
+                cpu::CounterTimerPhysicalTimerCompareValueRegisterAccessor(0).SetCompareValue(static_cast<u64>(value)).Store();
+            }
+
+    };
+
+}

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_interrupt_controller.hpp

@@ -0,0 +1,275 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <vapours.hpp>
+#include <mesosphere/kern_select_cpu.hpp>
+#include <mesosphere/kern_k_typed_address.hpp>
+
+namespace ams::kern::arch::arm64 {
+
+    struct GicDistributor {
+        u32 ctlr;
+        u32 typer;
+        u32 iidr;
+        u32 reserved_0x0c;
+        u32 statusr;
+        u32 reserved_0x14[3];
+        u32 impldef_0x20[8];
+        u32 setspi_nsr;
+        u32 reserved_0x44;
+        u32 clrspi_nsr;
+        u32 reserved_0x4c;
+        u32 setspi_sr;
+        u32 reserved_0x54;
+        u32 clrspi_sr;
+        u32 reserved_0x5c[9];
+        u32 igroupr[32];
+        u32 isenabler[32];
+        u32 icenabler[32];
+        u32 ispendr[32];
+        u32 icpendr[32];
+        u32 isactiver[32];
+        u32 icactiver[32];
+        union {
+            u8  bytes[1020];
+            u32 words[255];
+        } ipriorityr;
+        u32 _0x7fc;
+        union {
+            u8  bytes[1020];
+            u32 words[255];
+        } itargetsr;
+        u32 _0xbfc;
+        u32 icfgr[64];
+        u32 igrpmodr[32];
+        u32 _0xd80[32];
+        u32 nsacr[64];
+        u32 sgir;
+        u32 _0xf04[3];
+        u32 cpendsgir[4];
+        u32 spendsgir[4];
+        u32 reserved_0xf30[52];
+
+        static constexpr size_t SgirCpuTargetListShift = 16;
+
+        enum SgirTargetListFilter : u32 {
+            SgirTargetListFilter_CpuTargetList = (0 << 24),
+            SgirTargetListFilter_Others        = (1 << 24),
+            SgirTargetListFilter_Self          = (2 << 24),
+            SgirTargetListFilter_Reserved      = (3 << 24),
+        };
+    };
+    static_assert(std::is_pod<GicDistributor>::value);
+    static_assert(sizeof(GicDistributor) == 0x1000);
+
+    struct GicCpuInterface {
+        u32 ctlr;
+        u32 pmr;
+        u32 bpr;
+        u32 iar;
+        u32 eoir;
+        u32 rpr;
+        u32 hppir;
+        u32 abpr;
+        u32 aiar;
+        u32 aeoir;
+        u32 ahppir;
+        u32 statusr;
+        u32 reserved_30[4];
+        u32 impldef_40[36];
+        u32 apr[4];
+        u32 nsapr[4];
+        u32 reserved_f0[3];
+        u32 iidr;
+        u32 reserved_100[960];
+        u32 dir;
+        u32 _0x1004[1023];
+    };
+    static_assert(std::is_pod<GicCpuInterface>::value);
+    static_assert(sizeof(GicCpuInterface) == 0x2000);
+
+    struct KInterruptController {
+        NON_COPYABLE(KInterruptController);
+        NON_MOVEABLE(KInterruptController);
+        public:
+            static constexpr s32 NumSoftwareInterrupts = 16;
+            static constexpr s32 NumLocalInterrupts = NumSoftwareInterrupts + 16;
+            static constexpr s32 NumGlobalInterrupts = 988;
+            static constexpr s32 NumInterrupts = NumLocalInterrupts + NumGlobalInterrupts;
+            static constexpr s32 NumPriorityLevels = 4;
+        public:
+            struct LocalState {
+                u32 local_isenabler[NumLocalInterrupts / 32];
+                u32 local_ipriorityr[NumLocalInterrupts / 4];
+                u32 local_targetsr[NumLocalInterrupts / 4];
+                u32 local_icfgr[NumLocalInterrupts / 16];
+            };
+
+            struct GlobalState {
+                u32 global_isenabler[NumGlobalInterrupts / 32];
+                u32 global_ipriorityr[NumGlobalInterrupts / 4];
+                u32 global_targetsr[NumGlobalInterrupts / 4];
+                u32 global_icfgr[NumGlobalInterrupts / 16];
+            };
+
+            enum PriorityLevel : u8 {
+                PriorityLevel_High  = 0,
+                PriorityLevel_Low   = NumPriorityLevels - 1,
+
+                PriorityLevel_Timer     = 1,
+                PriorityLevel_Scheduler = 2,
+            };
+        private:
+            static inline u32 s_mask[cpu::NumCores];
+        private:
+            volatile GicDistributor  *gicd;
+            volatile GicCpuInterface *gicc;
+        public:
+            constexpr KInterruptController() : gicd(nullptr), gicc(nullptr) { /* ... */ }
+
+            void Initialize(s32 core_id);
+            void Finalize(s32 core_id);
+        public:
+            u32 GetIrq() const {
+                return this->gicc->iar;
+            }
+
+            static constexpr s32 ConvertRawIrq(u32 irq) {
+                return (irq == 0x3FF) ? -1 : (irq & 0x3FF);
+            }
+
+            void Enable(s32 irq) const {
+                this->gicd->isenabler[irq / BITSIZEOF(u32)] = (1u << (irq % BITSIZEOF(u32)));
+            }
+
+            void Disable(s32 irq) const {
+                this->gicd->icenabler[irq / BITSIZEOF(u32)] = (1u << (irq % BITSIZEOF(u32)));
+            }
+
+            void Clear(s32 irq) const {
+                this->gicd->icpendr[irq / BITSIZEOF(u32)] = (1u << (irq % BITSIZEOF(u32)));
+            }
+
+            void SetTarget(s32 irq, s32 core_id) const {
+                this->gicd->itargetsr.bytes[irq] |= GetGicMask(core_id);
+            }
+
+            void ClearTarget(s32 irq, s32 core_id) const {
+                this->gicd->itargetsr.bytes[irq] &= ~GetGicMask(core_id);
+            }
+
+            void SetPriorityLevel(s32 irq, s32 level) const {
+                MESOSPHERE_ASSERT(PriorityLevel_High <= level && level <= PriorityLevel_Low);
+                this->gicd->ipriorityr.bytes[irq] = ToGicPriorityValue(level);
+            }
+
+            s32 GetPriorityLevel(s32 irq) const {
+                return FromGicPriorityValue(this->gicd->ipriorityr.bytes[irq]);
+            }
+
+            void SetPriorityLevel(s32 level) const {
+                MESOSPHERE_ASSERT(PriorityLevel_High <= level && level <= PriorityLevel_Low);
+                this->gicc->pmr = ToGicPriorityValue(level);
+            }
+
+            void SetEdge(s32 irq) const {
+                u32 cfg = this->gicd->icfgr[irq / (BITSIZEOF(u32) / 2)];
+                cfg &= ~(0x3 << (2 * (irq % (BITSIZEOF(u32) / 2))));
+                cfg |=  (0x2 << (2 * (irq % (BITSIZEOF(u32) / 2))));
+                this->gicd->icfgr[irq / (BITSIZEOF(u32) / 2)] = cfg;
+            }
+
+            void SetLevel(s32 irq) const {
+                u32 cfg = this->gicd->icfgr[irq / (BITSIZEOF(u32) / 2)];
+                cfg &= ~(0x3 << (2 * (irq % (BITSIZEOF(u32) / 2))));
+                cfg |=  (0x0 << (2 * (irq % (BITSIZEOF(u32) / 2))));
+                this->gicd->icfgr[irq / (BITSIZEOF(u32) / 2)] = cfg;
+            }
+
+            void SendInterProcessorInterrupt(s32 irq, u64 core_mask) {
+                MESOSPHERE_ASSERT(IsSoftware(irq));
+                this->gicd->sgir = GetCpuTargetListMask(irq, core_mask);
+            }
+
+            void SendInterProcessorInterrupt(s32 irq) {
+                MESOSPHERE_ASSERT(IsSoftware(irq));
+                this->gicd->sgir = GicDistributor::SgirTargetListFilter_Others | irq;
+            }
+
+            void EndOfInterrupt(u32 irq) const {
+                this->gicc->eoir = irq;
+            }
+
+            bool IsInterruptDefined(s32 irq) {
+                const s32 num_interrupts = std::min(32 + 32 * (this->gicd->typer & 0x1F), static_cast<u32>(NumInterrupts));
+                return (0 <= irq && irq < num_interrupts);
+            }
+
+            /* TODO: Implement more KInterruptController functionality. */
+        public:
+            static constexpr ALWAYS_INLINE bool IsSoftware(s32 id) {
+                MESOSPHERE_ASSERT(0 <= id && id < NumInterrupts);
+                return id < NumSoftwareInterrupts;
+            }
+
+            static constexpr ALWAYS_INLINE bool IsLocal(s32 id) {
+                MESOSPHERE_ASSERT(0 <= id && id < NumInterrupts);
+                return id < NumLocalInterrupts;
+            }
+
+            static constexpr ALWAYS_INLINE bool IsGlobal(s32 id) {
+                MESOSPHERE_ASSERT(0 <= id && id < NumInterrupts);
+                return NumLocalInterrupts <= id;
+            }
+
+            static constexpr size_t GetGlobalInterruptIndex(s32 id) {
+                MESOSPHERE_ASSERT(IsGlobal(id));
+                return id - NumLocalInterrupts;
+            }
+
+            static constexpr size_t GetLocalInterruptIndex(s32 id) {
+                MESOSPHERE_ASSERT(IsLocal(id));
+                return id;
+            }
+        private:
+            static constexpr size_t PriorityShift = BITSIZEOF(u8) - __builtin_ctz(NumPriorityLevels);
+            static_assert(PriorityShift < BITSIZEOF(u8));
+
+            static constexpr ALWAYS_INLINE u8 ToGicPriorityValue(s32 level) {
+                return (level << PriorityShift) | ((1 << PriorityShift) - 1);
+            }
+
+            static constexpr ALWAYS_INLINE s32 FromGicPriorityValue(u8 priority) {
+                return (priority >> PriorityShift) & (NumPriorityLevels - 1);
+            }
+
+            static constexpr ALWAYS_INLINE s32 GetCpuTargetListMask(s32 irq, u64 core_mask) {
+                MESOSPHERE_ASSERT(IsSoftware(irq));
+                MESOSPHERE_ASSERT(core_mask < (1ul << cpu::NumCores));
+                return GicDistributor::SgirTargetListFilter_CpuTargetList | irq | (static_cast<u16>(core_mask) << GicDistributor::SgirCpuTargetListShift);
+            }
+
+            static ALWAYS_INLINE s32 GetGicMask(s32 core_id) {
+                return s_mask[core_id];
+            }
+
+            ALWAYS_INLINE void SetGicMask(s32 core_id) const {
+                s_mask[core_id] = this->gicd->itargetsr.bytes[0];
+            }
+
+            NOINLINE void SetupInterruptLines(s32 core_id) const;
+    };
+}

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_interrupt_manager.hpp

@@ -0,0 +1,130 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <vapours.hpp>
+#include <mesosphere/kern_select_cpu.hpp>
+#include <mesosphere/kern_k_spin_lock.hpp>
+#include <mesosphere/kern_k_interrupt_task.hpp>
+#include <mesosphere/kern_select_interrupt_controller.hpp>
+
+namespace ams::kern::arch::arm64 {
+
+    class KInterruptManager {
+        NON_COPYABLE(KInterruptManager);
+        NON_MOVEABLE(KInterruptManager);
+        private:
+            struct KCoreLocalInterruptEntry {
+                KInterruptHandler *handler;
+                bool manually_cleared;
+                bool needs_clear;
+                u8 priority;
+
+                constexpr KCoreLocalInterruptEntry()
+                    : handler(nullptr), manually_cleared(false), needs_clear(false), priority(KInterruptController::PriorityLevel_Low)
+                {
+                    /* ... */
+                }
+            };
+
+            struct KGlobalInterruptEntry {
+                KInterruptHandler *handler;
+                bool manually_cleared;
+                bool needs_clear;
+
+                constexpr KGlobalInterruptEntry() : handler(nullptr), manually_cleared(false), needs_clear(false) { /* ... */ }
+            };
+        private:
+            static KSpinLock s_lock;
+            static std::array<KGlobalInterruptEntry, KInterruptController::NumGlobalInterrupts> s_global_interrupts;
+            static KInterruptController::GlobalState s_global_state;
+            static bool s_global_state_saved;
+        private:
+            KCoreLocalInterruptEntry core_local_interrupts[KInterruptController::NumLocalInterrupts];
+            KInterruptController interrupt_controller;
+            KInterruptController::LocalState local_state;
+            bool local_state_saved;
+        private:
+            static ALWAYS_INLINE KSpinLock &GetLock() { return s_lock; }
+            static ALWAYS_INLINE KGlobalInterruptEntry &GetGlobalInterruptEntry(s32 irq) { return s_global_interrupts[KInterruptController::GetGlobalInterruptIndex(irq)]; }
+            ALWAYS_INLINE KCoreLocalInterruptEntry &GetLocalInterruptEntry(s32 irq) { return this->core_local_interrupts[KInterruptController::GetLocalInterruptIndex(irq)]; }
+
+            bool OnHandleInterrupt();
+        public:
+            constexpr KInterruptManager() : core_local_interrupts(), interrupt_controller(), local_state(), local_state_saved(false) { /* ... */ }
+            NOINLINE void Initialize(s32 core_id);
+            NOINLINE void Finalize(s32 core_id);
+
+            bool IsInterruptDefined(s32 irq) {
+                return this->interrupt_controller.IsInterruptDefined(irq);
+            }
+
+            NOINLINE Result BindHandler(KInterruptHandler *handler, s32 irq, s32 core_id, s32 priority, bool manual_clear, bool level);
+            NOINLINE Result UnbindHandler(s32 irq, s32 core);
+
+            NOINLINE Result ClearInterrupt(s32 irq);
+            NOINLINE Result ClearInterrupt(s32 irq, s32 core_id);
+
+            ALWAYS_INLINE void SendInterProcessorInterrupt(s32 irq, u64 core_mask) {
+                this->interrupt_controller.SendInterProcessorInterrupt(irq, core_mask);
+            }
+
+            ALWAYS_INLINE void SendInterProcessorInterrupt(s32 irq) {
+                this->interrupt_controller.SendInterProcessorInterrupt(irq);
+            }
+
+            static void HandleInterrupt(bool user_mode);
+
+            /* Implement more KInterruptManager functionality. */
+        private:
+            Result BindGlobal(KInterruptHandler *handler, s32 irq, s32 core_id, s32 priority, bool manual_clear, bool level);
+            Result BindLocal(KInterruptHandler *handler, s32 irq, s32 priority, bool manual_clear);
+            Result UnbindGlobal(s32 irq);
+            Result UnbindLocal(s32 irq);
+            Result ClearGlobal(s32 irq);
+            Result ClearLocal(s32 irq);
+        public:
+            static ALWAYS_INLINE u32 DisableInterrupts() {
+                u64 intr_state;
+                __asm__ __volatile__("mrs %[intr_state], daif\n"
+                                     "msr daifset, #2"
+                                     : [intr_state]"=r"(intr_state)
+                                     :: "memory");
+                return intr_state;
+            }
+
+            static ALWAYS_INLINE u32 EnableInterrupts() {
+                u64 intr_state;
+                __asm__ __volatile__("mrs %[intr_state], daif\n"
+                                     "msr daifclr, #2"
+                                     : [intr_state]"=r"(intr_state)
+                                     :: "memory");
+                return intr_state;
+            }
+
+            static ALWAYS_INLINE void RestoreInterrupts(u32 intr_state) {
+                u64 cur_state;
+                __asm__ __volatile__("mrs %[cur_state], daif" : [cur_state]"=r"(cur_state));
+                __asm__ __volatile__("msr daif, %[intr_state]" :: [intr_state]"r"((cur_state & ~0x80ul) | (intr_state & 0x80)));
+            }
+
+            static ALWAYS_INLINE bool AreInterruptsEnabled() {
+                u64 intr_state;
+                __asm__ __volatile__("mrs %[intr_state], daif" : [intr_state]"=r"(intr_state));
+                return (intr_state & 0x80) == 0;
+            }
+    };
+
+}

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_interrupt_name.hpp

@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+
+namespace ams::kern::arch::arm64 {
+
+    namespace interrupt_name {
+
+        enum KInterruptName : s32 {
+            /* SGIs */
+            KInterruptName_ThreadTerminate        = 4,
+            KInterruptName_CacheOperation         = 5,
+            KInterruptName_Scheduler              = 6,
+
+            KInterruptName_PerformanceCounter     = 8,
+
+            /* PPIs */
+    #if defined(ATMOSPHERE_BOARD_NINTENDO_NX)
+            KInterruptName_VirtualMaintenance     = 25,
+            KInterruptName_HypervisorTimer        = 26,
+            KInterruptName_VirtualTimer           = 27,
+            KInterruptName_LegacyNFiq             = 38,
+            KInterruptName_SecurePhysicalTimer    = 29,
+            KInterruptName_NonSecurePhysicalTimer = 30,
+            KInterruptName_LegacyNIrq             = 31,
+    #endif
+
+    #if defined(ATMOSPHERE_BOARD_NINTENDO_NX)
+            KInterruptName_MemoryController       = 109,
+    #endif
+        };
+
+    };
+
+}

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table.hpp

@@ -0,0 +1,275 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_select_cpu.hpp>
+#include <mesosphere/kern_k_page_table_base.hpp>
+#include <mesosphere/kern_k_page_group.hpp>
+#include <mesosphere/kern_k_page_table_manager.hpp>
+
+namespace ams::kern::arch::arm64 {
+
+    class KPageTable : public KPageTableBase {
+        NON_COPYABLE(KPageTable);
+        NON_MOVEABLE(KPageTable);
+        public:
+            using TraversalEntry   = KPageTableImpl::TraversalEntry;
+            using TraversalContext = KPageTableImpl::TraversalContext;
+
+            enum BlockType {
+                BlockType_L3Block,
+                BlockType_L3ContiguousBlock,
+                BlockType_L2Block,
+
+#ifdef ATMOSPHERE_BOARD_NINTENDO_NX
+                BlockType_L2TegraSmmuBlock,
+#endif
+
+                BlockType_L2ContiguousBlock,
+                BlockType_L1Block,
+
+                BlockType_Count,
+            };
+            static_assert(L3BlockSize == PageSize);
+            static constexpr size_t ContiguousPageSize = L3ContiguousBlockSize;
+
+#ifdef ATMOSPHERE_BOARD_NINTENDO_NX
+            static constexpr size_t L2TegraSmmuBlockSize = 2 * L2BlockSize;
+#endif
+            static constexpr size_t BlockSizes[BlockType_Count] = {
+                [BlockType_L3Block]           = L3BlockSize,
+                [BlockType_L3ContiguousBlock] = L3ContiguousBlockSize,
+                [BlockType_L2Block]           = L2BlockSize,
+#ifdef ATMOSPHERE_BOARD_NINTENDO_NX
+                [BlockType_L2TegraSmmuBlock]  = L2TegraSmmuBlockSize,
+#endif
+                [BlockType_L2ContiguousBlock] = L2ContiguousBlockSize,
+                [BlockType_L1Block]           = L1BlockSize,
+            };
+
+            static constexpr BlockType GetMaxBlockType() {
+                return BlockType_L1Block;
+            }
+
+            static constexpr size_t GetBlockSize(BlockType type) {
+                return BlockSizes[type];
+            }
+
+            static constexpr BlockType GetBlockType(size_t size) {
+                switch (size) {
+                    case L3BlockSize:           return BlockType_L3Block;
+                    case L3ContiguousBlockSize: return BlockType_L3ContiguousBlock;
+                    case L2BlockSize:           return BlockType_L2Block;
+#ifdef ATMOSPHERE_BOARD_NINTENDO_NX
+                    case L2TegraSmmuBlockSize:  return BlockType_L2TegraSmmuBlock;
+#endif
+                    case L2ContiguousBlockSize: return BlockType_L2ContiguousBlock;
+                    case L1BlockSize:           return BlockType_L1Block;
+                    MESOSPHERE_UNREACHABLE_DEFAULT_CASE();
+                }
+            }
+
+            static constexpr size_t GetSmallerAlignment(size_t alignment) {
+                MESOSPHERE_ASSERT(alignment > L3BlockSize);
+                return KPageTable::GetBlockSize(static_cast<KPageTable::BlockType>(KPageTable::GetBlockType(alignment) - 1));
+            }
+
+            static constexpr size_t GetLargerAlignment(size_t alignment) {
+                MESOSPHERE_ASSERT(alignment < L1BlockSize);
+                return KPageTable::GetBlockSize(static_cast<KPageTable::BlockType>(KPageTable::GetBlockType(alignment) + 1));
+            }
+        private:
+            KPageTableManager *manager;
+            u64 ttbr;
+            u8 asid;
+        protected:
+            virtual Result Operate(PageLinkedList *page_list, KProcessAddress virt_addr, size_t num_pages, KPhysicalAddress phys_addr, bool is_pa_valid, const KPageProperties properties, OperationType operation, bool reuse_ll) override;
+            virtual Result Operate(PageLinkedList *page_list, KProcessAddress virt_addr, size_t num_pages, const KPageGroup &page_group, const KPageProperties properties, OperationType operation, bool reuse_ll) override;
+            virtual void   FinalizeUpdate(PageLinkedList *page_list) override;
+
+            KPageTableManager &GetPageTableManager() const { return *this->manager; }
+        private:
+            constexpr PageTableEntry GetEntryTemplate(const KPageProperties properties) const {
+                /* Set basic attributes. */
+                PageTableEntry entry;
+                entry.SetPrivilegedExecuteNever(true);
+                entry.SetAccessFlag(PageTableEntry::AccessFlag_Accessed);
+                entry.SetShareable(PageTableEntry::Shareable_InnerShareable);
+
+                if (!this->IsKernel()) {
+                    entry.SetGlobal(false);
+                }
+
+                /* Set page attribute. */
+                if (properties.io) {
+                    MESOSPHERE_ABORT_UNLESS(!properties.uncached);
+                    MESOSPHERE_ABORT_UNLESS((properties.perm & (KMemoryPermission_KernelExecute | KMemoryPermission_UserExecute)) == 0);
+
+                    entry.SetPageAttribute(PageTableEntry::PageAttribute_Device_nGnRnE)
+                         .SetUserExecuteNever(true);
+                } else if (properties.uncached) {
+                    MESOSPHERE_ABORT_UNLESS((properties.perm & (KMemoryPermission_KernelExecute | KMemoryPermission_UserExecute)) == 0);
+
+                    entry.SetPageAttribute(PageTableEntry::PageAttribute_NormalMemoryNotCacheable);
+                } else {
+                    entry.SetPageAttribute(PageTableEntry::PageAttribute_NormalMemory);
+                }
+
+                /* Set user execute never bit. */
+                if (properties.perm != KMemoryPermission_UserReadExecute) {
+                    MESOSPHERE_ABORT_UNLESS((properties.perm & (KMemoryPermission_KernelExecute | KMemoryPermission_UserExecute)) == 0);
+                    entry.SetUserExecuteNever(true);
+                }
+
+                /* Set can be contiguous. */
+                entry.SetContiguousAllowed(!properties.non_contiguous);
+
+                /* Set AP[1] based on perm. */
+                switch (properties.perm & KMemoryPermission_UserReadWrite) {
+                    case KMemoryPermission_UserReadWrite:
+                    case KMemoryPermission_UserRead:
+                        entry.SetUserAccessible(true);
+                        break;
+                    case KMemoryPermission_KernelReadWrite:
+                    case KMemoryPermission_KernelRead:
+                        entry.SetUserAccessible(false);
+                        break;
+                    MESOSPHERE_UNREACHABLE_DEFAULT_CASE();
+                }
+
+                /* Set AP[2] based on perm. */
+                switch (properties.perm & KMemoryPermission_UserReadWrite) {
+                    case KMemoryPermission_UserReadWrite:
+                    case KMemoryPermission_KernelReadWrite:
+                        entry.SetReadOnly(false);
+                        break;
+                    case KMemoryPermission_KernelRead:
+                    case KMemoryPermission_UserRead:
+                        entry.SetReadOnly(true);
+                        break;
+                    MESOSPHERE_UNREACHABLE_DEFAULT_CASE();
+                }
+
+                return entry;
+            }
+        public:
+            constexpr KPageTable() : KPageTableBase(), manager(), ttbr(), asid() { /* ... */ }
+
+            static NOINLINE void Initialize(s32 core_id);
+
+            ALWAYS_INLINE void Activate(u32 proc_id) {
+                cpu::DataSynchronizationBarrier();
+                cpu::SwitchProcess(this->ttbr, proc_id);
+            }
+
+            NOINLINE Result InitializeForKernel(void *table, KVirtualAddress start, KVirtualAddress end);
+            NOINLINE Result InitializeForProcess(u32 id, ams::svc::CreateProcessFlag as_type, bool enable_aslr, bool from_back, KMemoryManager::Pool pool, KProcessAddress code_address, size_t code_size, KMemoryBlockSlabManager *mem_block_slab_manager, KBlockInfoManager *block_info_manager, KPageTableManager *pt_manager);
+            Result Finalize();
+        private:
+            Result Map(KProcessAddress virt_addr, KPhysicalAddress phys_addr, size_t num_pages, PageTableEntry entry_template, PageLinkedList *page_list, bool reuse_ll);
+            Result Unmap(KProcessAddress virt_addr, size_t num_pages, PageLinkedList *page_list, bool force, bool reuse_ll);
+
+            Result Map(KProcessAddress virt_addr, KPhysicalAddress phys_addr, size_t num_pages, PageTableEntry entry_template, size_t page_size, PageLinkedList *page_list, bool reuse_ll) {
+                switch (page_size) {
+                    case L1BlockSize:
+#ifdef ATMOSPHERE_BOARD_NINTENDO_NX
+                    case L2TegraSmmuBlockSize:
+#endif
+                    case L2BlockSize:
+                    case L3BlockSize:
+                        break;
+                    case L2ContiguousBlockSize:
+                    case L3ContiguousBlockSize:
+                        entry_template.SetContiguous(true);
+                        break;
+                    MESOSPHERE_UNREACHABLE_DEFAULT_CASE();
+                }
+                return this->Map(virt_addr, phys_addr, num_pages, entry_template, page_list, reuse_ll);
+            }
+
+            Result MapContiguous(KProcessAddress virt_addr, KPhysicalAddress phys_addr, size_t num_pages, PageTableEntry entry_template, PageLinkedList *page_list, bool reuse_ll);
+            Result MapGroup(KProcessAddress virt_addr, const KPageGroup &pg, size_t num_pages, PageTableEntry entry_template, PageLinkedList *page_list, bool reuse_ll);
+
+            bool MergePages(KProcessAddress virt_addr, PageLinkedList *page_list);
+
+            ALWAYS_INLINE Result SeparatePagesImpl(KProcessAddress virt_addr, size_t block_size, PageLinkedList *page_list, bool reuse_ll);
+            Result SeparatePages(KProcessAddress virt_addr, size_t block_size, PageLinkedList *page_list, bool reuse_ll);
+
+            Result ChangePermissions(KProcessAddress virt_addr, size_t num_pages, PageTableEntry entry_template, bool refresh_mapping, PageLinkedList *page_list, bool reuse_ll);
+
+            static void PteDataSynchronizationBarrier() {
+                cpu::DataSynchronizationBarrierInnerShareable();
+            }
+
+            static void ClearPageTable(KVirtualAddress table) {
+                cpu::ClearPageToZero(GetVoidPointer(table));
+            }
+
+            void OnTableUpdated() const {
+                cpu::InvalidateTlbByAsid(this->asid);
+            }
+
+            void OnKernelTableUpdated() const {
+                cpu::InvalidateEntireTlbDataOnly();
+            }
+
+            void OnKernelTableSinglePageUpdated(KProcessAddress virt_addr) const {
+                cpu::InvalidateTlbByVaDataOnly(virt_addr);
+            }
+
+            void NoteUpdated() const {
+                cpu::DataSynchronizationBarrier();
+
+                if (this->IsKernel()) {
+                    this->OnKernelTableUpdated();
+                } else {
+                    this->OnTableUpdated();
+                }
+            }
+
+            void NoteSingleKernelPageUpdated(KProcessAddress virt_addr) const {
+                MESOSPHERE_ASSERT(this->IsKernel());
+
+                cpu::DataSynchronizationBarrier();
+                this->OnKernelTableSinglePageUpdated(virt_addr);
+            }
+
+            KVirtualAddress AllocatePageTable(PageLinkedList *page_list, bool reuse_ll) const {
+                KVirtualAddress table = this->GetPageTableManager().Allocate();
+
+                if (table == Null<KVirtualAddress>) {
+                    if (reuse_ll && page_list->Peek()) {
+                        table = KVirtualAddress(reinterpret_cast<uintptr_t>(page_list->Pop()));
+                    } else {
+                        return Null<KVirtualAddress>;
+                    }
+                }
+
+                ClearPageTable(table);
+
+                MESOSPHERE_ASSERT(this->GetPageTableManager().GetRefCount(table) == 0);
+
+                return table;
+            }
+
+            void FreePageTable(PageLinkedList *page_list, KVirtualAddress table) const {
+                MESOSPHERE_ASSERT(this->GetPageTableManager().IsInPageTableHeap(table));
+                MESOSPHERE_ASSERT(this->GetPageTableManager().GetRefCount(table) == 0);
+                page_list->Push(table);
+            }
+    };
+
+}

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table_entry.hpp

@@ -0,0 +1,299 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_select_cpu.hpp>
+#include <mesosphere/kern_k_typed_address.hpp>
+
+namespace ams::kern::arch::arm64 {
+
+    constexpr size_t L1BlockSize           = 1_GB;
+    constexpr size_t L1ContiguousBlockSize = 0x10 * L1BlockSize;
+    constexpr size_t L2BlockSize           = 2_MB;
+    constexpr size_t L2ContiguousBlockSize = 0x10 * L2BlockSize;
+    constexpr size_t L3BlockSize           = PageSize;
+    constexpr size_t L3ContiguousBlockSize = 0x10 * L3BlockSize;
+
+    class PageTableEntry {
+        public:
+            struct InvalidTag{};
+
+            enum ExtensionTag : u64 {
+                ExtensionTag_IsValidBit  = (1ul << 56),
+                ExtensionTag_IsValid     = (ExtensionTag_IsValidBit | (1ul << 0)),
+                ExtensionTag_IsBlockMask = (ExtensionTag_IsValidBit | (1ul << 1)),
+            };
+
+            enum Permission : u64 {
+                Permission_KernelRWX = ((0ul << 53) | (1ul << 54) | (0ul << 6)),
+                Permission_KernelRX  = ((0ul << 53) | (1ul << 54) | (2ul << 6)),
+                Permission_KernelR   = ((1ul << 53) | (1ul << 54) | (2ul << 6)),
+                Permission_KernelRW  = ((1ul << 53) | (1ul << 54) | (0ul << 6)),
+
+                Permission_UserRX    = ((1ul << 53) | (0ul << 54) | (3ul << 6)),
+                Permission_UserR     = ((1ul << 53) | (1ul << 54) | (3ul << 6)),
+                Permission_UserRW    = ((1ul << 53) | (1ul << 54) | (1ul << 6)),
+            };
+
+            enum Shareable : u64 {
+                Shareable_NonShareable   = (0 << 8),
+                Shareable_OuterShareable = (2 << 8),
+                Shareable_InnerShareable = (3 << 8),
+            };
+
+            /* Official attributes are: */
+            /*    0x00, 0x04, 0xFF, 0x44. 4-7 are unused. */
+            enum PageAttribute : u64 {
+                PageAttribute_Device_nGnRnE            = (0 << 2),
+                PageAttribute_Device_nGnRE             = (1 << 2),
+                PageAttribute_NormalMemory             = (2 << 2),
+                PageAttribute_NormalMemoryNotCacheable = (3 << 2),
+            };
+
+            enum AccessFlag : u64 {
+                AccessFlag_NotAccessed = (0 << 10),
+                AccessFlag_Accessed    = (1 << 10),
+            };
+
+            enum Type : u64 {
+                Type_None    = 0x0,
+                Type_L1Block = 0x1,
+                Type_L1Table = 0x3,
+                Type_L2Block = 0x1,
+                Type_L2Table = 0x3,
+                Type_L3Block = 0x3,
+            };
+
+            enum ContigType : u64 {
+                ContigType_NotContiguous = (0x0ul << 52),
+                ContigType_Contiguous    = (0x1ul << 52),
+            };
+        protected:
+            u64 attributes;
+        public:
+            /* Take in a raw attribute. */
+            constexpr ALWAYS_INLINE PageTableEntry() : attributes() { /* ... */ }
+            constexpr ALWAYS_INLINE PageTableEntry(u64 attr) : attributes(attr) { /* ... */ }
+
+            constexpr ALWAYS_INLINE PageTableEntry(InvalidTag) : attributes(0) { /* ... */ }
+
+            /* Extend a previous attribute. */
+            constexpr ALWAYS_INLINE PageTableEntry(const PageTableEntry &rhs, u64 new_attr) : attributes(rhs.attributes | new_attr) { /* ... */ }
+
+            /* Construct a new attribute. */
+            constexpr ALWAYS_INLINE PageTableEntry(Permission perm, PageAttribute p_a, Shareable share)
+                : attributes(static_cast<u64>(perm) | static_cast<u64>(AccessFlag_Accessed) | static_cast<u64>(p_a) | static_cast<u64>(share) | static_cast<u64>(ExtensionTag_IsValid))
+            {
+                /* ... */
+            }
+        protected:
+            constexpr ALWAYS_INLINE u64 GetBits(size_t offset, size_t count) const {
+                return (this->attributes >> offset) & ((1ul << count) - 1);
+            }
+
+            constexpr ALWAYS_INLINE u64 SelectBits(size_t offset, size_t count) const {
+                return this->attributes & (((1ul << count) - 1) << offset);
+            }
+
+            constexpr ALWAYS_INLINE void SetBits(size_t offset, size_t count, u64 value) {
+                const u64 mask = ((1ul << count) - 1) << offset;
+                this->attributes &= ~mask;
+                this->attributes |= (value & (mask >> offset)) << offset;
+            }
+
+            constexpr ALWAYS_INLINE void SetBitsDirect(size_t offset, size_t count, u64 value) {
+                const u64 mask = ((1ul << count) - 1) << offset;
+                this->attributes &= ~mask;
+                this->attributes |= (value & mask);
+            }
+
+            constexpr ALWAYS_INLINE void SetBit(size_t offset, bool enabled) {
+                const u64 mask = 1ul << offset;
+                if (enabled) {
+                    this->attributes |= mask;
+                } else {
+                    this->attributes &= ~mask;
+                }
+            }
+        public:
+            constexpr ALWAYS_INLINE bool IsContiguousAllowed()       const { return this->GetBits(55, 1) != 0; }
+            constexpr ALWAYS_INLINE bool IsUserExecuteNever()        const { return this->GetBits(54, 1) != 0; }
+            constexpr ALWAYS_INLINE bool IsPrivilegedExecuteNever()  const { return this->GetBits(53, 1) != 0; }
+            constexpr ALWAYS_INLINE bool IsContiguous()              const { return this->GetBits(52, 1) != 0; }
+            constexpr ALWAYS_INLINE bool IsGlobal()                  const { return this->GetBits(11, 1) == 0; }
+            constexpr ALWAYS_INLINE AccessFlag GetAccessFlag()       const { return static_cast<AccessFlag>(this->GetBits(10, 1)); }
+            constexpr ALWAYS_INLINE Shareable GetShareable()         const { return static_cast<Shareable>(this->GetBits(8, 2)); }
+            constexpr ALWAYS_INLINE PageAttribute GetPageAttribute() const { return static_cast<PageAttribute>(this->GetBits(2, 3)); }
+            constexpr ALWAYS_INLINE bool IsReadOnly()                const { return this->GetBits(7, 1) != 0; }
+            constexpr ALWAYS_INLINE bool IsUserAccessible()          const { return this->GetBits(6, 1) != 0; }
+            constexpr ALWAYS_INLINE bool IsNonSecure()               const { return this->GetBits(5, 1) != 0; }
+            constexpr ALWAYS_INLINE bool IsBlock()                   const { return (this->attributes & ExtensionTag_IsBlockMask) == ExtensionTag_IsValidBit; }
+            constexpr ALWAYS_INLINE bool IsTable()                   const { return this->GetBits(0, 2) == 0x3; }
+            constexpr ALWAYS_INLINE bool IsEmpty()                   const { return this->GetBits(0, 2) == 0x0; }
+
+            constexpr ALWAYS_INLINE decltype(auto) SetContiguousAllowed(bool en)      { this->SetBit(55, !en); return *this; }
+            constexpr ALWAYS_INLINE decltype(auto) SetUserExecuteNever(bool en)       { this->SetBit(54, en); return *this; }
+            constexpr ALWAYS_INLINE decltype(auto) SetPrivilegedExecuteNever(bool en) { this->SetBit(53, en); return *this; }
+            constexpr ALWAYS_INLINE decltype(auto) SetContiguous(bool en)             { this->SetBit(52, en); return *this; }
+            constexpr ALWAYS_INLINE decltype(auto) SetGlobal(bool en)                 { this->SetBit(11, !en); return *this; }
+            constexpr ALWAYS_INLINE decltype(auto) SetAccessFlag(AccessFlag f)        { this->SetBitsDirect(10, 1, f); return *this; }
+            constexpr ALWAYS_INLINE decltype(auto) SetShareable(Shareable s)          { this->SetBitsDirect(8, 2, s); return *this; }
+            constexpr ALWAYS_INLINE decltype(auto) SetReadOnly(bool en)               { this->SetBit(7, en); return *this; }
+            constexpr ALWAYS_INLINE decltype(auto) SetUserAccessible(bool en)         { this->SetBit(6, en); return *this; }
+            constexpr ALWAYS_INLINE decltype(auto) SetPageAttribute(PageAttribute a)  { this->SetBitsDirect(2, 3, a); return *this; }
+
+            constexpr ALWAYS_INLINE u64 GetEntryTemplate() const {
+                constexpr u64 Mask = (0xFFF0000000000FFFul & ~u64(0x3ul | (0x1ul << 52)));
+                return this->attributes & Mask;
+            }
+
+            constexpr ALWAYS_INLINE bool Is(u64 attr) const {
+                return this->attributes == attr;
+            }
+
+            constexpr ALWAYS_INLINE u64 GetRawAttributesUnsafeForSwap() const {
+                return this->attributes;
+            }
+
+        protected:
+            constexpr ALWAYS_INLINE u64 GetRawAttributes() const {
+                return this->attributes;
+            }
+    };
+
+    static_assert(sizeof(PageTableEntry) == sizeof(u64));
+
+    constexpr inline PageTableEntry InvalidPageTableEntry = PageTableEntry(PageTableEntry::InvalidTag{});
+
+    constexpr inline size_t MaxPageTableEntries = PageSize / sizeof(PageTableEntry);
+
+    class L1PageTableEntry : public PageTableEntry {
+        public:
+            constexpr ALWAYS_INLINE L1PageTableEntry(InvalidTag) : PageTableEntry(InvalidTag{}) { /* ... */ }
+
+            constexpr ALWAYS_INLINE L1PageTableEntry(KPhysicalAddress phys_addr, bool pxn)
+                : PageTableEntry((0x3ul << 60) | (static_cast<u64>(pxn) << 59) | GetInteger(phys_addr) | 0x3)
+            {
+                /* ... */
+            }
+
+            constexpr ALWAYS_INLINE L1PageTableEntry(KPhysicalAddress phys_addr, bool is_kernel, bool pxn)
+                : PageTableEntry(((is_kernel ? 0x3ul : 0) << 60) | (static_cast<u64>(pxn) << 59) | GetInteger(phys_addr) | 0x3)
+            {
+                /* ... */
+            }
+
+            constexpr ALWAYS_INLINE L1PageTableEntry(KPhysicalAddress phys_addr, const PageTableEntry &attr, bool contig)
+                : PageTableEntry(attr, (static_cast<u64>(contig) << 52) | GetInteger(phys_addr) | PageTableEntry::ExtensionTag_IsValid)
+            {
+                /* ... */
+            }
+
+            constexpr ALWAYS_INLINE KPhysicalAddress GetBlock() const {
+                return this->SelectBits(30, 18);
+            }
+
+            constexpr ALWAYS_INLINE KPhysicalAddress GetTable() const {
+                return this->SelectBits(12, 36);
+            }
+
+            constexpr ALWAYS_INLINE bool GetTable(KPhysicalAddress &out) const {
+                if (this->IsTable()) {
+                    out = this->GetTable();
+                    return true;
+                } else {
+                    return false;
+                }
+            }
+
+            constexpr ALWAYS_INLINE bool IsCompatibleWithAttribute(const PageTableEntry &rhs, bool contig) const {
+                /* Check whether this has the same permission/etc as the desired attributes. */
+                return L1PageTableEntry(this->GetBlock(), rhs, contig).GetRawAttributes() == this->GetRawAttributes();
+            }
+    };
+
+    class L2PageTableEntry : public PageTableEntry {
+        public:
+            constexpr ALWAYS_INLINE L2PageTableEntry(InvalidTag) : PageTableEntry(InvalidTag{}) { /* ... */ }
+
+            constexpr ALWAYS_INLINE L2PageTableEntry(KPhysicalAddress phys_addr, bool pxn)
+                : PageTableEntry((0x3ul << 60) | (static_cast<u64>(pxn) << 59) | GetInteger(phys_addr) | 0x3)
+            {
+                /* ... */
+            }
+
+            constexpr ALWAYS_INLINE L2PageTableEntry(KPhysicalAddress phys_addr, bool is_kernel, bool pxn)
+                : PageTableEntry(((is_kernel ? 0x3ul : 0) << 60) | (static_cast<u64>(pxn) << 59) | GetInteger(phys_addr) | 0x3)
+            {
+                /* ... */
+            }
+
+            constexpr ALWAYS_INLINE L2PageTableEntry(KPhysicalAddress phys_addr, const PageTableEntry &attr, bool contig)
+                : PageTableEntry(attr, (static_cast<u64>(contig) << 52) | GetInteger(phys_addr) | PageTableEntry::ExtensionTag_IsValid)
+            {
+                /* ... */
+            }
+
+            constexpr ALWAYS_INLINE KPhysicalAddress GetBlock() const {
+                return this->SelectBits(21, 27);
+            }
+
+            constexpr ALWAYS_INLINE KPhysicalAddress GetTable() const {
+                return this->SelectBits(12, 36);
+            }
+
+            constexpr ALWAYS_INLINE bool GetTable(KPhysicalAddress &out) const {
+                if (this->IsTable()) {
+                    out = this->GetTable();
+                    return true;
+                } else {
+                    return false;
+                }
+            }
+
+            constexpr ALWAYS_INLINE bool IsCompatibleWithAttribute(const PageTableEntry &rhs, bool contig) const {
+                /* Check whether this has the same permission/etc as the desired attributes. */
+                return L2PageTableEntry(this->GetBlock(), rhs, contig).GetRawAttributes() == this->GetRawAttributes();
+            }
+    };
+
+    class L3PageTableEntry : public PageTableEntry {
+        public:
+            constexpr ALWAYS_INLINE L3PageTableEntry(InvalidTag) : PageTableEntry(InvalidTag{}) { /* ... */ }
+
+            constexpr ALWAYS_INLINE L3PageTableEntry(KPhysicalAddress phys_addr, const PageTableEntry &attr, bool contig)
+                : PageTableEntry(attr, (static_cast<u64>(contig) << 52) | GetInteger(phys_addr) | 0x2 | PageTableEntry::ExtensionTag_IsValid)
+            {
+                /* ... */
+            }
+
+            constexpr ALWAYS_INLINE bool IsBlock() const { return (GetRawAttributes() & ExtensionTag_IsBlockMask) == ExtensionTag_IsBlockMask; }
+
+            constexpr ALWAYS_INLINE KPhysicalAddress GetBlock() const {
+                return this->SelectBits(12, 36);
+            }
+
+            constexpr ALWAYS_INLINE bool IsCompatibleWithAttribute(const PageTableEntry &rhs, bool contig) const {
+                /* Check whether this has the same permission/etc as the desired attributes. */
+                return L3PageTableEntry(this->GetBlock(), rhs, contig).GetRawAttributes() == this->GetRawAttributes();
+            }
+    };
+
+    constexpr inline L1PageTableEntry InvalidL1PageTableEntry = L1PageTableEntry(PageTableEntry::InvalidTag{});
+    constexpr inline L2PageTableEntry InvalidL2PageTableEntry = L2PageTableEntry(PageTableEntry::InvalidTag{});
+    constexpr inline L3PageTableEntry InvalidL3PageTableEntry = L3PageTableEntry(PageTableEntry::InvalidTag{});
+
+}

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table_impl.hpp

@@ -0,0 +1,115 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_select_cpu.hpp>
+#include <mesosphere/kern_k_typed_address.hpp>
+#include <mesosphere/kern_k_memory_layout.hpp>
+#include <mesosphere/arch/arm64/kern_k_page_table_entry.hpp>
+
+namespace ams::kern::arch::arm64 {
+
+    class KPageTableImpl {
+        NON_COPYABLE(KPageTableImpl);
+        NON_MOVEABLE(KPageTableImpl);
+        public:
+            struct TraversalEntry {
+                KPhysicalAddress phys_addr;
+                size_t block_size;
+            };
+
+            struct TraversalContext {
+                const L1PageTableEntry *l1_entry;
+                const L2PageTableEntry *l2_entry;
+                const L3PageTableEntry *l3_entry;
+            };
+        private:
+            static constexpr size_t PageBits  = __builtin_ctzll(PageSize);
+            static constexpr size_t NumLevels = 3;
+            static constexpr size_t LevelBits = 9;
+            static_assert(NumLevels > 0);
+
+            template<size_t Offset, size_t Count>
+            static constexpr ALWAYS_INLINE u64 GetBits(u64 value) {
+                return (value >> Offset) & ((1ul << Count) - 1);
+            }
+
+            template<size_t Offset, size_t Count>
+            constexpr ALWAYS_INLINE u64 SelectBits(u64 value) {
+                return value & (((1ul << Count) - 1) << Offset);
+            }
+
+            static constexpr ALWAYS_INLINE uintptr_t GetL0Index(KProcessAddress addr) { return GetBits<PageBits + LevelBits * (NumLevels - 0), LevelBits>(GetInteger(addr)); }
+            static constexpr ALWAYS_INLINE uintptr_t GetL1Index(KProcessAddress addr) { return GetBits<PageBits + LevelBits * (NumLevels - 1), LevelBits>(GetInteger(addr)); }
+            static constexpr ALWAYS_INLINE uintptr_t GetL2Index(KProcessAddress addr) { return GetBits<PageBits + LevelBits * (NumLevels - 2), LevelBits>(GetInteger(addr)); }
+            static constexpr ALWAYS_INLINE uintptr_t GetL3Index(KProcessAddress addr) { return GetBits<PageBits + LevelBits * (NumLevels - 3), LevelBits>(GetInteger(addr)); }
+
+            static constexpr ALWAYS_INLINE uintptr_t GetL1Offset(KProcessAddress addr) { return GetBits<0, PageBits + LevelBits * (NumLevels - 1)>(GetInteger(addr)); }
+            static constexpr ALWAYS_INLINE uintptr_t GetL2Offset(KProcessAddress addr) { return GetBits<0, PageBits + LevelBits * (NumLevels - 2)>(GetInteger(addr)); }
+            static constexpr ALWAYS_INLINE uintptr_t GetL3Offset(KProcessAddress addr) { return GetBits<0, PageBits + LevelBits * (NumLevels - 3)>(GetInteger(addr)); }
+            static constexpr ALWAYS_INLINE uintptr_t GetContiguousL1Offset(KProcessAddress addr) { return GetBits<0, PageBits + LevelBits * (NumLevels - 1) + 4>(GetInteger(addr)); }
+            static constexpr ALWAYS_INLINE uintptr_t GetContiguousL2Offset(KProcessAddress addr) { return GetBits<0, PageBits + LevelBits * (NumLevels - 2) + 4>(GetInteger(addr)); }
+            static constexpr ALWAYS_INLINE uintptr_t GetContiguousL3Offset(KProcessAddress addr) { return GetBits<0, PageBits + LevelBits * (NumLevels - 3) + 4>(GetInteger(addr)); }
+
+            static ALWAYS_INLINE KVirtualAddress GetPageTableVirtualAddress(KPhysicalAddress addr) {
+                return KMemoryLayout::GetLinearVirtualAddress(addr);
+            }
+
+            ALWAYS_INLINE bool ExtractL1Entry(TraversalEntry *out_entry, TraversalContext *out_context, const L1PageTableEntry *l1_entry, KProcessAddress virt_addr) const;
+            ALWAYS_INLINE bool ExtractL2Entry(TraversalEntry *out_entry, TraversalContext *out_context, const L2PageTableEntry *l2_entry, KProcessAddress virt_addr) const;
+            ALWAYS_INLINE bool ExtractL3Entry(TraversalEntry *out_entry, TraversalContext *out_context, const L3PageTableEntry *l3_entry, KProcessAddress virt_addr) const;
+        private:
+            L1PageTableEntry *table;
+            bool is_kernel;
+            u32  num_entries;
+        public:
+            ALWAYS_INLINE KVirtualAddress GetTableEntry(KVirtualAddress table, size_t index) const {
+                return table + index * sizeof(PageTableEntry);
+            }
+
+            ALWAYS_INLINE L1PageTableEntry *GetL1Entry(KProcessAddress address) const {
+                return GetPointer<L1PageTableEntry>(GetTableEntry(KVirtualAddress(this->table), GetL1Index(address) & (this->num_entries - 1)));
+            }
+
+            ALWAYS_INLINE L2PageTableEntry *GetL2EntryFromTable(KVirtualAddress table, KProcessAddress address) const {
+                return GetPointer<L2PageTableEntry>(GetTableEntry(table, GetL2Index(address)));
+            }
+
+            ALWAYS_INLINE L2PageTableEntry *GetL2Entry(const L1PageTableEntry *entry, KProcessAddress address) const {
+                return GetL2EntryFromTable(KMemoryLayout::GetLinearVirtualAddress(entry->GetTable()), address);
+            }
+
+            ALWAYS_INLINE L3PageTableEntry *GetL3EntryFromTable(KVirtualAddress table, KProcessAddress address) const {
+                return GetPointer<L3PageTableEntry>(GetTableEntry(table, GetL3Index(address)));
+            }
+
+            ALWAYS_INLINE L3PageTableEntry *GetL3Entry(const L2PageTableEntry *entry, KProcessAddress address) const {
+                return GetL3EntryFromTable(KMemoryLayout::GetLinearVirtualAddress(entry->GetTable()), address);
+            }
+        public:
+            constexpr KPageTableImpl() : table(), is_kernel(), num_entries() { /* ... */ }
+
+            NOINLINE void InitializeForKernel(void *tb, KVirtualAddress start, KVirtualAddress end);
+            NOINLINE void InitializeForProcess(void *tb, KVirtualAddress start, KVirtualAddress end);
+            L1PageTableEntry *Finalize();
+
+            bool BeginTraversal(TraversalEntry *out_entry, TraversalContext *out_context, KProcessAddress address) const;
+            bool ContinueTraversal(TraversalEntry *out_entry, TraversalContext *context) const;
+
+            bool GetPhysicalAddress(KPhysicalAddress *out, KProcessAddress virt_addr) const;
+    };
+
+}

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_process_page_table.hpp

@@ -0,0 +1,122 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/arch/arm64/kern_k_page_table.hpp>
+
+namespace ams::kern::arch::arm64 {
+
+    class KProcessPageTable {
+        private:
+            KPageTable page_table;
+        public:
+            constexpr KProcessPageTable() : page_table() { /* ... */ }
+
+            void Activate(u64 id) {
+                /* Activate the page table with the specified contextidr. */
+                this->page_table.Activate(id);
+            }
+
+            Result Initialize(u32 id, ams::svc::CreateProcessFlag as_type, bool enable_aslr, bool from_back, KMemoryManager::Pool pool, KProcessAddress code_address, size_t code_size, KMemoryBlockSlabManager *mem_block_slab_manager, KBlockInfoManager *block_info_manager, KPageTableManager *pt_manager) {
+                return this->page_table.InitializeForProcess(id, as_type, enable_aslr, from_back, pool, code_address, code_size, mem_block_slab_manager, block_info_manager, pt_manager);
+            }
+
+            void Finalize() { this->page_table.Finalize(); }
+
+            Result SetMemoryPermission(KProcessAddress addr, size_t size, ams::svc::MemoryPermission perm) {
+                return this->page_table.SetMemoryPermission(addr, size, perm);
+            }
+
+            Result SetProcessMemoryPermission(KProcessAddress addr, size_t size, ams::svc::MemoryPermission perm) {
+                return this->page_table.SetProcessMemoryPermission(addr, size, perm);
+            }
+
+            Result SetHeapSize(KProcessAddress *out, size_t size) {
+                return this->page_table.SetHeapSize(out, size);
+            }
+
+            Result SetMaxHeapSize(size_t size) {
+                return this->page_table.SetMaxHeapSize(size);
+            }
+
+            Result QueryInfo(KMemoryInfo *out_info, ams::svc::PageInfo *out_page_info, KProcessAddress addr) const {
+                return this->page_table.QueryInfo(out_info, out_page_info, addr);
+            }
+
+            Result MapIo(KPhysicalAddress phys_addr, size_t size, KMemoryPermission perm) {
+                return this->page_table.MapIo(phys_addr, size, perm);
+            }
+
+            Result MapStatic(KPhysicalAddress phys_addr, size_t size, KMemoryPermission perm) {
+                return this->page_table.MapStatic(phys_addr, size, perm);
+            }
+
+            Result MapRegion(KMemoryRegionType region_type, KMemoryPermission perm) {
+                return this->page_table.MapRegion(region_type, perm);
+            }
+
+            Result MapPageGroup(KProcessAddress addr, const KPageGroup &pg, KMemoryState state, KMemoryPermission perm) {
+                return this->page_table.MapPageGroup(addr, pg, state, perm);
+            }
+
+            Result MapPages(KProcessAddress *out_addr, size_t num_pages, size_t alignment, KPhysicalAddress phys_addr, KMemoryState state, KMemoryPermission perm) {
+                return this->page_table.MapPages(out_addr, num_pages, alignment, phys_addr, state, perm);
+            }
+
+            Result MapPages(KProcessAddress *out_addr, size_t num_pages, KMemoryState state, KMemoryPermission perm) {
+                return this->page_table.MapPages(out_addr, num_pages, state, perm);
+            }
+
+            Result UnmapPages(KProcessAddress addr, size_t num_pages, KMemoryState state) {
+                return this->page_table.UnmapPages(addr, num_pages, state);
+            }
+
+            Result MakeAndOpenPageGroup(KPageGroup *out, KProcessAddress address, size_t num_pages, u32 state_mask, u32 state, u32 perm_mask, u32 perm, u32 attr_mask, u32 attr) {
+                return this->page_table.MakeAndOpenPageGroup(out, address, num_pages, state_mask, state, perm_mask, perm, attr_mask, attr);
+            }
+
+            bool GetPhysicalAddress(KPhysicalAddress *out, KProcessAddress address) const {
+                return this->page_table.GetPhysicalAddress(out, address);
+            }
+
+            bool Contains(KProcessAddress addr, size_t size) const { return this->page_table.Contains(addr, size); }
+            bool CanContain(KProcessAddress addr, size_t size, KMemoryState state) const { return this->page_table.CanContain(addr, size, state); }
+
+            KProcessAddress GetAddressSpaceStart()    const { return this->page_table.GetAddressSpaceStart(); }
+            KProcessAddress GetHeapRegionStart()      const { return this->page_table.GetHeapRegionStart(); }
+            KProcessAddress GetAliasRegionStart()     const { return this->page_table.GetAliasRegionStart(); }
+            KProcessAddress GetStackRegionStart()     const { return this->page_table.GetStackRegionStart(); }
+            KProcessAddress GetKernelMapRegionStart() const { return this->page_table.GetKernelMapRegionStart(); }
+            KProcessAddress GetAliasCodeRegionStart() const { return this->page_table.GetAliasCodeRegionStart(); }
+
+            size_t GetAddressSpaceSize()    const { return this->page_table.GetAddressSpaceSize(); }
+            size_t GetHeapRegionSize()      const { return this->page_table.GetHeapRegionSize(); }
+            size_t GetAliasRegionSize()     const { return this->page_table.GetAliasRegionSize(); }
+            size_t GetStackRegionSize()     const { return this->page_table.GetStackRegionSize(); }
+            size_t GetKernelMapRegionSize() const { return this->page_table.GetKernelMapRegionSize(); }
+            size_t GetAliasCodeRegionSize() const { return this->page_table.GetAliasCodeRegionSize(); }
+
+            KPhysicalAddress GetHeapPhysicalAddress(KVirtualAddress address) const {
+                /* TODO: Better way to convert address type? */
+                return this->page_table.GetHeapPhysicalAddress(address);
+            }
+
+            KBlockInfoManager *GetBlockInfoManager() {
+                return this->page_table.GetBlockInfoManager();
+            }
+    };
+
+}

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_spin_lock.hpp

@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <vapours.hpp>
+#include <mesosphere/kern_select_cpu.hpp>
+
+namespace ams::kern::arch::arm64 {
+
+    class KNotAlignedSpinLock {
+        private:
+            u32 packed_tickets;
+        public:
+            constexpr KNotAlignedSpinLock() : packed_tickets(0) { /* ... */ }
+
+            void Lock() {
+                u32 tmp0, tmp1;
+
+                __asm__ __volatile__(
+                    "    prfm   pstl1keep, %[packed_tickets]\n"
+                    "1:\n"
+                    "    ldaxr  %w[tmp0], %[packed_tickets]\n"
+                    "    add    %w[tmp0], %w[tmp0], #0x10000\n"
+                    "    stxr   %w[tmp1], %w[tmp0], %[packed_tickets]\n"
+                    "    cbnz   %w[tmp1], 1b\n"
+                    "    \n"
+                    "    and    %w[tmp1], %w[tmp0], #0xFFFF\n"
+                    "    cmp    %w[tmp1], %w[tmp0], lsr #16\n"
+                    "    b.eq   done"
+                    "    sevl\n"
+                    "2:\n"
+                    "    wfe\n"
+                    "    ldaxrh %w[tmp1], %[packed_tickets]\n"
+                    "    cmp    %w[tmp1], %w[tmp0], lsr #16\n"
+                    "    b.ne   2b\n"
+                    "done:\n"
+                    : [tmp0]"=&r"(tmp0), [tmp1]"=&r"(tmp1), [packed_tickets]"+Q"(this->packed_tickets)
+                    :
+                    : "cc", "memory"
+                );
+            }
+
+            void Unlock() {
+                const u32 value = this->packed_tickets + 1;
+                __asm__ __volatile__(
+                    "    stlrh %w[value], %[packed_tickets]\n"
+                    : [packed_tickets]"+Q"(this->packed_tickets)
+                    : [value]"r"(value)
+                    : "memory"
+                );
+            }
+    };
+    static_assert(sizeof(KNotAlignedSpinLock) == sizeof(u32));
+
+    class KAlignedSpinLock {
+        private:
+            alignas(cpu::DataCacheLineSize) u16 current_ticket;
+            alignas(cpu::DataCacheLineSize) u16 next_ticket;
+        public:
+            constexpr KAlignedSpinLock() : current_ticket(0), next_ticket(0) { /* ... */ }
+
+            void Lock() {
+                u32 tmp0, tmp1, got_lock;
+
+                __asm__ __volatile__(
+                    "    prfm   pstl1keep, %[next_ticket]\n"
+                    "1:\n"
+                    "    ldaxrh %w[tmp0], %[next_ticket]\n"
+                    "    add    %w[tmp1], %w[tmp0], #0x1\n"
+                    "    stxrh  %w[got_lock], %w[tmp1], %[next_ticket]\n"
+                    "    cbnz   %w[got_lock], 1b\n"
+                    "    \n"
+                    "    sevl\n"
+                    "2:\n"
+                    "    wfe\n"
+                    "    ldaxrh %w[tmp1], %[current_ticket]\n"
+                    "    cmp    %w[tmp1], %w[tmp0]\n"
+                    "    b.ne   2b\n"
+                    : [tmp0]"=&r"(tmp0), [tmp1]"=&r"(tmp1), [got_lock]"=&r"(got_lock), [next_ticket]"+Q"(this->next_ticket)
+                    : [current_ticket]"Q"(this->current_ticket)
+                    : "cc", "memory"
+                );
+            }
+
+            void Unlock() {
+                const u32 value = this->current_ticket + 1;
+                __asm__ __volatile__(
+                    "    stlrh %w[value], %[current_ticket]\n"
+                    : [current_ticket]"+Q"(this->current_ticket)
+                    : [value]"r"(value)
+                    : "memory"
+                );
+            }
+    };
+    static_assert(sizeof(KAlignedSpinLock) == 2 * cpu::DataCacheLineSize);
+
+    using KSpinLock = KAlignedSpinLock;
+
+}

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_supervisor_page_table.hpp

@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_select_cpu.hpp>
+#include <mesosphere/arch/arm64/kern_k_page_table.hpp>
+
+namespace ams::kern::arch::arm64 {
+
+    class KSupervisorPageTable {
+        private:
+            KPageTable page_table;
+            u64 ttbr0[cpu::NumCores];
+        public:
+            constexpr KSupervisorPageTable() : page_table(), ttbr0() { /* ... */ }
+
+            NOINLINE void Initialize(s32 core_id);
+
+            void Activate() {
+                /* Activate, using process id = 0xFFFFFFFF */
+                this->page_table.Activate(0xFFFFFFFF);
+            }
+
+            void ActivateForInit() {
+                this->Activate();
+
+                /* Invalidate entire TLB. */
+                cpu::InvalidateEntireTlb();
+            }
+
+            void Finalize(s32 core_id);
+
+            Result MapPages(KProcessAddress *out_addr, size_t num_pages, size_t alignment, KPhysicalAddress phys_addr, KProcessAddress region_start, size_t region_num_pages, KMemoryState state, KMemoryPermission perm) {
+                return this->page_table.MapPages(out_addr, num_pages, alignment, phys_addr, region_start, region_num_pages, state, perm);
+            }
+
+            Result UnmapPages(KProcessAddress address, size_t num_pages, KMemoryState state) {
+                return this->page_table.UnmapPages(address, num_pages, state);
+            }
+
+            Result MapPageGroup(KProcessAddress *out_addr, const KPageGroup &pg, KProcessAddress region_start, size_t region_num_pages, KMemoryState state, KMemoryPermission perm) {
+                return this->page_table.MapPageGroup(out_addr, pg, region_start, region_num_pages, state, perm);
+            }
+
+            Result UnmapPageGroup(KProcessAddress address, const KPageGroup &pg, KMemoryState state) {
+                return this->page_table.UnmapPageGroup(address, pg, state);
+            }
+
+            bool GetPhysicalAddress(KPhysicalAddress *out, KProcessAddress address) const {
+                return this->page_table.GetPhysicalAddress(out, address);
+            }
+    };
+
+}

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_thread_context.hpp

@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_k_typed_address.hpp>
+
+namespace ams::kern {
+
+    class KThread;
+
+}
+
+namespace ams::kern::arch::arm64 {
+
+    class KThreadContext {
+        public:
+            static constexpr size_t NumCalleeSavedRegisters = (29 - 19) + 1;
+            static constexpr size_t NumFpuRegisters = 32;
+        private:
+            union {
+                u64 registers[NumCalleeSavedRegisters];
+                struct {
+                    u64 x19;
+                    u64 x20;
+                    u64 x21;
+                    u64 x22;
+                    u64 x23;
+                    u64 x24;
+                    u64 x25;
+                    u64 x26;
+                    u64 x27;
+                    u64 x28;
+                    u64 x29;
+                };
+            } callee_saved;
+            u64 lr;
+            u64 sp;
+            u64 cpacr;
+            u64 fpcr;
+            u64 fpsr;
+            alignas(0x10) u128 fpu_registers[NumFpuRegisters];
+            bool locked;
+        private:
+            static void RestoreFpuRegisters64(const KThreadContext &);
+            static void RestoreFpuRegisters32(const KThreadContext &);
+        public:
+            constexpr explicit KThreadContext() : callee_saved(), lr(), sp(), cpacr(), fpcr(), fpsr(), fpu_registers(), locked() { /* ... */ }
+
+            Result Initialize(KVirtualAddress u_pc, KVirtualAddress k_sp, KVirtualAddress u_sp, uintptr_t arg, bool is_user, bool is_64_bit, bool is_main);
+            Result Finalize();
+
+            void SetArguments(uintptr_t arg0, uintptr_t arg1);
+
+            static void FpuContextSwitchHandler(KThread *thread);
+
+            /* TODO: More methods (especially FPU management) */
+    };
+
+}

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_userspace_memory_access.hpp

@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+
+namespace ams::kern::arch::arm64 {
+
+    void UserspaceAccessFunctionAreaBegin();
+
+    class UserspaceAccess {
+        public:
+            static bool CopyMemoryFromUser(void *dst, const void *src, size_t size);
+            static bool CopyMemoryFromUserAligned32Bit(void *dst, const void *src, size_t size);
+            static bool CopyMemoryFromUserAligned64Bit(void *dst, const void *src, size_t size);
+            static bool CopyMemoryFromUserSize32Bit(void *dst, const void *src);
+            static s32  CopyStringFromUser(void *dst, const void *src, size_t size);
+
+            static bool CopyMemoryToUser(void *dst, const void *src, size_t size);
+            static bool CopyMemoryToUserAligned32Bit(void *dst, const void *src, size_t size);
+            static bool CopyMemoryToUserAligned64Bit(void *dst, const void *src, size_t size);
+            static bool CopyMemoryToUserSize32Bit(void *dst, const void *src);
+            static s32  CopyStringToUser(void *dst, const void *src, size_t size);
+
+            static bool ClearMemory(void *dst, size_t size);
+            static bool ClearMemoryAligned32Bit(void *dst, size_t size);
+            static bool ClearMemoryAligned64Bit(void *dst, size_t size);
+            static bool ClearMemorySize32Bit(void *dst);
+
+            static bool StoreDataCache(uintptr_t start, uintptr_t end);
+            static bool FlushDataCache(uintptr_t start, uintptr_t end);
+            static bool InvalidateDataCache(uintptr_t start, uintptr_t end);
+            static bool InvalidateInstructionCache(uintptr_t start, uintptr_t end);
+
+            static bool ReadIoMemory32Bit(void *dst, const void *src, size_t size);
+            static bool ReadIoMemory16Bit(void *dst, const void *src, size_t size);
+            static bool ReadIoMemory8Bit(void *dst, const void *src, size_t size);
+            static bool WriteIoMemory32Bit(void *dst, const void *src, size_t size);
+            static bool WriteIoMemory16Bit(void *dst, const void *src, size_t size);
+            static bool WriteIoMemory8Bit(void *dst, const void *src, size_t size);
+    };
+
+
+    void UserspaceAccessFunctionAreaEnd();
+
+}

libraries/libmesosphere/include/mesosphere/board/nintendo/nx/kern_k_device_page_table.hpp

@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_k_page_group.hpp>
+#include <mesosphere/kern_k_memory_manager.hpp>
+#include <mesosphere/kern_select_page_table.hpp>
+
+namespace ams::kern::board::nintendo::nx {
+
+    using KDeviceVirtualAddress = u64;
+
+    class KDevicePageTable {
+        private:
+            static constexpr size_t TableCount = 4;
+        private:
+            KVirtualAddress tables[TableCount];
+            u8 table_asids[TableCount];
+            u64 attached_device;
+            u32 attached_value;
+            u32 detached_value;
+            u32 hs_attached_value;
+            u32 hs_detached_value;
+        private:
+            static ALWAYS_INLINE KVirtualAddress GetHeapVirtualAddress(KPhysicalAddress addr) {
+                return KPageTable::GetHeapVirtualAddress(addr);
+            }
+
+            static ALWAYS_INLINE KPhysicalAddress GetHeapPhysicalAddress(KVirtualAddress addr) {
+                return KPageTable::GetHeapPhysicalAddress(addr);
+            }
+
+            static ALWAYS_INLINE KVirtualAddress GetPageTableVirtualAddress(KPhysicalAddress addr) {
+                return KPageTable::GetPageTableVirtualAddress(addr);
+            }
+
+            static ALWAYS_INLINE KPhysicalAddress GetPageTablePhysicalAddress(KVirtualAddress addr) {
+                return KPageTable::GetPageTablePhysicalAddress(addr);
+            }
+        public:
+            constexpr KDevicePageTable() : tables(), table_asids(), attached_device(), attached_value(), detached_value(), hs_attached_value(), hs_detached_value() { /* ... */ }
+
+            static void Initialize();
+    };
+
+}

libraries/libmesosphere/include/mesosphere/board/nintendo/nx/kern_k_system_control.hpp

@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+
+namespace ams::kern::board::nintendo::nx {
+
+    class KSystemControl {
+        public:
+            class Init {
+                public:
+                    /* Initialization. */
+                    static size_t GetIntendedMemorySize();
+                    static KPhysicalAddress GetKernelPhysicalBaseAddress(uintptr_t base_address);
+                    static bool ShouldIncreaseThreadResourceLimit();
+                    static void CpuOn(u64 core_id, uintptr_t entrypoint, uintptr_t arg);
+                    static size_t GetApplicationPoolSize();
+                    static size_t GetAppletPoolSize();
+                    static size_t GetMinimumNonSecureSystemPoolSize();
+
+                    /* Randomness. */
+                    static void GenerateRandomBytes(void *dst, size_t size);
+                    static u64  GenerateRandomRange(u64 min, u64 max);
+            };
+        public:
+            /* Initialization. */
+            static NOINLINE void InitializePhase1();
+            static NOINLINE void InitializePhase2();
+            static NOINLINE u32 GetInitialProcessBinaryPool();
+
+            /* Randomness. */
+            static void GenerateRandomBytes(void *dst, size_t size);
+            static u64  GenerateRandomRange(u64 min, u64 max);
+
+            /* Privileged Access. */
+            static void ReadWriteRegisterPrivileged(u32 *out, ams::svc::PhysicalAddress address, u32 mask, u32 value);
+            static void ReadWriteRegister(u32 *out, ams::svc::PhysicalAddress address, u32 mask, u32 value);
+
+            static ALWAYS_INLINE u32 ReadRegisterPrivileged(ams::svc::PhysicalAddress address) {
+                u32 v;
+                ReadWriteRegisterPrivileged(std::addressof(v), address, 0x00000000u, 0);
+                return v;
+            }
+
+            static ALWAYS_INLINE void WriteRegisterPrivileged(ams::svc::PhysicalAddress address, u32 value) {
+                u32 v;
+                ReadWriteRegisterPrivileged(std::addressof(v), address, 0xFFFFFFFFu, value);
+            }
+
+            /* Power management. */
+            static void SleepSystem();
+            static NORETURN void StopSystem();
+
+            /* User access. */
+            static void CallSecureMonitorFromUser(ams::svc::lp64::SecureMonitorArguments *args);
+    };
+
+}

libraries/libmesosphere/include/mesosphere/init/kern_init_arguments_select.hpp

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_k_typed_address.hpp>
+
+#ifdef ATMOSPHERE_ARCH_ARM64
+    #include <mesosphere/arch/arm64/init/kern_k_init_arguments.hpp>
+#else
+    #error "Unknown architecture for KInitArguments"
+#endif
+
+namespace ams::kern::init {
+
+    KPhysicalAddress GetInitArgumentsAddress(s32 core_id);
+    void SetInitArguments(s32 core_id, KPhysicalAddress address, uintptr_t arg);
+    void StoreInitArguments();
+
+}

libraries/libmesosphere/include/mesosphere/init/kern_init_elf.hpp

@@ -17,23 +17,21 @@
 #include <vapours.hpp>
 
 #ifdef ATMOSPHERE_ARCH_ARM64
-    #include "kern_init_elf64.hpp"
+    #include <mesosphere/init/kern_init_elf64.hpp>
+
+    namespace ams::kern::init::Elf {
+        using namespace ams::kern::init::Elf::Elf64;
+
+        enum RelocationType {
+            R_ARCHITECTURE_RELATIVE = 0x403, /* Real name R_AARCH64_RELATIVE */
+        };
+    }
 #else
     #error "Unknown Architecture"
 #endif
 
 namespace ams::kern::init::Elf {
 
-    #ifdef ATMOSPHERE_ARCH_ARM64
-        using namespace ams::kern::init::Elf::Elf64;
-
-        enum RelocationType {
-            R_ARCHITECTURE_RELATIVE = 0x403, /* Real name R_AARCH64_RELATIVE */
-        };
-    #else
-        #error "Unknown Architecture"
-    #endif
-
     /* API to apply relocations or call init array. */
     void ApplyRelocations(uintptr_t base_address, const Dyn *dynamic);
     void CallInitArrayFuncs(uintptr_t init_array_start, uintptr_t init_array_end);

libraries/libmesosphere/include/mesosphere/init/kern_init_layout.hpp

@@ -27,7 +27,7 @@ namespace ams::kern::init {
         u32 rw_end_offset;
         u32 bss_offset;
         u32 bss_end_offset;
-        u32 ini_end_offset;
+        u32 ini_load_offset;
         u32 dynamic_offset;
         u32 init_array_offset;
         u32 init_array_end_offset;

libraries/libmesosphere/include/mesosphere/init/kern_init_page_table_select.hpp

@@ -14,9 +14,16 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 #pragma once
+#include <mesosphere/kern_common.hpp>
 
 #ifdef ATMOSPHERE_ARCH_ARM64
-    #include "../arch/arm64/init/kern_k_init_page_table.hpp"
+    #include <mesosphere/arch/arm64/init/kern_k_init_page_table.hpp>
+
+    namespace ams::kern::init {
+        using ams::kern::arch::arm64::PageTableEntry;
+        using ams::kern::arch::arm64::init::KInitialPageTable;
+        using ams::kern::arch::arm64::init::KInitialPageAllocator;
+    }
 #else
     #error "Unknown architecture for KInitialPageTable"
 #endif

libraries/libmesosphere/include/mesosphere/init/kern_init_slab_setup.hpp

@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <vapours.hpp>
+#include <mesosphere/kern_k_slab_heap.hpp>
+
+namespace ams::kern::init {
+
+    struct KSlabResourceCounts {
+        size_t num_KProcess;
+        size_t num_KThread;
+        size_t num_KEvent;
+        size_t num_KInterruptEvent;
+        size_t num_KPort;
+        size_t num_KSharedMemory;
+        size_t num_KTransferMemory;
+        size_t num_KCodeMemory;
+        size_t num_KDeviceAddressSpace;
+        size_t num_KSession;
+        size_t num_KLightSession;
+        size_t num_KObjectName;
+        size_t num_KResourceLimit;
+        size_t num_KDebug;
+    };
+
+    NOINLINE void InitializeSlabResourceCounts();
+    const KSlabResourceCounts &GetSlabResourceCounts();
+
+    size_t CalculateTotalSlabHeapSize();
+    NOINLINE void InitializeKPageBufferSlabHeap();
+    NOINLINE void InitializeSlabHeaps();
+
+}

libraries/libmesosphere/include/mesosphere/kern_common.hpp

@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <vapours.hpp>
+
+namespace ams::kern {
+
+    constexpr size_t PageSize = 4_KB;
+
+    enum TargetFirmware : u32 {
+        TargetFirmware_1_0_0  = ATMOSPHERE_TARGET_FIRMWARE_100,
+        TargetFirmware_2_0_0  = ATMOSPHERE_TARGET_FIRMWARE_200,
+        TargetFirmware_3_0_0  = ATMOSPHERE_TARGET_FIRMWARE_300,
+        TargetFirmware_4_0_0  = ATMOSPHERE_TARGET_FIRMWARE_400,
+        TargetFirmware_5_0_0  = ATMOSPHERE_TARGET_FIRMWARE_500,
+        TargetFirmware_6_0_0  = ATMOSPHERE_TARGET_FIRMWARE_600,
+        TargetFirmware_6_2_0  = ATMOSPHERE_TARGET_FIRMWARE_620,
+        TargetFirmware_7_0_0  = ATMOSPHERE_TARGET_FIRMWARE_700,
+        TargetFirmware_8_0_0  = ATMOSPHERE_TARGET_FIRMWARE_800,
+        TargetFirmware_8_1_0  = ATMOSPHERE_TARGET_FIRMWARE_810,
+        TargetFirmware_9_0_0  = ATMOSPHERE_TARGET_FIRMWARE_900,
+        TargetFirmware_9_1_0  = ATMOSPHERE_TARGET_FIRMWARE_910,
+        TargetFirmware_10_0_0 = ATMOSPHERE_TARGET_FIRMWARE_1000,
+    };
+
+    TargetFirmware GetTargetFirmware();
+
+}
+
+#if 1 || defined(AMS_BUILD_FOR_AUDITING)
+#define MESOSPHERE_BUILD_FOR_AUDITING
+#endif
+
+#if defined(MESOSPHERE_BUILD_FOR_AUDITING) || defined(AMS_BUILD_FOR_DEBUGGING)
+#define MESOSPHERE_BUILD_FOR_DEBUGGING
+#endif
+
+#ifdef  MESOSPHERE_BUILD_FOR_DEBUGGING
+#define MESOSPHERE_ENABLE_ASSERTIONS
+#define MESOSPHERE_ENABLE_DEBUG_PRINT
+#endif
+
+#include <mesosphere/svc/kern_svc_results.hpp>

libraries/libmesosphere/include/mesosphere/kern_debug_log.hpp

@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+
+namespace ams::kern {
+
+    class KDebugLog {
+        private:
+            static NOINLINE void VSNPrintf(char *dst, const size_t dst_size, const char *format, ::std::va_list vl);
+        public:
+            static NOINLINE void Initialize();
+
+            static NOINLINE void Printf(const char *format, ...) __attribute__((format(printf, 1, 2)));
+            static NOINLINE void VPrintf(const char *format, ::std::va_list vl);
+    };
+
+}
+
+#ifndef MESOSPHERE_DEBUG_LOG_SELECTED
+
+    #ifdef ATMOSPHERE_BOARD_NINTENDO_NX
+        #define MESOSPHERE_DEBUG_LOG_USE_UART_C
+    #else
+        #error "Unknown board for Default Debug Log Source"
+    #endif
+
+    #define MESOSPHERE_DEBUG_LOG_SELECTED
+
+#endif
+
+#define MESOSPHERE_RELEASE_LOG(fmt, ...) ::ams::kern::KDebugLog::Printf((fmt), ## __VA_ARGS__)
+#define MESOSPHERE_RELEASE_VLOG(fmt, vl) ::ams::kern::KDebugLog::VPrintf((fmt), (vl))
+
+#ifdef MESOSPHERE_ENABLE_DEBUG_PRINT
+#define MESOSPHERE_LOG(fmt, ...) MESOSPHERE_RELEASE_LOG((fmt), ## __VA_ARGS__)
+#define MESOSPHERE_VLOG(fmt, vl) MESOSPHERE_RELEASE_VLOG((fmt), (vl))
+#else
+#define MESOSPHERE_LOG(fmt, ...) do { MESOSPHERE_UNUSED(fmt); MESOSPHERE_UNUSED(__VA_ARGS__); } while (0)
+#define MESOSPHERE_VLOG(fmt, vl) do { MESOSPHERE_UNUSED(fmt); MESOSPHERE_UNUSED(vl); } while (0)
+#endif

libraries/libmesosphere/include/mesosphere/kern_initial_process.hpp

@@ -14,8 +14,8 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 #pragma once
-#include <vapours.hpp>
-#include "kern_panic.hpp"
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_k_initial_process_reader.hpp>
 
 namespace ams::kern {
 
@@ -29,4 +29,10 @@ namespace ams::kern {
         u32 reserved;
     };
 
+    NOINLINE void CopyInitialProcessBinaryToKernelMemory();
+    NOINLINE void CreateAndRunInitialProcesses();
+
+    u64 GetInitialProcessIdMin();
+    u64 GetInitialProcessIdMax();
+
 }

libraries/libmesosphere/include/mesosphere/kern_k_address_arbiter.hpp

@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_k_condition_variable.hpp>
+
+namespace ams::kern {
+
+    class KAddressArbiter {
+        public:
+            using ThreadTree = KConditionVariable::ThreadTree;
+        private:
+            ThreadTree tree;
+        public:
+            constexpr KAddressArbiter() : tree() { /* ... */ }
+
+            Result SignalToAddress(uintptr_t addr, ams::svc::SignalType type, s32 value, s32 count) {
+                switch (type) {
+                    case ams::svc::SignalType_Signal:
+                        return this->Signal(addr, count);
+                    case ams::svc::SignalType_SignalAndIncrementIfEqual:
+                        return this->SignalAndIncrementIfEqual(addr, value, count);
+                    case ams::svc::SignalType_SignalAndModifyByWaitingCountIfEqual:
+                        return this->SignalAndModifyByWaitingCountIfEqual(addr, value, count);
+                    MESOSPHERE_UNREACHABLE_DEFAULT_CASE();
+                }
+            }
+
+            Result WaitForAddress(uintptr_t addr, ams::svc::ArbitrationType type, s32 value, s64 timeout) {
+                switch (type) {
+                    case ams::svc::ArbitrationType_WaitIfLessThan:
+                        return this->WaitIfLessThan(addr, value, false, timeout);
+                    case ams::svc::ArbitrationType_DecrementAndWaitIfLessThan:
+                        return this->WaitIfLessThan(addr, value, true, timeout);
+                    case ams::svc::ArbitrationType_WaitIfEqual:
+                        return this->WaitIfEqual(addr, value, timeout);
+                    MESOSPHERE_UNREACHABLE_DEFAULT_CASE();
+                }
+            }
+        private:
+            Result Signal(uintptr_t addr, s32 count);
+            Result SignalAndIncrementIfEqual(uintptr_t addr, s32 value, s32 count);
+            Result SignalAndModifyByWaitingCountIfEqual(uintptr_t addr, s32 value, s32 count);
+            Result WaitIfLessThan(uintptr_t addr, s32 value, bool decrement, s64 timeout);
+            Result WaitIfEqual(uintptr_t addr, s32 value, s64 timeout);
+    };
+
+}

libraries/libmesosphere/include/mesosphere/kern_k_address_space_info.hpp

@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+
+namespace ams::kern {
+
+
+
+    struct KAddressSpaceInfo {
+        public:
+            enum Type {
+                Type_32Bit      = 0,
+                Type_Small64Bit = 1,
+                Type_Large64Bit = 2,
+                Type_Heap       = 3,
+                Type_Stack      = 4,
+                Type_Alias      = 5,
+
+                Type_Count,
+            };
+        private:
+            size_t bit_width;
+            size_t address;
+            size_t size;
+            Type type;
+        public:
+            static uintptr_t GetAddressSpaceStart(size_t width, Type type);
+            static size_t GetAddressSpaceSize(size_t width, Type type);
+
+            constexpr KAddressSpaceInfo(size_t bw, size_t a, size_t s, Type t) : bit_width(bw), address(a), size(s), type(t) { /* ... */ }
+
+            constexpr size_t GetWidth() const { return this->bit_width; }
+            constexpr size_t GetAddress() const { return this->address; }
+            constexpr size_t GetSize() const { return this->size; }
+            constexpr Type GetType() const { return this->type; }
+    };
+
+}

libraries/libmesosphere/include/mesosphere/kern_k_affinity_mask.hpp

@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_select_cpu.hpp>
+
+namespace ams::kern {
+
+    class KAffinityMask {
+        private:
+            static constexpr u64 AllowedAffinityMask = (1ul << cpu::NumCores) - 1;
+        private:
+            u64 mask;
+        private:
+            static constexpr ALWAYS_INLINE u64 GetCoreBit(s32 core) {
+                MESOSPHERE_ASSERT(0 <= core && core < static_cast<s32>(cpu::NumCores));
+                return (1ul << core);
+            }
+        public:
+            constexpr ALWAYS_INLINE KAffinityMask() : mask(0) { MESOSPHERE_ASSERT_THIS(); }
+
+            constexpr ALWAYS_INLINE u64 GetAffinityMask() const { return this->mask; }
+
+            constexpr ALWAYS_INLINE void SetAffinityMask(u64 new_mask) {
+                MESOSPHERE_ASSERT((new_mask & ~AllowedAffinityMask) == 0);
+                this->mask = new_mask;
+            }
+
+            constexpr ALWAYS_INLINE bool GetAffinity(s32 core) const {
+                return this->mask & GetCoreBit(core);
+            }
+
+            constexpr ALWAYS_INLINE void SetAffinity(s32 core, bool set) {
+                MESOSPHERE_ASSERT(0 <= core && core < static_cast<s32>(cpu::NumCores));
+
+                if (set) {
+                    this->mask |= GetCoreBit(core);
+                } else {
+                    this->mask &= ~GetCoreBit(core);
+                }
+            }
+
+            constexpr ALWAYS_INLINE void SetAll() {
+                this->mask = AllowedAffinityMask;
+            }
+    };
+
+}

libraries/libmesosphere/include/mesosphere/kern_k_auto_object.hpp

@@ -0,0 +1,232 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_k_typed_address.hpp>
+#include <mesosphere/kern_k_class_token.hpp>
+
+namespace ams::kern {
+
+    class KProcess;
+
+    #define MESOSPHERE_AUTOOBJECT_TRAITS(CLASS, BASE_CLASS)                                                     \
+        NON_COPYABLE(CLASS);                                                                                    \
+        NON_MOVEABLE(CLASS);                                                                                    \
+        private:                                                                                                \
+            friend class ::ams::kern::KClassTokenGenerator;                                                     \
+            static constexpr inline auto ObjectType = ::ams::kern::KClassTokenGenerator::ObjectType::CLASS;     \
+            static constexpr inline const char * const TypeName = #CLASS;                                       \
+            static constexpr inline ClassTokenType ClassToken() { return ::ams::kern::ClassToken<CLASS>; }      \
+        public:                                                                                                 \
+            using BaseClass = BASE_CLASS;                                                                       \
+            static constexpr ALWAYS_INLINE TypeObj GetStaticTypeObj() {                                         \
+                constexpr ClassTokenType Token = ClassToken();                                                  \
+                return TypeObj(TypeName, Token);                                                                \
+            }                                                                                                   \
+            static constexpr ALWAYS_INLINE const char *GetStaticTypeName() { return TypeName; }                 \
+            virtual TypeObj GetTypeObj() const { return GetStaticTypeObj(); }                                   \
+            virtual const char *GetTypeName() { return GetStaticTypeName(); }                                   \
+        private:
+
+
+    class KAutoObject {
+        protected:
+            class TypeObj {
+                private:
+                    const char *name;
+                    ClassTokenType class_token;
+                public:
+                    constexpr explicit TypeObj(const char *n, ClassTokenType tok) : name(n), class_token(tok) { /* ... */ }
+
+                    constexpr ALWAYS_INLINE const char *GetName() const { return this->name; }
+                    constexpr ALWAYS_INLINE ClassTokenType GetClassToken() const { return this->class_token; }
+
+                    constexpr ALWAYS_INLINE bool operator==(const TypeObj &rhs) {
+                        return this->GetClassToken() == rhs.GetClassToken();
+                    }
+
+                    constexpr ALWAYS_INLINE bool operator!=(const TypeObj &rhs) {
+                        return this->GetClassToken() != rhs.GetClassToken();
+                    }
+
+                    constexpr ALWAYS_INLINE bool IsDerivedFrom(const TypeObj &rhs) {
+                        return (this->GetClassToken() | rhs.GetClassToken()) == this->GetClassToken();
+                    }
+            };
+        private:
+            MESOSPHERE_AUTOOBJECT_TRAITS(KAutoObject, KAutoObject);
+        private:
+            std::atomic<u32> ref_count;
+        public:
+            static KAutoObject *Create(KAutoObject *ptr);
+        public:
+            constexpr ALWAYS_INLINE explicit KAutoObject() : ref_count(0) { MESOSPHERE_ASSERT_THIS(); }
+            virtual ~KAutoObject() { MESOSPHERE_ASSERT_THIS(); }
+
+            /* Destroy is responsible for destroying the auto object's resources when ref_count hits zero. */
+            virtual void Destroy() { MESOSPHERE_ASSERT_THIS(); }
+
+            /* Finalize is responsible for cleaning up resource, but does not destroy the object. */
+            virtual void Finalize() { MESOSPHERE_ASSERT_THIS(); }
+
+            virtual KProcess *GetOwner() const { return nullptr; }
+
+            u32 GetReferenceCount() const {
+                return this->ref_count;
+            }
+
+            ALWAYS_INLINE bool IsDerivedFrom(const TypeObj &rhs) const {
+                return this->GetTypeObj().IsDerivedFrom(rhs);
+            }
+
+            ALWAYS_INLINE bool IsDerivedFrom(const KAutoObject &rhs) const {
+                return this->IsDerivedFrom(rhs.GetTypeObj());
+            }
+
+            template<typename Derived>
+            ALWAYS_INLINE Derived DynamicCast() {
+                static_assert(std::is_pointer<Derived>::value);
+                using DerivedType = typename std::remove_pointer<Derived>::type;
+
+                if (AMS_LIKELY(this->IsDerivedFrom(DerivedType::GetStaticTypeObj()))) {
+                    return static_cast<Derived>(this);
+                } else {
+                    return nullptr;
+                }
+            }
+
+            template<typename Derived>
+            ALWAYS_INLINE const Derived DynamicCast() const {
+                static_assert(std::is_pointer<Derived>::value);
+                using DerivedType = typename std::remove_pointer<Derived>::type;
+
+                if (AMS_LIKELY(this->IsDerivedFrom(DerivedType::GetStaticTypeObj()))) {
+                    return static_cast<Derived>(this);
+                } else {
+                    return nullptr;
+                }
+            }
+
+            ALWAYS_INLINE bool Open() {
+                MESOSPHERE_ASSERT_THIS();
+
+                /* Atomically increment the reference count, only if it's positive. */
+                u32 cur_ref_count = this->ref_count.load(std::memory_order_acquire);
+                do {
+                    if (AMS_UNLIKELY(cur_ref_count == 0)) {
+                        return false;
+                    }
+                    MESOSPHERE_ABORT_UNLESS(cur_ref_count < cur_ref_count + 1);
+                } while (!this->ref_count.compare_exchange_weak(cur_ref_count, cur_ref_count + 1, std::memory_order_relaxed));
+
+                return true;
+            }
+
+            ALWAYS_INLINE void Close() {
+                MESOSPHERE_ASSERT_THIS();
+
+                /* Atomically decrement the reference count, not allowing it to become negative. */
+                u32 cur_ref_count = this->ref_count.load(std::memory_order_acquire);
+                do {
+                    MESOSPHERE_ABORT_UNLESS(cur_ref_count > 0);
+                } while (!this->ref_count.compare_exchange_weak(cur_ref_count, cur_ref_count - 1, std::memory_order_relaxed));
+
+                /* If ref count hits zero, destroy the object. */
+                if (cur_ref_count - 1 == 0) {
+                    this->Destroy();
+                }
+            }
+    };
+
+    class KAutoObjectWithListContainer;
+
+    class KAutoObjectWithList : public KAutoObject {
+        private:
+            friend class KAutoObjectWithListContainer;
+        private:
+            util::IntrusiveRedBlackTreeNode list_node;
+        public:
+            static ALWAYS_INLINE int Compare(const KAutoObjectWithList &lhs, const KAutoObjectWithList &rhs) {
+                const u64 lid = lhs.GetId();
+                const u64 rid = rhs.GetId();
+
+                if (lid < rid) {
+                    return -1;
+                } else if (lid > rid) {
+                    return 1;
+                } else {
+                    return 0;
+                }
+            }
+        public:
+            virtual u64 GetId() const {
+                return reinterpret_cast<u64>(this);
+            }
+    };
+
+    template<typename T>
+    class KScopedAutoObject {
+        static_assert(std::is_base_of<KAutoObject, T>::value);
+        NON_COPYABLE(KScopedAutoObject);
+        private:
+            T *obj;
+        private:
+            constexpr ALWAYS_INLINE void Swap(KScopedAutoObject &rhs) {
+                /* TODO: C++20 constexpr std::swap */
+                T *tmp = rhs.obj;
+                rhs.obj = this->obj;
+                this->obj = tmp;
+            }
+        public:
+            constexpr ALWAYS_INLINE KScopedAutoObject() : obj(nullptr) { /* ... */ }
+            constexpr ALWAYS_INLINE KScopedAutoObject(T *o) : obj(o) {
+                if (this->obj != nullptr) {
+                    this->obj->Open();
+                }
+            }
+
+            ALWAYS_INLINE ~KScopedAutoObject() {
+                if (this->obj != nullptr) {
+                    this->obj->Close();
+                }
+                this->obj = nullptr;
+            }
+
+            constexpr ALWAYS_INLINE KScopedAutoObject(KScopedAutoObject &&rhs) {
+                this->obj = rhs.obj;
+                rhs.obj = nullptr;
+            }
+
+            constexpr ALWAYS_INLINE KScopedAutoObject &operator=(KScopedAutoObject &&rhs) {
+                rhs.Swap(*this);
+                return *this;
+            }
+
+            constexpr ALWAYS_INLINE T *operator->() { return this->obj; }
+            constexpr ALWAYS_INLINE T &operator*() { return *this->obj; }
+
+            constexpr ALWAYS_INLINE void Reset(T *o) {
+                KScopedAutoObject(o).Swap(*this);
+            }
+
+            constexpr ALWAYS_INLINE T *GetPointerUnsafe() { return this->obj; }
+
+            constexpr ALWAYS_INLINE bool IsNull() const { return this->obj == nullptr; }
+            constexpr ALWAYS_INLINE bool IsNotNull() const { return this->obj != nullptr; }
+    };
+
+
+}

libraries/libmesosphere/include/mesosphere/kern_k_auto_object_container.hpp

@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_k_auto_object.hpp>
+#include <mesosphere/kern_k_light_lock.hpp>
+
+namespace ams::kern {
+
+    class KAutoObjectWithListContainer {
+        NON_COPYABLE(KAutoObjectWithListContainer);
+        NON_MOVEABLE(KAutoObjectWithListContainer);
+        private:
+            using ListType = util::IntrusiveRedBlackTreeMemberTraits<&KAutoObjectWithList::list_node>::TreeType<KAutoObjectWithList>;
+        public:
+            class ListAccessor : public KScopedLightLock {
+                private:
+                    ListType &list;
+                public:
+                    explicit ListAccessor(KAutoObjectWithListContainer *container) : KScopedLightLock(container->lock), list(container->object_list) { /* ... */ }
+                    explicit ListAccessor(KAutoObjectWithListContainer &container) : KScopedLightLock(container.lock), list(container.object_list) { /* ... */ }
+
+                    typename ListType::iterator begin() const {
+                        return this->list.begin();
+                    }
+
+                    typename ListType::iterator end() const {
+                        return this->list.end();
+                    }
+
+                    typename ListType::iterator find(typename ListType::const_reference ref) const {
+                        return this->list.find(ref);
+                    }
+            };
+
+            friend class ListAccessor;
+        private:
+            KLightLock lock;
+            ListType object_list;
+        public:
+            constexpr KAutoObjectWithListContainer() : lock(), object_list() { MESOSPHERE_ASSERT_THIS(); }
+
+            void Initialize() { MESOSPHERE_ASSERT_THIS(); }
+            void Finalize() { MESOSPHERE_ASSERT_THIS(); }
+
+            Result Register(KAutoObjectWithList *obj);
+            Result Unregister(KAutoObjectWithList *obj);
+            size_t GetOwnedCount(KProcess *owner);
+    };
+
+
+}

libraries/libmesosphere/include/mesosphere/kern_k_capabilities.hpp

@@ -0,0 +1,280 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_k_thread.hpp>
+#include <mesosphere/kern_select_page_table.hpp>
+#include <mesosphere/kern_svc.hpp>
+
+namespace ams::kern {
+
+    class KCapabilities {
+        private:
+            static constexpr size_t SvcFlagCount = svc::NumSupervisorCalls / BITSIZEOF(u8);
+            static constexpr size_t IrqFlagCount = /* TODO */0x80;
+
+            enum class CapabilityType : u32 {
+                CorePriority  = (1u <<  3) - 1,
+                SyscallMask   = (1u <<  4) - 1,
+                MapRange      = (1u <<  6) - 1,
+                MapIoPage     = (1u <<  7) - 1,
+                MapRegion     = (1u << 10) - 1,
+                InterruptPair = (1u << 11) - 1,
+                ProgramType   = (1u << 13) - 1,
+                KernelVersion = (1u << 14) - 1,
+                HandleTable   = (1u << 15) - 1,
+                DebugFlags    = (1u << 16) - 1,
+
+                Invalid       = 0u,
+                Padding       = ~0u,
+            };
+
+            using RawCapabilityValue = util::BitPack32::Field<0, BITSIZEOF(util::BitPack32), u32>;
+
+            static constexpr CapabilityType GetCapabilityType(const util::BitPack32 cap) {
+                const u32 value = cap.Get<RawCapabilityValue>();
+                return static_cast<CapabilityType>((~value & (value + 1)) - 1);
+            }
+
+            static constexpr u32 GetCapabilityFlag(CapabilityType type) {
+                return static_cast<u32>(type) + 1;
+            }
+
+            static constexpr u32 CountTrailingZero(u32 flag) {
+                for (u32 i = 0; i < BITSIZEOF(u32); i++) {
+                    if (flag & (1u << i)) {
+                        return i;
+                    }
+                }
+                return BITSIZEOF(u32);
+            }
+
+            static constexpr u32 GetCapabilityId(CapabilityType type) {
+                const u32 flag = GetCapabilityFlag(type);
+                if (true /* C++20: std::is_constant_evaluated() */) {
+                    return CountTrailingZero(flag);
+                } else {
+                    return static_cast<u32>(__builtin_ctz(flag));
+                }
+            }
+
+            template<size_t Index, size_t Count, typename T = u32>
+            using Field = util::BitPack32::Field<Index, Count, T>;
+
+            #define DEFINE_FIELD(name, prev, ...) using name = Field<prev::Next, __VA_ARGS__>
+
+            template<CapabilityType Type>
+            static constexpr inline u32 CapabilityFlag = []() -> u32 {
+                return static_cast<u32>(Type) + 1;
+            }();
+
+            template<CapabilityType Type>
+            static constexpr inline u32 CapabilityId = []() -> u32 {
+                const u32 flag = static_cast<u32>(Type) + 1;
+                if (true /* C++20: std::is_constant_evaluated() */) {
+                    for (u32 i = 0; i < BITSIZEOF(u32); i++) {
+                        if (flag & (1u << i)) {
+                            return i;
+                        }
+                    }
+                    return BITSIZEOF(u32);
+                } else {
+                    return __builtin_ctz(flag);
+                }
+            }();
+
+            struct CorePriority {
+                using IdBits = Field<0, CapabilityId<CapabilityType::CorePriority> + 1>;
+
+                DEFINE_FIELD(LowestThreadPriority,  IdBits,                6);
+                DEFINE_FIELD(HighestThreadPriority, LowestThreadPriority,  6);
+                DEFINE_FIELD(MinimumCoreId,         HighestThreadPriority, 8);
+                DEFINE_FIELD(MaximumCoreId,         MinimumCoreId,         8);
+            };
+
+            struct SyscallMask {
+                using IdBits = Field<0, CapabilityId<CapabilityType::SyscallMask> + 1>;
+
+                DEFINE_FIELD(Mask,  IdBits, 24);
+                DEFINE_FIELD(Index, Mask,    3);
+            };
+
+            static constexpr u64 PhysicalMapAllowedMask = (1ul << 36) - 1;
+
+            struct MapRange {
+                using IdBits = Field<0, CapabilityId<CapabilityType::MapRange> + 1>;
+
+                DEFINE_FIELD(Address,  IdBits,  24);
+                DEFINE_FIELD(ReadOnly, Address,  1, bool);
+            };
+
+            struct MapRangeSize {
+                using IdBits = Field<0, CapabilityId<CapabilityType::MapRange> + 1>;
+
+                DEFINE_FIELD(Pages,    IdBits,   20);
+                DEFINE_FIELD(Reserved, Pages,     4);
+                DEFINE_FIELD(Normal,   Reserved,  1, bool);
+            };
+
+            struct MapIoPage {
+                using IdBits = Field<0, CapabilityId<CapabilityType::MapIoPage> + 1>;
+
+                DEFINE_FIELD(Address, IdBits, 24);
+            };
+
+            enum class RegionType : u32 {
+                None              = 0,
+                KernelTraceBuffer = 1,
+                OnMemoryBootImage = 2,
+                DTB               = 3,
+            };
+
+            struct MapRegion {
+                using IdBits = Field<0, CapabilityId<CapabilityType::MapRegion> + 1>;
+
+                DEFINE_FIELD(Region0,   IdBits,      6, RegionType);
+                DEFINE_FIELD(ReadOnly0, Region0,     1, bool);
+                DEFINE_FIELD(Region1,   ReadOnly0,   6, RegionType);
+                DEFINE_FIELD(ReadOnly1, Region1,     1, bool);
+                DEFINE_FIELD(Region2,   ReadOnly1,   6, RegionType);
+                DEFINE_FIELD(ReadOnly2, Region2,     1, bool);
+            };
+
+            static const u32 PaddingInterruptId = 0x3FF;
+
+            struct InterruptPair {
+                using IdBits = Field<0, CapabilityId<CapabilityType::InterruptPair> + 1>;
+
+                DEFINE_FIELD(InterruptId0, IdBits,       10);
+                DEFINE_FIELD(InterruptId1, InterruptId0, 10);
+            };
+
+
+            struct ProgramType {
+                using IdBits = Field<0, CapabilityId<CapabilityType::ProgramType> + 1>;
+
+                DEFINE_FIELD(Type,     IdBits,  3);
+                DEFINE_FIELD(Reserved, Type,   15);
+            };
+
+            struct KernelVersion {
+                using IdBits = Field<0, CapabilityId<CapabilityType::KernelVersion> + 1>;
+
+                DEFINE_FIELD(MinorVersion, IdBits,        4);
+                DEFINE_FIELD(MajorVersion, MinorVersion, 13);
+            };
+
+            struct HandleTable {
+                using IdBits = Field<0, CapabilityId<CapabilityType::HandleTable> + 1>;
+
+                DEFINE_FIELD(Size,     IdBits, 10);
+                DEFINE_FIELD(Reserved, Size,    6);
+            };
+
+            struct DebugFlags {
+                using IdBits = Field<0, CapabilityId<CapabilityType::DebugFlags> + 1>;
+
+                DEFINE_FIELD(AllowDebug, IdBits,      1, bool);
+                DEFINE_FIELD(ForceDebug, AllowDebug,  1, bool);
+                DEFINE_FIELD(Reserved,   ForceDebug, 13);
+            };
+
+            #undef DEFINE_FIELD
+
+            static constexpr u32 InitializeOnceFlags = CapabilityFlag<CapabilityType::CorePriority>  |
+                                                       CapabilityFlag<CapabilityType::ProgramType>   |
+                                                       CapabilityFlag<CapabilityType::KernelVersion> |
+                                                       CapabilityFlag<CapabilityType::HandleTable>   |
+                                                       CapabilityFlag<CapabilityType::DebugFlags>;
+        private:
+            u8 svc_access_flags[SvcFlagCount]{};
+            u8 irq_access_flags[IrqFlagCount]{};
+            u64 core_mask{};
+            u64 priority_mask{};
+            util::BitPack32 debug_capabilities;
+            s32 handle_table_size{};
+            util::BitPack32 intended_kernel_version;
+            u32 program_type{};
+        private:
+            static constexpr ALWAYS_INLINE void SetSvcAllowedImpl(u8 *data, u32 id) {
+                constexpr size_t BitsPerWord = BITSIZEOF(*data);
+                MESOSPHERE_ASSERT(id < svc::SvcId_Count);
+                data[id / BitsPerWord] |= (1ul << (id % BitsPerWord));
+            }
+
+            static constexpr ALWAYS_INLINE void ClearSvcAllowedImpl(u8 *data, u32 id) {
+                constexpr size_t BitsPerWord = BITSIZEOF(*data);
+                MESOSPHERE_ASSERT(id < svc::SvcId_Count);
+                data[id / BitsPerWord] &= ~(1ul << (id % BitsPerWord));
+            }
+
+            bool SetSvcAllowed(u32 id) {
+                if (id < BITSIZEOF(this->svc_access_flags)) {
+                    SetSvcAllowedImpl(this->svc_access_flags, id);
+                    return true;
+                } else {
+                    return false;
+                }
+            }
+
+            bool SetInterruptAllowed(u32 id) {
+                constexpr size_t BitsPerWord = BITSIZEOF(this->irq_access_flags[0]);
+                if (id < BITSIZEOF(this->irq_access_flags)) {
+                    this->irq_access_flags[id / BitsPerWord] = (1ul << (id % BitsPerWord));
+                    return true;
+                } else {
+                    return false;
+                }
+            }
+
+            Result SetCorePriorityCapability(const util::BitPack32 cap);
+            Result SetSyscallMaskCapability(const util::BitPack32 cap, u32 &set_svc);
+            Result MapRange(const util::BitPack32 cap, const util::BitPack32 size_cap, KProcessPageTable *page_table);
+            Result MapIoPage(const util::BitPack32 cap, KProcessPageTable *page_table);
+            Result MapRegion(const util::BitPack32 cap, KProcessPageTable *page_table);
+            Result SetInterruptPairCapability(const util::BitPack32 cap);
+            Result SetProgramTypeCapability(const util::BitPack32 cap);
+            Result SetKernelVersionCapability(const util::BitPack32 cap);
+            Result SetHandleTableCapability(const util::BitPack32 cap);
+            Result SetDebugFlagsCapability(const util::BitPack32 cap);
+
+            Result SetCapability(const util::BitPack32 cap, u32 &set_flags, u32 &set_svc, KProcessPageTable *page_table);
+            Result SetCapabilities(const u32 *caps, s32 num_caps, KProcessPageTable *page_table);
+        public:
+            constexpr KCapabilities() : debug_capabilities(0), intended_kernel_version(0) { /* ... */ }
+
+            Result Initialize(const u32 *caps, s32 num_caps, KProcessPageTable *page_table);
+
+            constexpr u64 GetCoreMask() const { return this->core_mask; }
+            constexpr u64 GetPriorityMask() const { return this->priority_mask; }
+            constexpr s32 GetHandleTableSize() const { return this->handle_table_size; }
+
+            ALWAYS_INLINE void CopySvcPermissionsTo(KThread::StackParameters &sp) const {
+                static_assert(sizeof(svc_access_flags) == sizeof(sp.svc_permission));
+                std::memcpy(sp.svc_permission, this->svc_access_flags, sizeof(this->svc_access_flags));
+
+                /* Clear specific SVCs based on our state. */
+                ClearSvcAllowedImpl(sp.svc_permission, svc::SvcId_ReturnFromException);
+                ClearSvcAllowedImpl(sp.svc_permission, svc::SvcId_SynchronizePreemptionState);
+                if (sp.is_preemption_state_pinned) {
+                    ClearSvcAllowedImpl(sp.svc_permission, svc::SvcId_GetInfo);
+                }
+            }
+
+            /* TODO: Member functions. */
+    };
+
+}

libraries/libmesosphere/include/mesosphere/kern_k_class_token.hpp

@@ -0,0 +1,127 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_k_typed_address.hpp>
+
+namespace ams::kern {
+
+    class KAutoObject;
+
+    class KClassTokenGenerator {
+        public:
+            using TokenBaseType = u16;
+        public:
+            static constexpr size_t BaseClassBits = 8;
+            static constexpr size_t FinalClassBits = (sizeof(TokenBaseType) * CHAR_BIT) - BaseClassBits;
+            /* One bit per base class. */
+            static constexpr size_t NumBaseClasses = BaseClassBits;
+            /* Final classes are permutations of three bits. */
+            static constexpr size_t NumFinalClasses = [] {
+                TokenBaseType index = 0;
+                for (size_t i = 0; i < FinalClassBits; i++) {
+                    for (size_t j = i + 1; j < FinalClassBits; j++) {
+                        for (size_t k = j + 1; k < FinalClassBits; k++) {
+                            index++;
+                        }
+                    }
+                }
+                return index;
+            }();
+        private:
+            template<TokenBaseType Index>
+            static constexpr inline TokenBaseType BaseClassToken = BIT(Index);
+
+            template<TokenBaseType Index>
+            static constexpr inline TokenBaseType FinalClassToken = [] {
+                TokenBaseType index = 0;
+                for (size_t i = 0; i < FinalClassBits; i++) {
+                    for (size_t j = i + 1; j < FinalClassBits; j++) {
+                        for (size_t k = j + 1; k < FinalClassBits; k++) {
+                            if ((index++) == Index) {
+                                return ((1ul << i) | (1ul << j) | (1ul << k)) << BaseClassBits;
+                            }
+                        }
+                    }
+                }
+                __builtin_unreachable();
+            }();
+
+            template<typename T>
+            static constexpr inline TokenBaseType GetClassToken() {
+                static_assert(std::is_base_of<KAutoObject, T>::value);
+                if constexpr (std::is_same<T, KAutoObject>::value) {
+                    static_assert(T::ObjectType == ObjectType::KAutoObject);
+                    return 0;
+                } else if constexpr (!std::is_final<T>::value) {
+                    static_assert(ObjectType::BaseClassesStart <= T::ObjectType && T::ObjectType < ObjectType::BaseClassesEnd);
+                    constexpr auto ClassIndex = static_cast<TokenBaseType>(T::ObjectType) - static_cast<TokenBaseType>(ObjectType::BaseClassesStart);
+                    return BaseClassToken<ClassIndex> | GetClassToken<typename T::BaseClass>();
+                } else if constexpr (ObjectType::FinalClassesStart <= T::ObjectType && T::ObjectType < ObjectType::FinalClassesEnd) {
+                    constexpr auto ClassIndex = static_cast<TokenBaseType>(T::ObjectType) - static_cast<TokenBaseType>(ObjectType::FinalClassesStart);
+                    return FinalClassToken<ClassIndex> | GetClassToken<typename T::BaseClass>();
+                } else {
+                    static_assert(!std::is_same<T, T>::value, "GetClassToken: Invalid Type");
+                }
+            };
+        public:
+            enum class ObjectType {
+                KAutoObject,
+
+                BaseClassesStart,
+
+                KSynchronizationObject = BaseClassesStart,
+                KReadableEvent,
+
+                BaseClassesEnd,
+
+                FinalClassesStart = BaseClassesEnd,
+
+                KInterruptEvent = FinalClassesStart,
+                KDebug,
+                KThread,
+                KServerPort,
+                KServerSession,
+                KClientPort,
+                KClientSession,
+                KProcess,
+                KResourceLimit,
+                KLightSession,
+                KPort,
+                KSession,
+                KSharedMemory,
+                KEvent,
+                KWritableEvent,
+                KLightClientSession,
+                KLightServerSession,
+                KTransferMemory,
+                KDeviceAddressSpace,
+                KSessionRequest,
+                KCodeMemory,
+
+                FinalClassesEnd = FinalClassesStart + NumFinalClasses,
+            };
+
+            template<typename T>
+            static constexpr inline TokenBaseType ClassToken = GetClassToken<T>();
+    };
+
+    using ClassTokenType = KClassTokenGenerator::TokenBaseType;
+
+    template<typename T>
+    static constexpr inline ClassTokenType ClassToken = KClassTokenGenerator::ClassToken<T>;
+
+}