dmnt: revise per WerWolv's feedback.

subrepo:
  subdir:   "libraries"
  merged:   "c1fe12fc"
upstream:
  origin:   "https://github.com/Atmosphere-NX/Atmosphere-libs"
  branch:   "master"
  commit:   "c1fe12fc"
git-subrepo:
  version:  "0.4.1"
  origin:   "???"
  commit:   "???"

.gitignore

@@ -64,6 +64,7 @@ dkms.conf
 # Distribution files
 *.tgz
 *.zip
+*.bz2
 
 # IDA binaries
 *.id0

Makefile

@@ -58,6 +58,7 @@ dist-no-debug: all
 	mkdir atmosphere-$(AMSVER)/switch
 	mkdir -p atmosphere-$(AMSVER)/atmosphere/contents/0100000000000008
 	mkdir -p atmosphere-$(AMSVER)/atmosphere/contents/010000000000000D
+	mkdir -p atmosphere-$(AMSVER)/atmosphere/contents/010000000000002B
 	mkdir -p atmosphere-$(AMSVER)/atmosphere/contents/0100000000000032
 	mkdir -p atmosphere-$(AMSVER)/atmosphere/contents/0100000000000034
 	mkdir -p atmosphere-$(AMSVER)/atmosphere/contents/0100000000000036
@@ -82,6 +83,7 @@ dist-no-debug: all
 	cp -r config_templates/hbl_html atmosphere-$(AMSVER)/atmosphere/hbl_html
 	cp stratosphere/boot2/boot2.nsp atmosphere-$(AMSVER)/atmosphere/contents/0100000000000008/exefs.nsp
 	cp stratosphere/dmnt/dmnt.nsp atmosphere-$(AMSVER)/atmosphere/contents/010000000000000D/exefs.nsp
+	cp stratosphere/erpt/erpt.nsp atmosphere-$(AMSVER)/atmosphere/contents/010000000000002B/exefs.nsp
 	cp stratosphere/eclct.stub/eclct.stub.nsp atmosphere-$(AMSVER)/atmosphere/contents/0100000000000032/exefs.nsp
 	cp stratosphere/fatal/fatal.nsp atmosphere-$(AMSVER)/atmosphere/contents/0100000000000034/exefs.nsp
 	cp stratosphere/creport/creport.nsp atmosphere-$(AMSVER)/atmosphere/contents/0100000000000036/exefs.nsp
@@ -133,6 +135,7 @@ dist: dist-no-debug
 	cp stratosphere/ro/ro.elf atmosphere-$(AMSVER)-debug/ro.elf
 	cp stratosphere/sm/sm.elf atmosphere-$(AMSVER)-debug/sm.elf
 	cp stratosphere/spl/spl.elf atmosphere-$(AMSVER)-debug/spl.elf
+	cp stratosphere/erpt/erpt.elf atmosphere-$(AMSVER)-debug/erpt.elf
 	cd atmosphere-$(AMSVER)-debug; zip -r ../atmosphere-$(AMSVER)-debug.zip ./*; cd ../;
 	rm -r atmosphere-$(AMSVER)-debug
 	mv atmosphere-$(AMSVER)-debug.zip out/atmosphere-$(AMSVER)-debug.zip

README.md

@@ -27,7 +27,8 @@ This software is licensed under the terms of the GPLv2, with exemptions for spec
 You can find a copy of the license in the [LICENSE file](LICENSE).
 
 Exemptions:
-* The [yuzu emulator project](https://github.com/yuzu-emu/yuzu) is exempt from GPLv2 licensing and may (at its option) instead license any source code authored for the Atmosphère project as GPLv2 or later.
+* The [yuzu Nintendo Switch emulator](https://github.com/yuzu-emu/yuzu) and the [Ryujinx Team and Contributors](https://github.com/orgs/Ryujinx) are exempt from GPLv2 licensing. They are permitted, each at their individual discretion, to instead license any source code authored for the Atmosphère project as either GPLv2 or later or the [MIT license](https://github.com/Atmosphere-NX/Atmosphere/blob/master/docs/licensing_exemptions/MIT_LICENSE). In doing so, they may alter, supplement, or entirely remove the copyright notice for each file they choose to relicense. Neither the Atmosphère project nor its individual contributors shall assert their moral rights against any of the aforementioned projects.
+* [Nintendo](https://github.com/Nintendo) is exempt from GPLv2 licensing and may (at its option) instead license any source code authored for the Atmosphère project under the Zero-Clause BSD license.
 
 Credits
 =====

config_templates/BCT.ini

@@ -18,3 +18,5 @@ enable_user_pmu_access = 0
 [stratosphere]
 ; To force-enable nogc, add nogc = 1
 ; To force-disable nogc, add nogc = 0
+
+; To opt in to using Atmosphere's NCM reimplementation, add enable_ncm = 1

docs/changelog.md

@@ -1,4 +1,82 @@
 # Changelog
+## 0.11.1
++ A bug was fixed that could cause owls to flicker under certain circumstances.
+  + For those interested in technical details, in 10.0.0 kernelldr/kernel no longer set cpuactlr_el1, assuming that it was set correctly by the secure monitor.
+  + However, exosphere did not set cpuactlr_el1. This meant that the register held the reset value going into boot.
+  + This caused a variety of highly erratic symptoms, including causing basically any game to crash seemingly randomly.
++ A number of other major inaccuracies in exosphere were corrected.
++ General system stability improvements to enhance the user's experience.
+## 0.11.0
++ Support was added for 10.0.0.
+  + Exosphere has been updated to reflect the new key import semantics in 10.0.0.
+  + kernel_ldr now implements physical ASLR for the kernel's backing pages.
+  + Loader, NCM, and PM have been updated to reflect the changes Nintendo made in 10.0.0.
+  + Creport was updated to use the new `pgl` service to terminate processes instead of `ns:dev`.
++ A reimplementation of the `erpt` (error reports) system module was added.
+  + In previous versions of Atmosphere, a majority of error reports were prevented via a combination of custom creport, fatal, and stubbed eclct.
+  + However, error reports were still generated via some system actions.
+    + Most notably, any time the error applet appeared, an error report was generated.
+    + By default, atmosphere disabled the *uploading* of error reports, but going online in OFW after an error report occurred in Atmosphere could lead to undesirable telemetry.
+  + Atmosphere's `erpt` reimplementation allows the system to interact with existing error reports as expected.
+  + However, all new error reports are instead saved to the sd card (`/atmosphere/erpt_reports`), and are not committed to the system savegame.
+    + Users curious about what kind of telemetry is being prevented can view the reports as they're generated in there.
+    + Reports are saved as msgpack (as this is what Nintendo uses).
+  + Please note, not all telemetry is disabled. Play reports and System reports will continue to function unmodified.
+  + With atmosphere's `erpt` implementation, homebrew can now use the native error applet to display errors without worrying about generating undesirable telemetry.
++ libstratosphere and libvapours received a number of improvements.
+  + With thanks to @Adubbz for his work, the NCM namespace now has client code.
+    + This lays the groundwork for first-class system update/downgrade homebrew support in the near future.
+  + In particular, code implementing the os namespace is significantly more accurate.
+  + In addition, Nintendo's allocators were implemented, allowing for identical memory efficiency versus Nintendo's implementations.
++ General system stability improvements to enhance the user's experience.
+## 0.10.5
++ Changes were made to the way fs.mitm builds images when providing a layeredfs romfs.
+  + Building romfs metadata previously had a memory cost of about ~4-5x the file table size.
+  + This caused games that have particularly enormous file metadata tables (> 4 MB) to exhaust fs.mitm's 16 MB memory pool.
+  + The code that creates romfs images was thus changed to be significantly more memory efficient, again.
+  + Memory requirements have been lowered from ~4x file table size to ~2x file table size + 0.5 MB.
+  + There is a slight speed penalty to this, but testing on Football Manager 2020 only took an extra ~1.5 seconds for the game to boot with many modded files.
+    + This shouldn't be noticeable thanks to the async changes made in 0.10.2.
+  + If you encounter a game that exhausts ams.mitm's memory (crashing it) when loading layeredfs mods, please contact SciresM.
+    + Romfs building can be made even more memory efficient, but unless games show up with even more absurdly huge file tables it seems not worth the speed trade-off.
++ A bug was fixed that caused Atmosphere's fatal error context to not dump TLS for certain processes.
++ General system stability improvements to enhance the user's experience.
+## 0.10.4
++ With major thanks to @Adubbz for his work, the NCM system module has now been re-implemented.
+  + This was a major stepping stone towards the goal of having implementations everything in the Switch's package1/package2 firmware.
+  + This also lays the groundwork for libstratosphere providing bindings for changing the installed version of the Switch's OS.
+  + **Please Note**: The NCM implementation will initially be opt-in.
+    + The Atmosphere team is confident in our NCM implementation (and we have tested it on every firmware version).
+    + That said, this is our first system module that manages NAND savegames -- and caution is a habit.
+    + We do not anticipate any issues that didn't come up in testing, so this is just our being particularly careful.
+    + Users interested in opting in to using our implementation should set `stratosphere!ncm_enabled = 1` in BCT.ini.
+      + In the unlikely event that any issues are encountered, please report them to @SciresM.
+    + The NCM implementation will stop being opt-in in a future update, after thorough testing has been done in practice.
++ A bug was fixed in emummc that caused Nintendo path to be corrupted on 1.0.0.
+  + This manifested as the emummc folder being created inside the virtual NAND instead of the SD card.
+  + It's unlikely there are any negative consequences to this in practice.
+    + If you want to be truly sure, you can re-clone sysmmc before updating a 1.0.0 emummc to latest firmware.
++ Stratosphere system modules now use new Nintendo-style FS bindings instead of stdio.
+  + This saves a modest amount of memory due to leaner code, and greatly increases the accuracy of several components.
+  + These bindings will make it easier for other system modules using libstratosphere to interact with the filesystem.
+  + This also lays the groundwork for changes necessary to support per-emummc Atmosphere folders in a future update.
++ Atmosphere's fatal error context now dumps 0x100 of TLS.
+  + This will make it much easier to fix bugs when an error report is dumped for whatever caused the crash.
++ General system stability improvements to enhance the user's experience.
+## 0.10.3
++ Support was added for 9.2.0.
++ Support was added for redirecting manual html content for games.
+  + This works like normal layeredfs, replacing content placed in `/atmosphere/contents/<program id>/manual_html/`.
+  + This allows for game mods/translations to provide custom manual content, if they so choose.
++ A number of improvements were made to Atmosphere's memory usage, including:
+  + `fatal` now uses STB instead of freetype for rendering.
+    + This saves around 1 MB of memory, and makes our fatal substantially leaner than Nintendo's.
+  + `sm` no longer wastes 2 MiB unnecessarily.
++ fusee/sept's sdmmc access now better matches official behavior.
+  + This improves compatibility with some SD cards.
++ `ro` has been updated to reflect changes made in 9.1.0.
++ The temporary auto-migration added in 0.10.0 has been removed, since the transitionary period is well over.
++ General system stability improvements to enhance the user's experience.
 ## 0.10.2
 + hbl configuration was made more flexible.
   + Up to eight specific program ids can now be specified to have their own override keys.

docs/licensing_exemptions/MIT_LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018-2020 Atmosphère-NX
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

emummc/.gitrepo

@@ -6,7 +6,7 @@
 [subrepo]
 	remote = https://github.com/m4xw/emuMMC
 	branch = develop
-	commit = bd81a674a946c30b566e1732a95c18f19b701558
-	parent = 6ee525201ccef107c61d81ba73c891e3eb5f0215
+	commit = b168ddf5fbb31013ff529a4859110c82b11eb361
+	parent = c07f54f3709a4710e0aead6c91139fa0893b5e5c
 	method = rebase
-	cmdver = 0.4.0
+	cmdver = 0.4.1

emummc/README.md

@@ -1,21 +1,21 @@
 # emuMMC
-*A SDMMC driver replacement for Nintendo's Filesystem Services, by **m4xw***
+*A SDMMC driver replacement for Nintendo's Filesystem Services, by **m4xw***  
 
 ### Supported Horizon Versions
-**1.0.0 - 9.1.0**
+**1.0.0 - 10.0.0**
 
 ## Features
-* Arbitrary SDMMC backend selection
+* Arbitrary SDMMC backend selection  
   **This allows loading eMMC from SD or even SD from eMMC**
-* On the fly hooking / patching, fully self-infesting
+* On the fly hooking / patching, fully self-infesting  
   **Only one payload required for all versions!**
-* File-based SDMMC backend support (from SD)
+* File-based SDMMC backend support (from SD)  
   **This allows loading eMMC images from hekate-backups (split or not)**
-* SDMMC device based sector offset (*currently eMMC only*)
+* SDMMC device based sector offset (*currently eMMC only*)  
   **Raw partition support for eMMC from SD with less performance overhead**
-* Full support for `/Nintendo` folder redirection to a arbitrary path
+* Full support for `/Nintendo` folder redirection to a arbitrary path  
   **No 8 char length restriction!**
-* exosphere based context configuration
+* exosphere based context configuration  
   **This includes full support for multiple emuMMC images**
 
 ## Compiling

emummc/source/FS/FS_offsets.c

@@ -45,6 +45,8 @@
 #include "offsets/900_exfat.h"
 #include "offsets/910.h"
 #include "offsets/910_exfat.h"
+#include "offsets/1000.h"
+#include "offsets/1000_exfat.h"
 #include "../utils/fatal.h"
 
 #define GET_OFFSET_STRUCT_NAME(vers) g_offsets##vers
@@ -100,6 +102,8 @@ DEFINE_OFFSET_STRUCT(_900);
 DEFINE_OFFSET_STRUCT(_900_EXFAT);
 DEFINE_OFFSET_STRUCT(_910);
 DEFINE_OFFSET_STRUCT(_910_EXFAT);
+DEFINE_OFFSET_STRUCT(_1000);
+DEFINE_OFFSET_STRUCT(_1000_EXFAT);
 
 const fs_offsets_t *get_fs_offsets(enum FS_VER version) {
     switch (version) {
@@ -161,6 +165,10 @@ const fs_offsets_t *get_fs_offsets(enum FS_VER version) {
             return &(GET_OFFSET_STRUCT_NAME(_910));
         case FS_VER_9_1_0_EXFAT:
             return &(GET_OFFSET_STRUCT_NAME(_910_EXFAT));
+        case FS_VER_10_0_0:
+            return &(GET_OFFSET_STRUCT_NAME(_1000));
+        case FS_VER_10_0_0_EXFAT:
+            return &(GET_OFFSET_STRUCT_NAME(_1000_EXFAT));
         default:
             fatal_abort(Fatal_UnknownVersion);
     }

emummc/source/FS/FS_versions.h

@@ -65,6 +65,9 @@ enum FS_VER
     FS_VER_9_1_0,
     FS_VER_9_1_0_EXFAT,
 
+    FS_VER_10_0_0,
+    FS_VER_10_0_0_EXFAT,
+
     FS_VER_MAX,
 };
 

emummc/source/FS/offsets/100.h

@@ -48,10 +48,8 @@
 // Nintendo Paths
 #define FS_OFFSET_100_NINTENDO_PATHS \
 { \
-    {.opcode_reg = 9, .adrp_offset = 0x00032C58, .add_rel_offset = 4}, \
-    {.opcode_reg = 8, .adrp_offset = 0x00032C60, .add_rel_offset = 4}, \
-    {.opcode_reg = 9, .adrp_offset = 0x00032F3C, .add_rel_offset = 4}, \
-    {.opcode_reg = 8, .adrp_offset = 0x00032F44, .add_rel_offset = 4}, \
+    {.opcode_reg = 8, .adrp_offset = 0x00032C58, .add_rel_offset = 8}, \
+    {.opcode_reg = 9, .adrp_offset = 0x00032F40, .add_rel_offset = 8}, \
     {.opcode_reg = 0, .adrp_offset = 0, .add_rel_offset = 0}, \
 }
 

emummc/source/FS/offsets/1000.h

@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2019 m4xw <m4x@m4xw.net>
+ * Copyright (c) 2019 Atmosphere-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef __FS_1000_H__
+#define __FS_1000_H__
+
+// Accessor vtable getters
+#define FS_OFFSET_1000_SDMMC_ACCESSOR_GC   0x14DC90
+#define FS_OFFSET_1000_SDMMC_ACCESSOR_SD   0x14BDA0
+#define FS_OFFSET_1000_SDMMC_ACCESSOR_NAND 0x146C20
+
+// Hooks
+#define FS_OFFSET_1000_SDMMC_WRAPPER_READ  0x142380
+#define FS_OFFSET_1000_SDMMC_WRAPPER_WRITE 0x142460
+#define FS_OFFSET_1000_RTLD                0x634
+#define FS_OFFSET_1000_RTLD_DESTINATION    0x9C
+
+#define FS_OFFSET_1000_CLKRST_SET_MIN_V_CLK_RATE 0x1415A0
+
+// Misc funcs
+#define FS_OFFSET_1000_LOCK_MUTEX          0x28910
+#define FS_OFFSET_1000_UNLOCK_MUTEX        0x28960
+
+#define FS_OFFSET_1000_SDMMC_WRAPPER_CONTROLLER_CLOSE 0x1422E0
+
+// Misc Data
+#define FS_OFFSET_1000_SD_MUTEX            0xE273E8
+#define FS_OFFSET_1000_NAND_MUTEX          0xE22DA0
+#define FS_OFFSET_1000_ACTIVE_PARTITION    0xE22DE0
+#define FS_OFFSET_1000_SDMMC_DAS_HANDLE    0xE0AB90
+
+// NOPs
+#define FS_OFFSET_1000_SD_DAS_INIT         0x151CEC
+
+// Nintendo Paths
+#define FS_OFFSET_1000_NINTENDO_PATHS \
+{ \
+	{.opcode_reg = 3, .adrp_offset = 0x0006BBA4, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 3, .adrp_offset = 0x00078520, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 3, .adrp_offset = 0x0007ED0C, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 4, .adrp_offset = 0x0009115C, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 0, .adrp_offset = 0, .add_rel_offset = 0}, \
+}
+
+#endif // __FS_1000_H__

emummc/source/FS/offsets/1000_exfat.h

@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2019 m4xw <m4x@m4xw.net>
+ * Copyright (c) 2019 Atmosphere-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef __FS_1000_EXFAT_H__
+#define __FS_1000_EXFAT_H__
+
+// Accessor vtable getters
+#define FS_OFFSET_1000_EXFAT_SDMMC_ACCESSOR_GC   0x14DC90
+#define FS_OFFSET_1000_EXFAT_SDMMC_ACCESSOR_SD   0x14BDA0
+#define FS_OFFSET_1000_EXFAT_SDMMC_ACCESSOR_NAND 0x146C20
+
+// Hooks
+#define FS_OFFSET_1000_EXFAT_SDMMC_WRAPPER_READ  0x142380
+#define FS_OFFSET_1000_EXFAT_SDMMC_WRAPPER_WRITE 0x142460
+#define FS_OFFSET_1000_EXFAT_RTLD                0x634
+#define FS_OFFSET_1000_EXFAT_RTLD_DESTINATION    0x9C
+
+#define FS_OFFSET_1000_EXFAT_CLKRST_SET_MIN_V_CLK_RATE 0x1415A0
+
+// Misc funcs
+#define FS_OFFSET_1000_EXFAT_LOCK_MUTEX          0x28910
+#define FS_OFFSET_1000_EXFAT_UNLOCK_MUTEX        0x28960
+
+#define FS_OFFSET_1000_EXFAT_SDMMC_WRAPPER_CONTROLLER_CLOSE 0x1422E0
+
+// Misc Data
+#define FS_OFFSET_1000_EXFAT_SD_MUTEX            0xE353E8
+#define FS_OFFSET_1000_EXFAT_NAND_MUTEX          0xE30DA0
+#define FS_OFFSET_1000_EXFAT_ACTIVE_PARTITION    0xE30DE0
+#define FS_OFFSET_1000_EXFAT_SDMMC_DAS_HANDLE    0xE18B90
+
+// NOPs
+#define FS_OFFSET_1000_EXFAT_SD_DAS_INIT         0x151CEC
+
+// Nintendo Paths
+#define FS_OFFSET_1000_EXFAT_NINTENDO_PATHS \
+{ \
+	{.opcode_reg = 3, .adrp_offset = 0x0006BBA4, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 3, .adrp_offset = 0x00078520, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 3, .adrp_offset = 0x0007ED0C, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 4, .adrp_offset = 0x0009115C, .add_rel_offset = 0x00000004}, \
+	{.opcode_reg = 0, .adrp_offset = 0, .add_rel_offset = 0}, \
+}
+
+#endif // __FS_1000_EXFAT_H__

emummc/source/emuMMC/emummc.c

@@ -36,7 +36,6 @@ sdmmc_storage_t sd_storage;
 
 // init vars
 bool custom_driver = true;
-extern const volatile emuMMC_ctx_t emuMMC_ctx;
 
 // FS funcs
 _sdmmc_accessor_gc sdmmc_accessor_gc;
@@ -344,7 +343,7 @@ uint64_t sdmmc_wrapper_controller_close(int mmc_id)
         {
             return 0;
         }
-        
+
         if (mmc_id == FS_SDMMC_EMMC)
         {
             // Close file handles and unmount

emummc/source/main.c

@@ -48,6 +48,9 @@ extern char __argdata__;
 // TODO
 static char nintendo_path[0x80] = "Nintendo";
 
+// 1.0.0 requires special path handling because it has separate album and contents paths.
+#define FS_100_ALBUM_PATH    0
+#define FS_100_CONTENTS_PATH 1
 static char nintendo_path_album_100[0x100] = "/Nintendo/Album";
 static char nintendo_path_contents_100[0x100] = "/Nintendo/Contents";
 
@@ -275,23 +278,18 @@ void setup_nintendo_paths(void)
         // 1.0.0 needs special handling because it uses two paths.
         // Do album path
         {
-            int path_len = snprintf(nintendo_path_album_100, sizeof(nintendo_path_album_100), "/%s/Album", nintendo_path);
+            snprintf(nintendo_path_album_100, sizeof(nintendo_path_album_100), "/%s/Album", nintendo_path);
             intptr_t nintendo_album_path_location = (intptr_t)&nintendo_path_album_100;
-            intptr_t album_path_location = nintendo_album_path_location + path_len - 6; // "/Album"
-            uintptr_t fs_n_adrp_opcode_location = INJECT_OFFSET(uintptr_t, fs_offsets->nintendo_paths[0].adrp_offset);
-            uintptr_t fs_adrp_opcode_location = INJECT_OFFSET(uintptr_t, fs_offsets->nintendo_paths[1].adrp_offset);
-            write_adrp_add(fs_offsets->nintendo_paths[0].opcode_reg, fs_n_adrp_opcode_location, fs_offsets->nintendo_paths[0].add_rel_offset, nintendo_album_path_location);
-            write_adrp_add(fs_offsets->nintendo_paths[1].opcode_reg, fs_adrp_opcode_location, fs_offsets->nintendo_paths[1].add_rel_offset, album_path_location);
+            uintptr_t fs_adrp_opcode_location = INJECT_OFFSET(uintptr_t, fs_offsets->nintendo_paths[FS_100_ALBUM_PATH].adrp_offset);
+            write_adrp_add(fs_offsets->nintendo_paths[FS_100_ALBUM_PATH].opcode_reg, fs_adrp_opcode_location, fs_offsets->nintendo_paths[FS_100_ALBUM_PATH].add_rel_offset, nintendo_album_path_location);
         }
+
         // Do contents path
         {
-            int path_len = snprintf(nintendo_path_contents_100, sizeof(nintendo_path_contents_100), "/%s/Contents", nintendo_path);
+            snprintf(nintendo_path_contents_100, sizeof(nintendo_path_contents_100), "/%s/Contents", nintendo_path);
             intptr_t nintendo_contents_path_location = (intptr_t)&nintendo_path_contents_100;
-            intptr_t contents_path_location = nintendo_contents_path_location + path_len - 9; // "/Contents"
-            uintptr_t fs_n_adrp_opcode_location = INJECT_OFFSET(uintptr_t, fs_offsets->nintendo_paths[2].adrp_offset);
-            uintptr_t fs_adrp_opcode_location = INJECT_OFFSET(uintptr_t, fs_offsets->nintendo_paths[3].adrp_offset);
-            write_adrp_add(fs_offsets->nintendo_paths[2].opcode_reg, fs_n_adrp_opcode_location, fs_offsets->nintendo_paths[2].add_rel_offset, nintendo_contents_path_location);
-            write_adrp_add(fs_offsets->nintendo_paths[3].opcode_reg, fs_adrp_opcode_location, fs_offsets->nintendo_paths[3].add_rel_offset, contents_path_location);
+            uintptr_t fs_adrp_opcode_location = INJECT_OFFSET(uintptr_t, fs_offsets->nintendo_paths[FS_100_CONTENTS_PATH].adrp_offset);
+            write_adrp_add(fs_offsets->nintendo_paths[FS_100_CONTENTS_PATH].opcode_reg, fs_adrp_opcode_location, fs_offsets->nintendo_paths[FS_100_CONTENTS_PATH].add_rel_offset, nintendo_contents_path_location);
         }
     }
 }

emummc/source/nx/svc.h

@@ -50,8 +50,18 @@ extern "C" {
  * @return Result code.
  * @note Syscall number 0x55.
  * @warning This is a privileged syscall. Use \ref envIsSyscallHinted to check if it is available.
+ * @warning Only exists on [10.0.0+]. For older versions use \ref svcLegacyQueryIoMapping.
  */
-Result svcQueryIoMapping(u64* virtaddr, u64 physaddr, u64 size);
+Result svcQueryIoMapping(u64* virtaddr, u64* out_size, u64 physaddr, u64 size);
+
+/**
+ * @brief Returns a virtual address mapped to a given IO range.
+ * @return Result code.
+ * @note Syscall number 0x55.
+ * @warning This is a privileged syscall. Use \ref envIsSyscallHinted to check if it is available.
+ * @warning Only exists on [1.0.0-9.2.0]. For newer versions use \ref svcQueryIoMapping.
+ */
+Result svcLegacyQueryIoMapping(u64* virtaddr, u64 physaddr, u64 size);
 
 /**
  * @brief Attaches a device address space to a device.

emummc/source/nx/svc.s

@@ -17,6 +17,15 @@
 .endm
 
 SVC_BEGIN svcQueryIoMapping
+	STP X0, X1, [SP, #-16]!
+	SVC 0x55
+	LDP X3, X4, [SP], #16
+	STR X1, [X3]
+	STR X2, [X4]
+	RET
+SVC_END
+
+SVC_BEGIN svcLegacyQueryIoMapping
 	STR X0, [SP, #-16]!
 	SVC 0x55
 	LDR X2, [SP], #16

emummc/source/utils/fatal.h

@@ -25,11 +25,12 @@ enum FatalReason
     Fatal_InvalidAccessor,
     Fatal_ReadNoAccessor,
     Fatal_WriteNoAccessor,
-    Fatal_IoMapping,
+    Fatal_IoMappingLegacy,
     Fatal_UnknownVersion,
     Fatal_BadResult,
     Fatal_GetConfig,
     Fatal_CloseAccessor,
+    Fatal_IoMapping,
     Fatal_Max
 };
 

emummc/source/utils/util.c

@@ -38,8 +38,15 @@ static inline uintptr_t _GetIoMapping(u64 io_addr, u64 io_size)
     u64 vaddr;
     u64 aligned_addr = (io_addr & ~0xFFFul);
     u64 aligned_size = io_size + (io_addr - aligned_addr);
-    if (svcQueryIoMapping(&vaddr, aligned_addr, aligned_size) != 0) {
-        fatal_abort(Fatal_IoMapping);
+    if (emuMMC_ctx.fs_ver >= FS_VER_10_0_0) {
+        u64 out_size;
+        if (svcQueryIoMapping(&vaddr, &out_size, aligned_addr, aligned_size) != 0) {
+            fatal_abort(Fatal_IoMapping);
+        }
+    } else {
+        if (svcLegacyQueryIoMapping(&vaddr, aligned_addr, aligned_size) != 0) {
+            fatal_abort(Fatal_IoMappingLegacy);
+        }
     }
     return (uintptr_t)(vaddr + (io_addr - aligned_addr));
 }

emummc/source/utils/util.h

@@ -19,6 +19,7 @@
 #define _UTIL_H_
 
 #include "types.h"
+#include "../emuMMC/emummc_ctx.h"
 
 intptr_t QueryIoMapping(u64 addr, u64 size);
 #define byte_swap_32(num) ((num >> 24) & 0xff) | ((num << 8) & 0xff0000) | \
@@ -37,4 +38,6 @@ void usleep(u64 ticks);
 void msleep(u64 milliseconds);
 void exec_cfg(u32 *base, const cfg_op_t *ops, u32 num_ops);
 
+extern volatile emuMMC_ctx_t emuMMC_ctx;
+
 #endif

exosphere/src/fuse.c

@@ -263,19 +263,20 @@ uint32_t fuse_get_5x_key_generation(void) {
 /* Returns the fuse version expected for the firmware. */
 uint32_t fuse_get_expected_fuse_version(uint32_t target_firmware) {
     static const uint8_t expected_versions[ATMOSPHERE_TARGET_FIRMWARE_COUNT+1] = {
-        [ATMOSPHERE_TARGET_FIRMWARE_100] = 1,
-        [ATMOSPHERE_TARGET_FIRMWARE_200] = 2,
-        [ATMOSPHERE_TARGET_FIRMWARE_300] = 3,
-     /* [ATMOSPHERE_TARGET_FIRMWARE_302] = 4, */
-        [ATMOSPHERE_TARGET_FIRMWARE_400] = 5,
-        [ATMOSPHERE_TARGET_FIRMWARE_500] = 6,
-        [ATMOSPHERE_TARGET_FIRMWARE_600] = 7,
-        [ATMOSPHERE_TARGET_FIRMWARE_620] = 8,
-        [ATMOSPHERE_TARGET_FIRMWARE_700] = 9,
-        [ATMOSPHERE_TARGET_FIRMWARE_800] = 9,
-        [ATMOSPHERE_TARGET_FIRMWARE_810] = 10,
-        [ATMOSPHERE_TARGET_FIRMWARE_900] = 11,
-        [ATMOSPHERE_TARGET_FIRMWARE_910] = 12,
+        [ATMOSPHERE_TARGET_FIRMWARE_100]  = 1,
+        [ATMOSPHERE_TARGET_FIRMWARE_200]  = 2,
+        [ATMOSPHERE_TARGET_FIRMWARE_300]  = 3,
+     /* [ATMOSPHERE_TARGET_FIRMWARE_302]  = 4, */
+        [ATMOSPHERE_TARGET_FIRMWARE_400]  = 5,
+        [ATMOSPHERE_TARGET_FIRMWARE_500]  = 6,
+        [ATMOSPHERE_TARGET_FIRMWARE_600]  = 7,
+        [ATMOSPHERE_TARGET_FIRMWARE_620]  = 8,
+        [ATMOSPHERE_TARGET_FIRMWARE_700]  = 9,
+        [ATMOSPHERE_TARGET_FIRMWARE_800]  = 9,
+        [ATMOSPHERE_TARGET_FIRMWARE_810]  = 10,
+        [ATMOSPHERE_TARGET_FIRMWARE_900]  = 11,
+        [ATMOSPHERE_TARGET_FIRMWARE_910]  = 12,
+        [ATMOSPHERE_TARGET_FIRMWARE_1000] = 13,
     };
 
     if (target_firmware > ATMOSPHERE_TARGET_FIRMWARE_COUNT) {

exosphere/src/mc0.h

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef EXOSPHERE_MC0_H
+#define EXOSPHERE_MC0_H
+
+#include <stdint.h>
+#include "memory_map.h"
+
+/* Exosphere driver for the Tegra X1 MC0. */
+
+static inline uintptr_t get_mc0_base(void) {
+    return MMIO_GET_DEVICE_ADDRESS(MMIO_DEVID_MC0);
+}
+
+#define MC0_BASE  (get_mc0_base())
+#define MAKE_MC0_REG(n) MAKE_REG32(MC0_BASE + n)
+
+#endif

exosphere/src/mc1.h

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef EXOSPHERE_MC0_H
+#define EXOSPHERE_MC0_H
+
+#include <stdint.h>
+#include "memory_map.h"
+
+/* Exosphere driver for the Tegra X1 MC1. */
+
+static inline uintptr_t get_mc1_base(void) {
+    return MMIO_GET_DEVICE_ADDRESS(MMIO_DEVID_MC1);
+}
+
+#define MC1_BASE  (get_mc1_base())
+#define MAKE_MC1_REG(n) MAKE_REG32(MC1_BASE + n)
+
+#endif

exosphere/src/memory_map.h

@@ -13,7 +13,7 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
- 
+
 #ifndef EXOSPHERE_MEMORY_MAP_H
 #define EXOSPHERE_MEMORY_MAP_H
 
@@ -48,9 +48,11 @@
 #define _MMAPDEV15      ( 0x6000D000ull, 0x1000ull, true  ) /* GPIO-1 - GPIO-8 */
 #define _MMAPDEV16      ( 0x7000C000ull, 0x1000ull, true  ) /* I2C-I2C4 */
 #define _MMAPDEV17      ( 0x6000F000ull, 0x1000ull, true  ) /* Exception vectors */
-#define _MMAPDEV18      ( 0x00000000ull, 0x1000ull, true  ) /* AMS irampage, NOT mapped at startup */
-#define _MMAPDEV19      ( 0x00000000ull, 0x1000ull, true  ) /* AMS userpage, NOT mapped at startup */
-#define _MMAPDEV20      ( 0x40038000ull, 0x5000ull, true  ) /* DEBUG: IRAM */
+#define _MMAPDEV18      ( 0x7001C000ull, 0x1000ull, true  ) /* MC0 */
+#define _MMAPDEV19      ( 0x7001D000ull, 0x1000ull, true  ) /* MC1 */
+#define _MMAPDEV20      ( 0x00000000ull, 0x1000ull, true  ) /* AMS irampage, NOT mapped at startup */
+#define _MMAPDEV21      ( 0x00000000ull, 0x1000ull, true  ) /* AMS userpage, NOT mapped at startup */
+#define _MMAPDEV22      ( 0x40038000ull, 0x1000ull, true  ) /* DEBUG: IRAM */
 
 /* MMIO 7.0.0+. (addr).  */
 #define _MMAPDEV7X0       ( 0x50041000ull ) /* ARM Interrupt Distributor */
@@ -71,9 +73,11 @@
 #define _MMAPDEV7X15      ( 0x6000D000ull ) /* GPIO-1 - GPIO-8 */
 #define _MMAPDEV7X16      ( 0x7000C000ull ) /* I2C-I2C4 */
 #define _MMAPDEV7X17      ( 0x6000F000ull ) /* Exception vectors */
-#define _MMAPDEV7X18      ( 0x00000000ull ) /* AMS irampage, NOT mapped at startup */
-#define _MMAPDEV7X19      ( 0x00000000ull ) /* AMS userpage, NOT mapped at startup */
-#define _MMAPDEV7X20      ( 0x40038000ull ) /* DEBUG: IRAM */
+#define _MMAPDEV7X18      ( 0x7001C000ull ) /* MC0 */
+#define _MMAPDEV7X19      ( 0x7001D000ull ) /* MC1 */
+#define _MMAPDEV7X20      ( 0x00000000ull ) /* AMS irampage, NOT mapped at startup */
+#define _MMAPDEV7X21      ( 0x00000000ull ) /* AMS userpage, NOT mapped at startup */
+#define _MMAPDEV7X22      ( 0x40038000ull ) /* DEBUG: IRAM */
 
 /* LP0 entry ram segments (addr, size, additional attributes) */
 #define _MMAPLP0ES0  ( 0x40020000ull, 0x10000ull, MMU_PTE_BLOCK_NS | ATTRIB_MEMTYPE_DEVICE ) /* Encrypted TZRAM */
@@ -133,10 +137,12 @@
 #define MMIO_DEVID_GPIO                 15
 #define MMIO_DEVID_DTV_I2C234           16
 #define MMIO_DEVID_EXCEPTION_VECTORS    17
-#define MMIO_DEVID_AMS_IRAM_PAGE        18
-#define MMIO_DEVID_AMS_USER_PAGE        19
-#define MMIO_DEVID_DEBUG_IRAM           20
-#define MMIO_DEVID_MAX                  21
+#define MMIO_DEVID_MC0                  18
+#define MMIO_DEVID_MC1                  19
+#define MMIO_DEVID_AMS_IRAM_PAGE        20
+#define MMIO_DEVID_AMS_USER_PAGE        21
+#define MMIO_DEVID_DEBUG_IRAM           22
+#define MMIO_DEVID_MAX                  23
 
 #define LP0_ENTRY_RAM_SEGMENT_ID_ENCRYPTED_TZRAM    0
 #define LP0_ENTRY_RAM_SEGMENT_ID_LP0_ENTRY_CODE     1

exosphere/src/package2.c

@@ -149,6 +149,7 @@ static void setup_se(void) {
         case ATMOSPHERE_TARGET_FIRMWARE_810:
         case ATMOSPHERE_TARGET_FIRMWARE_900:
         case ATMOSPHERE_TARGET_FIRMWARE_910:
+        case ATMOSPHERE_TARGET_FIRMWARE_1000:
             derive_new_device_keys(KEYSLOT_SWITCH_5XNEWDEVICEKEYGENKEY);
             break;
     }
@@ -338,7 +339,7 @@ static bool validate_package2_metadata(package2_meta_t *metadata) {
 
     /* Perform version checks. */
     /* We will be compatible with all package2s released before current, but not newer ones. */
-    if (metadata->version_max >= PACKAGE2_MINVER_THEORETICAL && metadata->version_min < PACKAGE2_MAXVER_910_CURRENT) {
+    if (metadata->version_max >= PACKAGE2_MINVER_THEORETICAL && metadata->version_min < PACKAGE2_MAXVER_1000_CURRENT) {
         return true;
     }
 
@@ -466,6 +467,7 @@ static void copy_warmboot_bin_to_dram() {
         case ATMOSPHERE_TARGET_FIRMWARE_810:
         case ATMOSPHERE_TARGET_FIRMWARE_900:
         case ATMOSPHERE_TARGET_FIRMWARE_910:
+        case ATMOSPHERE_TARGET_FIRMWARE_1000:
             warmboot_src = (uint8_t *)0x4003E000;
             break;
     }
@@ -551,6 +553,9 @@ void load_package2(coldboot_crt0_reloc_list_t *reloc_list) {
             case ATMOSPHERE_TARGET_FIRMWARE_910:
                 MAKE_REG32(PMC_BASE + 0x360) = 0x18C;
                 break;
+            case ATMOSPHERE_TARGET_FIRMWARE_1000:
+                MAKE_REG32(PMC_BASE + 0x360) = 0x1AD;
+                break;
         }
     }
 

exosphere/src/package2.h

@@ -73,7 +73,8 @@ static inline uintptr_t get_nx_bootloader_mailbox_base(unsigned int targetfw) {
 #define PACKAGE2_MAXVER_700_800 0xA
 #define PACKAGE2_MAXVER_810 0xB
 #define PACKAGE2_MAXVER_900 0xC
-#define PACKAGE2_MAXVER_910_CURRENT 0xD
+#define PACKAGE2_MAXVER_910_920 0xD
+#define PACKAGE2_MAXVER_1000_CURRENT 0xE
 
 #define PACKAGE2_MINVER_100 0x3
 #define PACKAGE2_MINVER_200 0x4
@@ -86,7 +87,8 @@ static inline uintptr_t get_nx_bootloader_mailbox_base(unsigned int targetfw) {
 #define PACKAGE2_MINVER_700_800 0xB
 #define PACKAGE2_MINVER_810 0xC
 #define PACKAGE2_MINVER_900 0xD
-#define PACKAGE2_MINVER_910_CURRENT 0xE
+#define PACKAGE2_MINVER_910_920 0xE
+#define PACKAGE2_MINVER_1000_CURRENT 0xF
 
 typedef struct {
     union {

exosphere/src/rsa_common.c

@@ -0,0 +1,20 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "rsa_common.h"
+
+/* Instantiate the shared RSA data inside a single translation unit. */
+rsa_shared_data_t g_rsa_shared_data = {};

exosphere/src/rsa_common.h

@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef EXOSPHERE_RSA_COMMON_H
+#define EXOSPHERE_RSA_COMMON_H
+#include <stdint.h>
+
+typedef union {
+    struct {
+        uint8_t user_data[0x100];
+    } storage_exp_mod;
+    struct {
+        uint32_t master_key_rev;
+        uint32_t type;
+        uint64_t expected_label_hash[4];
+    } unwrap_titlekey;
+} rsa_shared_data_t __attribute__((aligned(4)));
+
+_Static_assert(sizeof(rsa_shared_data_t) == 0x100);
+
+extern rsa_shared_data_t g_rsa_shared_data;
+
+#endif

exosphere/src/se.c

@@ -13,7 +13,7 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
- 
+
 #include <string.h>
 
 #include "utils.h"
@@ -47,24 +47,25 @@ void ll_init(volatile se_ll_t *ll, void *buffer, size_t size) {
         ll->addr_info.address = 0;
         ll->addr_info.size = 0;
     }
-    
+
     flush_dcache_range((uint8_t *)ll, (uint8_t *)ll + sizeof(*ll));
 }
 
 void set_security_engine_callback(unsigned int (*callback)(void)) {
-    if (callback == NULL || g_se_callback != NULL) {
-        generic_panic();
-    }
-
+    /* Set the callback. */
     g_se_callback = callback;
+
+    /* Enable SE Interrupt firing for async op. */
+    se_get_regs()->SE_INT_ENABLE = 0x10;
 }
 
 /* Fires on Security Engine operation completion. */
 void se_operation_completed(void) {
     se_get_regs()->SE_INT_ENABLE = 0;
-    if (g_se_callback != NULL) {
-        g_se_callback();
+    unsigned int (*callback)(void) = g_se_callback;
+    if (callback != NULL) {
         g_se_callback = NULL;
+        callback();
     }
 }
 
@@ -103,7 +104,7 @@ void se_validate_stored_vector(void) {
 
     uint8_t calc_vector[0x10];
     se_generate_test_vector(calc_vector);
-    
+
     /* Ensure nobody's messed with the security engine while we slept. */
     if (memcmp(calc_vector, g_se_stored_test_vector, 0x10) != 0) {
         generic_panic();
@@ -122,7 +123,7 @@ void se_generate_stored_vector(void) {
 /* Set the flags for an AES keyslot. */
 void set_aes_keyslot_flags(unsigned int keyslot, unsigned int flags) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     if (keyslot >= KEYSLOT_AES_MAX) {
         generic_panic();
     }
@@ -141,7 +142,7 @@ void set_aes_keyslot_flags(unsigned int keyslot, unsigned int flags) {
 /* Set the flags for an RSA keyslot. */
 void set_rsa_keyslot_flags(unsigned int keyslot, unsigned int flags) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     if (keyslot >= KEYSLOT_RSA_MAX) {
         generic_panic();
     }
@@ -160,7 +161,7 @@ void set_rsa_keyslot_flags(unsigned int keyslot, unsigned int flags) {
 
 void clear_aes_keyslot(unsigned int keyslot) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     if (keyslot >= KEYSLOT_AES_MAX) {
         generic_panic();
     }
@@ -174,7 +175,7 @@ void clear_aes_keyslot(unsigned int keyslot) {
 
 void clear_rsa_keyslot(unsigned int keyslot) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     if (keyslot >= KEYSLOT_RSA_MAX) {
         generic_panic();
     }
@@ -194,7 +195,7 @@ void clear_rsa_keyslot(unsigned int keyslot) {
 
 void set_aes_keyslot(unsigned int keyslot, const void *key, size_t key_size) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     if (keyslot >= KEYSLOT_AES_MAX || key_size > KEYSIZE_AES_MAX) {
         generic_panic();
     }
@@ -207,7 +208,7 @@ void set_aes_keyslot(unsigned int keyslot, const void *key, size_t key_size) {
 
 void set_rsa_keyslot(unsigned int keyslot, const void  *modulus, size_t modulus_size, const void *exponent, size_t exp_size) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     if (keyslot >= KEYSLOT_RSA_MAX || modulus_size > KEYSIZE_RSA_MAX || exp_size > KEYSIZE_RSA_MAX) {
         generic_panic();
     }
@@ -228,7 +229,7 @@ void set_rsa_keyslot(unsigned int keyslot, const void  *modulus, size_t modulus_
 
 void set_aes_keyslot_iv(unsigned int keyslot, const void *iv, size_t iv_size) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     if (keyslot >= KEYSLOT_AES_MAX || iv_size > 0x10) {
         generic_panic();
     }
@@ -241,7 +242,7 @@ void set_aes_keyslot_iv(unsigned int keyslot, const void *iv, size_t iv_size) {
 
 void clear_aes_keyslot_iv(unsigned int keyslot) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     if (keyslot >= KEYSLOT_AES_MAX) {
         generic_panic();
     }
@@ -260,7 +261,7 @@ void set_se_ctr(const void *ctr) {
 
 void decrypt_data_into_keyslot(unsigned int keyslot_dst, unsigned int keyslot_src, const void *wrapped_key, size_t wrapped_key_size) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     if (keyslot_dst >= KEYSLOT_AES_MAX || keyslot_src >= KEYSIZE_AES_MAX || wrapped_key_size > KEYSIZE_AES_MAX) {
         generic_panic();
     }
@@ -276,7 +277,7 @@ void decrypt_data_into_keyslot(unsigned int keyslot_dst, unsigned int keyslot_sr
 
 void se_aes_crypt_insecure_internal(unsigned int keyslot, uint32_t out_ll_paddr, uint32_t in_ll_paddr, size_t size, unsigned int crypt_config, bool encrypt, unsigned int (*callback)(void)) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     if (keyslot >= KEYSLOT_AES_MAX) {
         generic_panic();
     }
@@ -304,9 +305,6 @@ void se_aes_crypt_insecure_internal(unsigned int keyslot, uint32_t out_ll_paddr,
     /* Set the callback, for after the async operation. */
     set_security_engine_callback(callback);
 
-    /* Enable SE Interrupt firing for async op. */
-    se->SE_INT_ENABLE = 0x10;
-
     /* Setup Input/Output lists */
     se->SE_IN_LL_ADDR = in_ll_paddr;
     se->SE_OUT_LL_ADDR = out_ll_paddr;
@@ -338,7 +336,7 @@ void se_aes_cbc_decrypt_insecure(unsigned int keyslot, uint32_t out_ll_paddr, ui
     se_aes_crypt_insecure_internal(keyslot, out_ll_paddr, in_ll_paddr, size, 0x66, false, callback);
 }
 
-void se_exp_mod(unsigned int keyslot, void *buf, size_t size, unsigned int (*callback)(void)) {
+void se_exp_mod(unsigned int keyslot, const void *buf, size_t size, unsigned int (*callback)(void)) {
     volatile tegra_se_t *se = se_get_regs();
     uint8_t stack_buf[KEYSIZE_RSA_MAX];
 
@@ -348,7 +346,7 @@ void se_exp_mod(unsigned int keyslot, void *buf, size_t size, unsigned int (*cal
 
     /* Endian swap the input. */
     for (size_t i = 0; i < size; i++) {
-        stack_buf[i] = *((uint8_t *)buf + size - i - 1);
+        stack_buf[i] = *((const uint8_t *)buf + size - i - 1);
     }
 
     se->SE_CONFIG = (ALG_RSA | DST_RSAREG);
@@ -358,9 +356,6 @@ void se_exp_mod(unsigned int keyslot, void *buf, size_t size, unsigned int (*cal
 
     set_security_engine_callback(callback);
 
-    /* Enable SE interrupt firing for async op. */
-    se->SE_INT_ENABLE = 0x10;
-
     flush_dcache_range(stack_buf, stack_buf + KEYSIZE_RSA_MAX);
     trigger_se_rsa_op(stack_buf, size);
 
@@ -468,7 +463,7 @@ bool se_rsa2048_pss_verify(const void *signature, size_t signature_size, const v
 void trigger_se_rsa_op(void *buf, size_t size) {
     volatile tegra_se_t *se = se_get_regs();
     se_ll_t in_ll;
-    
+
     ll_init(&in_ll, (void *)buf, size);
 
     /* Set the input LL. */
@@ -491,19 +486,19 @@ void trigger_se_blocking_op(unsigned int op, void *dst, size_t dst_size, const v
 
     ll_init(&in_ll, (void *)src, src_size);
     ll_init(&out_ll, dst, dst_size);
-    
+
     __dsb_sy();
 
     /* Set the LLs. */
     se->SE_IN_LL_ADDR = (uint32_t) get_physical_address(&in_ll);
     se->SE_OUT_LL_ADDR = (uint32_t) get_physical_address(&out_ll);
-    
+
     /* Set registers for operation. */
     se->SE_ERR_STATUS = se->SE_ERR_STATUS;
     se->SE_INT_STATUS = se->SE_INT_STATUS;
     se->SE_OPERATION = op;
     (void)(se->SE_OPERATION);
-    
+
     __dsb_ish();
 
     while (!(se->SE_INT_STATUS & 0x10)) { /* Wait a while */ }
@@ -538,7 +533,7 @@ void se_perform_aes_block_operation(void *dst, size_t dst_size, const void *src,
 
 void se_aes_ctr_crypt(unsigned int keyslot, void *dst, size_t dst_size, const void *src, size_t src_size, const void *ctr, size_t ctr_size) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     if (keyslot >= KEYSLOT_AES_MAX || ctr_size != 0x10) {
         generic_panic();
     }
@@ -548,7 +543,7 @@ void se_aes_ctr_crypt(unsigned int keyslot, void *dst, size_t dst_size, const vo
     }
     if (dst_size) {
         flush_dcache_range((uint8_t *)dst, (uint8_t *)dst + dst_size);
-    }  
+    }
 
     unsigned int num_blocks = src_size >> 4;
 
@@ -576,12 +571,12 @@ void se_aes_ctr_crypt(unsigned int keyslot, void *dst, size_t dst_size, const vo
 
     if (dst_size) {
         flush_dcache_range((uint8_t *)dst, (uint8_t *)dst + dst_size);
-    }  
+    }
 }
 
 void se_aes_ecb_encrypt_block(unsigned int keyslot, void *dst, size_t dst_size, const void *src, size_t src_size, unsigned int config_high) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     if (keyslot >= KEYSLOT_AES_MAX || dst_size != 0x10 || src_size != 0x10) {
         generic_panic();
     }
@@ -606,7 +601,7 @@ void se_aes_256_ecb_encrypt_block(unsigned int keyslot, void *dst, size_t dst_si
 
 void se_aes_ecb_decrypt_block(unsigned int keyslot, void *dst, size_t dst_size, const void *src, size_t src_size) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     if (keyslot >= KEYSLOT_AES_MAX || dst_size != 0x10 || src_size != 0x10) {
         generic_panic();
     }
@@ -632,15 +627,15 @@ void shift_left_xor_rb(uint8_t *key) {
 
 void se_compute_aes_cmac(unsigned int keyslot, void *cmac, size_t cmac_size, const void *data, size_t data_size, unsigned int config_high) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     if (keyslot >= KEYSLOT_AES_MAX) {
         generic_panic();
     }
-    
+
     if (data_size) {
         flush_dcache_range((uint8_t *)data, (uint8_t *)data + data_size);
     }
-    
+
     /* Generate the derived key, to be XOR'd with final output block. */
     uint8_t derived_key[0x10] = {0};
     se_aes_ecb_encrypt_block(keyslot, derived_key, sizeof(derived_key), derived_key, sizeof(derived_key), config_high);
@@ -652,7 +647,7 @@ void se_compute_aes_cmac(unsigned int keyslot, void *cmac, size_t cmac_size, con
     se->SE_CONFIG = (ALG_AES_ENC | DST_HASHREG) | (config_high << 16);
     se->SE_CRYPTO_CONFIG = (keyslot << 24) | (0x145);
     clear_aes_keyslot_iv(keyslot);
-    
+
     unsigned int num_blocks = (data_size + 0xF) >> 4;
     /* Handle aligned blocks. */
     if (num_blocks > 1) {
@@ -660,7 +655,7 @@ void se_compute_aes_cmac(unsigned int keyslot, void *cmac, size_t cmac_size, con
         trigger_se_blocking_op(OP_START, NULL, 0, data, data_size);
         se->SE_CRYPTO_CONFIG |= 0x80;
     }
-    
+
     /* Create final block. */
     uint8_t last_block[0x10] = {0};
     if (data_size & 0xF) {
@@ -669,11 +664,11 @@ void se_compute_aes_cmac(unsigned int keyslot, void *cmac, size_t cmac_size, con
     } else if (data_size >= 0x10) {
         memcpy(last_block, data + data_size - 0x10, 0x10);
     }
-    
+
     for (unsigned int i = 0; i < 0x10; i++) {
         last_block[i] ^= derived_key[i];
     }
-    
+
     /* Perform last operation. */
     se->SE_CRYPTO_LAST_BLOCK = 0;
     flush_dcache_range(last_block, last_block + sizeof(last_block));
@@ -694,11 +689,11 @@ void se_compute_aes_256_cmac(unsigned int keyslot, void *cmac, size_t cmac_size,
 
 void se_aes_256_cbc_encrypt(unsigned int keyslot, void *dst, size_t dst_size, const void *src, size_t src_size, const void *iv) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     if (keyslot >= KEYSLOT_AES_MAX || src_size < 0x10) {
         generic_panic();
     }
-    
+
     se->SE_CONFIG = (ALG_AES_ENC | DST_MEMORY) | (0x202 << 16);
     se->SE_CRYPTO_CONFIG = (keyslot << 24) | 0x144;
     set_aes_keyslot_iv(keyslot, iv, 0x10);
@@ -709,7 +704,7 @@ void se_aes_256_cbc_encrypt(unsigned int keyslot, void *dst, size_t dst_size, co
 /* SHA256 Implementation. */
 void se_calculate_sha256(void *dst, const void *src, size_t src_size) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     /* Setup config for SHA256, size = BITS(src_size) */
     se->SE_CONFIG = (ENCMODE_SHA256 | ALG_SHA | DST_HASHREG);
     se->SE_SHA_CONFIG = 1;
@@ -721,7 +716,7 @@ void se_calculate_sha256(void *dst, const void *src, size_t src_size) {
     se->SE_SHA_MSG_LEFT[1] = 0;
     se->SE_SHA_MSG_LEFT[2] = 0;
     se->SE_SHA_MSG_LEFT[3] = 0;
-    
+
     /* Trigger the operation. */
     trigger_se_blocking_op(OP_START, NULL, 0, src, src_size);
 
@@ -734,7 +729,7 @@ void se_calculate_sha256(void *dst, const void *src, size_t src_size) {
 /* RNG API */
 void se_initialize_rng(unsigned int keyslot) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     if (keyslot >= KEYSLOT_AES_MAX) {
         generic_panic();
     }
@@ -754,7 +749,7 @@ void se_initialize_rng(unsigned int keyslot) {
 
 void se_generate_random(unsigned int keyslot, void *dst, size_t size) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     if (keyslot >= KEYSLOT_AES_MAX) {
         generic_panic();
     }
@@ -777,7 +772,7 @@ void se_generate_random(unsigned int keyslot, void *dst, size_t size) {
 /* SE context save API. */
 void se_set_in_context_save_mode(bool is_context_save_mode) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     uint32_t val = se->SE_SE_SECURITY;
     if (is_context_save_mode) {
         val |= 0x10000;
@@ -791,7 +786,7 @@ void se_set_in_context_save_mode(bool is_context_save_mode) {
 
 void se_generate_random_key(unsigned int dst_keyslot, unsigned int rng_keyslot) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     if (dst_keyslot >= KEYSLOT_AES_MAX || rng_keyslot >= KEYSLOT_AES_MAX) {
         generic_panic();
     }
@@ -801,7 +796,7 @@ void se_generate_random_key(unsigned int dst_keyslot, unsigned int rng_keyslot)
     se->SE_CRYPTO_CONFIG = (rng_keyslot << 24) | 0x108;
     se->SE_RNG_CONFIG = 4;
     se->SE_CRYPTO_LAST_BLOCK = 0;
-    
+
     /* Generate low part of key. */
     se->SE_CRYPTO_KEYTABLE_DST = (dst_keyslot << 8);
     trigger_se_blocking_op(OP_START, NULL, 0, NULL, 0);
@@ -812,7 +807,7 @@ void se_generate_random_key(unsigned int dst_keyslot, unsigned int rng_keyslot)
 
 void se_generate_srk(unsigned int srkgen_keyslot) {
     volatile tegra_se_t *se = se_get_regs();
-    
+
     se->SE_CONFIG = (ALG_RNG | DST_SRK);
     se->SE_CRYPTO_CONFIG = (srkgen_keyslot << 24) | 0x108;
     se->SE_RNG_CONFIG = 6;
@@ -847,24 +842,24 @@ void se_save_context(unsigned int srkgen_keyslot, unsigned int rng_keyslot, void
     /* Generate the SRK (context save encryption key). */
     se_generate_random_key(srkgen_keyslot, rng_keyslot);
     se_generate_srk(srkgen_keyslot);
-    
+
     flush_dcache_range(work_buf, work_buf + 0x10);
     se_generate_random(rng_keyslot, work_buf, 0x10);
     flush_dcache_range(work_buf, work_buf + 0x10);
-        
+
     /* Save random initial block. */
     se->SE_CONFIG = (ALG_AES_ENC | DST_MEMORY);
     se->SE_CTX_SAVE_CONFIG = (CTX_SAVE_SRC_MEM);
     se->SE_CRYPTO_LAST_BLOCK = 0;
     se_encrypt_with_srk(dst, 0x10, work_buf, 0x10);
-    
+
     /* Save Sticky Bits. */
     for (unsigned int i = 0; i < 0x2; i++) {
         se->SE_CTX_SAVE_CONFIG = (CTX_SAVE_SRC_STICKY_BITS) | (i << CTX_SAVE_STICKY_BIT_INDEX_SHIFT);
         se->SE_CRYPTO_LAST_BLOCK = 0;
         se_encrypt_with_srk(dst + 0x10 + (i * 0x10), 0x10, NULL, 0);
     }
-    
+
     /* Save AES Key Table. */
     for (unsigned int i = 0; i < KEYSLOT_AES_MAX; i++) {
         se->SE_CTX_SAVE_CONFIG = (CTX_SAVE_SRC_KEYTABLE_AES) | (i << CTX_SAVE_KEY_INDEX_SHIFT) | (CTX_SAVE_KEY_LOW_BITS);
@@ -874,21 +869,21 @@ void se_save_context(unsigned int srkgen_keyslot, unsigned int rng_keyslot, void
         se->SE_CRYPTO_LAST_BLOCK = 0;
         se_encrypt_with_srk(dst + 0x40 + (i * 0x20), 0x10, NULL, 0);
     }
-    
+
     /* Save AES Original IVs. */
     for (unsigned int i = 0; i < KEYSLOT_AES_MAX; i++) {
         se->SE_CTX_SAVE_CONFIG = (CTX_SAVE_SRC_KEYTABLE_AES) | (i << CTX_SAVE_KEY_INDEX_SHIFT) | (CTX_SAVE_KEY_ORIGINAL_IV);
         se->SE_CRYPTO_LAST_BLOCK = 0;
         se_encrypt_with_srk(dst + 0x230 + (i * 0x10), 0x10, NULL, 0);
     }
-    
+
     /* Save AES Updated IVs */
     for (unsigned int i = 0; i < KEYSLOT_AES_MAX; i++) {
         se->SE_CTX_SAVE_CONFIG = (CTX_SAVE_SRC_KEYTABLE_AES) | (i << CTX_SAVE_KEY_INDEX_SHIFT) | (CTX_SAVE_KEY_UPDATED_IV);
         se->SE_CRYPTO_LAST_BLOCK = 0;
         se_encrypt_with_srk(dst + 0x330 + (i * 0x10), 0x10, NULL, 0);
     }
-    
+
     /* Save RSA Keytable. */
     uint8_t *rsa_ctx_out = (uint8_t *)dst + 0x430;
     for (unsigned int rsa_key = 0; rsa_key < KEYSLOT_RSA_MAX; rsa_key++) {
@@ -901,13 +896,13 @@ void se_save_context(unsigned int srkgen_keyslot, unsigned int rng_keyslot, void
             }
         }
     }
-    
+
     /* Save "Known Pattern. " */
     static const uint8_t context_save_known_pattern[0x10] = {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f};
     se->SE_CTX_SAVE_CONFIG = (CTX_SAVE_SRC_MEM);
     se->SE_CRYPTO_LAST_BLOCK = 0;
     se_encrypt_with_srk(dst + 0x830, 0x10, context_save_known_pattern, 0x10);
-    
+
     /* Save SRK into PMC registers. */
     se->SE_CTX_SAVE_CONFIG = (CTX_SAVE_SRC_SRK);
     se->SE_CRYPTO_LAST_BLOCK = 0;

exosphere/src/se.h

@@ -13,7 +13,7 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
- 
+
 #ifndef EXOSPHERE_SE_H
 #define EXOSPHERE_SE_H
 
@@ -213,7 +213,7 @@ void se_aes_256_cbc_encrypt(unsigned int keyslot, void *dst, size_t dst_size, co
 void se_calculate_sha256(void *dst, const void *src, size_t src_size);
 
 /* RSA API */
-void se_exp_mod(unsigned int keyslot, void *buf, size_t size, unsigned int (*callback)(void));
+void se_exp_mod(unsigned int keyslot, const void *buf, size_t size, unsigned int (*callback)(void));
 void se_get_exp_mod_output(void *buf, size_t size);
 void se_synchronous_exp_mod(unsigned int keyslot, void *dst, size_t dst_size, const void *src, size_t src_size);
 bool se_rsa2048_pss_verify(const void *signature, size_t signature_size, const void *modulus, size_t modulus_size, const void *data, size_t data_size);

exosphere/src/smc_api.c

@@ -25,6 +25,8 @@
 #include "synchronization.h"
 #include "masterkey.h"
 #include "mc.h"
+#include "mc0.h"
+#include "mc1.h"
 #include "memory_map.h"
 #include "pmc.h"
 #include "randomcache.h"
@@ -188,6 +190,7 @@ void set_version_specific_smcs(void) {
         case ATMOSPHERE_TARGET_FIRMWARE_810:
         case ATMOSPHERE_TARGET_FIRMWARE_900:
         case ATMOSPHERE_TARGET_FIRMWARE_910:
+        case ATMOSPHERE_TARGET_FIRMWARE_1000:
             /* No more LoadSecureExpModKey. */
             g_smc_user_table[0xE].handler = NULL;
             g_smc_user_table[0xC].id = 0xC300D60C;
@@ -433,19 +436,18 @@ uint32_t smc_get_result(smc_args_t *args) {
 }
 
 uint32_t smc_exp_mod_get_result(void *buf, uint64_t size) {
-    if (get_exp_mod_done() != 1) {
-        return 3;
+    uint32_t res = get_exp_mod_result();
+    if (res == 0) {
+        if (size == 0x100) {
+            se_get_exp_mod_output(buf, 0x100);
+            /* smc_exp_mod is done now. */
+            clear_user_smc_in_progress();
+            res = 0;
+        } else {
+            res = 2;
+        }
     }
-
-    if (size != 0x100) {
-        return 2;
-    }
-
-    se_get_exp_mod_output(buf, 0x100);
-
-    /* smc_exp_mod is done now. */
-    clear_user_smc_in_progress();
-    return 0;
+    return res;
 }
 
 uint32_t smc_exp_mod(smc_args_t *args) {
@@ -508,30 +510,31 @@ uint32_t smc_unwrap_rsa_oaep_wrapped_titlekey_get_result(void *buf, uint64_t siz
     uint8_t aes_wrapped_titlekey[0x10];
     uint8_t titlekey[0x10];
     uint64_t sealed_titlekey[2];
-    if (get_exp_mod_done() != 1) {
-        return 3;
+    uint32_t res = get_exp_mod_result();
+    if (res == 0) {
+        if (size == 0x10) {
+            se_get_exp_mod_output(rsa_wrapped_titlekey, 0x100);
+            if (tkey_rsa_oaep_unwrap(aes_wrapped_titlekey, 0x10, rsa_wrapped_titlekey, 0x100) == 0x10) {
+                tkey_aes_unwrap(titlekey, 0x10, aes_wrapped_titlekey, 0x10);
+                seal_titlekey(sealed_titlekey, 0x10, titlekey, 0x10);
+
+                p_sealed_key[0] = sealed_titlekey[0];
+                p_sealed_key[1] = sealed_titlekey[1];
+
+                res = 0;
+            } else {
+                /* Failed to extract RSA OAEP wrapped key. */
+                res = 2;
+            }
+
+            /* smc_unwrap_rsa_oaep_wrapped_titlekey is done now. */
+            clear_user_smc_in_progress();
+        } else {
+            res = 2;
+        }
     }
 
-    if (size != 0x10) {
-        return 2;
-    }
-
-    se_get_exp_mod_output(rsa_wrapped_titlekey, 0x100);
-    if (tkey_rsa_oaep_unwrap(aes_wrapped_titlekey, 0x10, rsa_wrapped_titlekey, 0x100) != 0x10) {
-        /* Failed to extract RSA OAEP wrapped key. */
-        clear_user_smc_in_progress();
-        return 2;
-    }
-
-    tkey_aes_unwrap(titlekey, 0x10, aes_wrapped_titlekey, 0x10);
-    seal_titlekey(sealed_titlekey, 0x10, titlekey, 0x10);
-
-    p_sealed_key[0] = sealed_titlekey[0];
-    p_sealed_key[1] = sealed_titlekey[1];
-
-    /* smc_unwrap_rsa_oaep_wrapped_titlekey is done now. */
-    clear_user_smc_in_progress();
-    return 0;
+    return res;
 }
 
 uint32_t smc_unwrap_rsa_oaep_wrapped_titlekey(smc_args_t *args) {
@@ -616,7 +619,7 @@ uint32_t smc_read_write_register(smc_args_t *args) {
     }
     /* Check for PMC registers. */
     if (0x7000E400 <= address && address <= 0x7000EFFF) {
-        const uint8_t pmc_whitelist[0x28] = {
+        static const uint8_t pmc_whitelist[0x28] = {
             0xB9, 0xF9, 0x07, 0x00, 0x00, 0x00, 0x80, 0x03,
             0x00, 0x00, 0x00, 0x17, 0x00, 0xC4, 0x07, 0x00,
             0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x20, 0x00,
@@ -632,39 +635,83 @@ uint32_t smc_read_write_register(smc_args_t *args) {
         } else {
             return 2;
         }
-    } else if (exosphere_get_target_firmware() >= ATMOSPHERE_TARGET_FIRMWARE_400 && MMIO_GET_DEVICE_PA(MMIO_DEVID_MC) <= address &&
-               address < MMIO_GET_DEVICE_PA(MMIO_DEVID_MC) + MMIO_GET_DEVICE_SIZE(MMIO_DEVID_MC)) {
-        /* Memory Controller RW supported only on 4.0.0+ */
-        const uint8_t mc_whitelist[0x68] = {
-            0x9F, 0x31, 0x30, 0x00, 0xF0, 0xFF, 0xF7, 0x01,
-            0xCD, 0xFE, 0xC0, 0xFE, 0x00, 0x00, 0x00, 0x00,
-            0x03, 0x40, 0x73, 0x3E, 0x2F, 0x00, 0x00, 0x6E,
-            0x30, 0x05, 0x06, 0xB0, 0x71, 0xC8, 0x43, 0x04,
-            0x80, 0x1F, 0x08, 0x80, 0x03, 0x00, 0x0E, 0x00,
-            0x08, 0x00, 0xE0, 0x00, 0x0E, 0x00, 0x00, 0x00,
-            0x00, 0x00, 0x00, 0x30, 0xF0, 0x03, 0x03, 0x30,
-            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-            0x00, 0x00, 0x00, 0x31, 0x00, 0x40, 0x00, 0x00,
-            0x00, 0x03, 0x00, 0x00, 0xE4, 0xFF, 0xFF, 0x01,
-            0x00, 0x00, 0x00, 0x00, 0x0C, 0x00, 0xFE, 0x0F,
-            0x01, 0x00, 0x80, 0x00, 0x00, 0x08, 0x00, 0x00
-        };
-        uint32_t offset = (uint32_t)(address - 0x70019000);
-        uint32_t wl_ind = (offset >> 5);
-        /* If address is whitelisted, allow write. */
-        if (wl_ind < sizeof(mc_whitelist) && (mc_whitelist[wl_ind] & (1 << ((offset >> 2) & 0x7)))) {
-            p_mmio = (volatile uint32_t *)(MMIO_GET_DEVICE_ADDRESS(MMIO_DEVID_MC) + offset);
-        } else {
-            /* These addresses are not allowed by the whitelist. */
-            /* They correspond to SMMU DISABLE for the BPMP, and for APB-DMA. */
-            /* However, smcReadWriteRegister returns 0 for these addresses despite not actually performing the write. */
-            /* This is "probably" to fuck with hackers who got access to smcReadWriteRegister and are trying to get */
-            /* control of the BPMP for jamais vu etc., since there's no other reason to return 0 despite failure. */
-            if (address == 0x7001923C || address == 0x70019298) {
-                return 0;
+    } else {
+        if (exosphere_get_target_firmware() >= ATMOSPHERE_TARGET_FIRMWARE_500) {
+            static const uint8_t mc_whitelist_5x[0xD00/(sizeof(uint32_t) * 8)] = {
+                0x9F, 0x31, 0x30, 0x00, 0xF0, 0xFF, 0xF7, 0x01,
+                0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+                0x03, 0x40, 0x73, 0x3E, 0x2F, 0x00, 0x00, 0x6E,
+                0x30, 0x05, 0x06, 0xB0, 0x71, 0xC8, 0x43, 0x04,
+                0x80, 0xFF, 0x08, 0x80, 0x03, 0x38, 0x8E, 0x1F,
+                0xC8, 0xFF, 0xFF, 0x00, 0x0E, 0x00, 0x00, 0x00,
+                0xF0, 0x1F, 0x00, 0x30, 0xF0, 0x03, 0x03, 0x30,
+                0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+                0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+                0x00, 0x00, 0x00, 0x31, 0x00, 0x40, 0x00, 0x00,
+                0x00, 0x00, 0x00, 0x00, 0xE4, 0xFF, 0xFF, 0x01,
+                0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF8, 0x0F,
+                0x01, 0x00, 0x80, 0x00, 0x00, 0x08, 0x00, 0x00
+            };
+            static const uint8_t mc01_whitelist_5x[0xC00/(sizeof(uint32_t) * 8)] = {
+                0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+                0xCD, 0xFE, 0xC0, 0xFE, 0x00, 0x00, 0x00, 0x00,
+                0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+                0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+                0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+                0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+                0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+                0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+                0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+                0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+                0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+                0x00, 0x00, 0x00, 0x00, 0x0C, 0x00, 0x06, 0x00,
+            };
+            static const struct {
+                uint32_t phys_addr;
+                uint32_t size;
+                uint64_t virt_addr;
+                const uint8_t *whitelist;
+            } register_whitelists[3] = {
+                { MMIO_GET_DEVICE_PA(MMIO_DEVID_MC),  sizeof(mc_whitelist_5x)   * (sizeof(uint32_t) * 8), MMIO_GET_DEVICE_ADDRESS(MMIO_DEVID_MC),    mc_whitelist_5x },
+                { MMIO_GET_DEVICE_PA(MMIO_DEVID_MC0), sizeof(mc01_whitelist_5x) * (sizeof(uint32_t) * 8), MMIO_GET_DEVICE_ADDRESS(MMIO_DEVID_MC0), mc01_whitelist_5x },
+                { MMIO_GET_DEVICE_PA(MMIO_DEVID_MC1), sizeof(mc01_whitelist_5x) * (sizeof(uint32_t) * 8), MMIO_GET_DEVICE_ADDRESS(MMIO_DEVID_MC1), mc01_whitelist_5x },
+            };
+            for (unsigned int which = 0; which < 3; which++) {
+                if (register_whitelists[which].phys_addr <= address && address < register_whitelists[which].phys_addr + register_whitelists[which].size) {
+                    uint32_t offset = (uint32_t)(address - register_whitelists[which].phys_addr);
+                    uint32_t wl_ind = (offset >> 5);
+                    /* If address is whitelisted, allow write. */
+                    if (register_whitelists[which].whitelist[wl_ind] & (1 << ((offset >> 2) & 0x7))) {
+                        p_mmio = (volatile uint32_t *)(register_whitelists[which].virt_addr + offset);
+                    }
+                    break;
+                }
+            }
+        } else if (exosphere_get_target_firmware() >= ATMOSPHERE_TARGET_FIRMWARE_400) {
+            if (MMIO_GET_DEVICE_PA(MMIO_DEVID_MC) <= address && address < MMIO_GET_DEVICE_PA(MMIO_DEVID_MC) + 0xD00) {
+                /* Memory Controller RW supported only on 4.0.0+ */
+                static const uint8_t mc_whitelist[0x68] = {
+                    0x9F, 0x31, 0x30, 0x00, 0xF0, 0xFF, 0xF7, 0x01,
+                    0xCD, 0xFE, 0xC0, 0xFE, 0x00, 0x00, 0x00, 0x00,
+                    0x03, 0x40, 0x73, 0x3E, 0x2F, 0x00, 0x00, 0x6E,
+                    0x30, 0x05, 0x06, 0xB0, 0x71, 0xC8, 0x43, 0x04,
+                    0x80, 0x1F, 0x08, 0x80, 0x03, 0x00, 0x0E, 0x00,
+                    0x08, 0x00, 0xE0, 0x00, 0x0E, 0x00, 0x00, 0x00,
+                    0x00, 0x00, 0x00, 0x30, 0xF0, 0x03, 0x03, 0x30,
+                    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+                    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+                    0x00, 0x00, 0x00, 0x31, 0x00, 0x40, 0x00, 0x00,
+                    0x00, 0x03, 0x00, 0x00, 0xE4, 0xFF, 0xFF, 0x01,
+                    0x00, 0x00, 0x00, 0x00, 0x0C, 0x00, 0xFE, 0x0F,
+                    0x01, 0x00, 0x80, 0x00, 0x00, 0x08, 0x00, 0x00
+                };
+                uint32_t offset = (uint32_t)(address - MMIO_GET_DEVICE_PA(MMIO_DEVID_MC));
+                uint32_t wl_ind = (offset >> 5);
+                /* If address is whitelisted, allow write. */
+                if (mc_whitelist[wl_ind] & (1 << ((offset >> 2) & 0x7))) {
+                    p_mmio = (volatile uint32_t *)(MMIO_GET_DEVICE_ADDRESS(MMIO_DEVID_MC) + offset);
+                }
             }
-            return 2;
         }
     }
 
@@ -683,9 +730,16 @@ uint32_t smc_read_write_register(smc_args_t *args) {
         /* Return old value. */
         args->X[1] = old_value;
         return 0;
+    } else if (exosphere_get_target_firmware() >= ATMOSPHERE_TARGET_FIRMWARE_400 && (address == 0x7001923C || address == 0x70019298)) {
+        /* These addresses are not allowed by the whitelist. */
+        /* They correspond to SMMU DISABLE for the BPMP, and for APB-DMA. */
+        /* However, smcReadWriteRegister returns 0 for these addresses despite not actually performing the write. */
+        /* This is "probably" to fuck with hackers who got access to smcReadWriteRegister and are trying to get */
+        /* control of the BPMP for jamais vu etc., since there's no other reason to return 0 despite failure. */
+        return 0;
+    } else {
+        return 2;
     }
-
-    return 2;
 }
 
 

exosphere/src/smc_user.c

@@ -34,12 +34,93 @@
 
 /* Globals. */
 static bool g_crypt_aes_done = false;
-static bool g_exp_mod_done = false;
+static uint32_t g_exp_mod_result = 0;
 
-static uint8_t g_imported_exponents[4][0x100];
+static __attribute__((aligned(4))) uint8_t g_imported_exponents[4][0x100];
+static __attribute__((aligned(4))) uint8_t g_imported_moduli[4][0x100];
+static bool g_is_modulus_verified[4];
+
+static __attribute__((aligned(4))) const uint8_t g_rsa_public_key[4] = { 0x00, 0x01, 0x00, 0x01 };
+
+static __attribute__((aligned(4))) const uint8_t g_rsa_test_vector[0x100] = {
+    'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D',
+    'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D',
+    'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D',
+    'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D',
+    'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D',
+    'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D',
+    'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D',
+    'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D',
+    'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D',
+    'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D',
+    'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D',
+    'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D',
+    'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D',
+    'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D',
+    'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D',
+    'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D', 'D'
+};
+
+static uint32_t g_test_exp_mod_keyslot = 0;
+static uint32_t g_test_exp_mod_usecase = 0;
+static bool g_test_exp_mod_in_progress = false;
 
 static uint8_t g_rsausecase_to_cryptousecase[5] = {1, 2, 3, 5, 6};
 
+static void import_rsa_exponent(unsigned int which, const uint8_t *exponent, uint64_t size) {
+    g_is_modulus_verified[which] = false;
+    for (unsigned int i = 0; i < 0x100; i++) {
+        g_imported_exponents[which][i] = exponent[i];
+        g_imported_moduli[which][i]    = 0;
+    }
+}
+
+static void import_rsa_modulus(unsigned int which, const uint8_t *modulus, uint64_t size) {
+    uint64_t clamped_size = 0x100;
+    if (size <= 0x100) {
+        clamped_size = size;
+    }
+    if (clamped_size != 0) {
+        /* The official secure monitor implements this via bit-fiddling, */
+        /* and to prevent accidental inaccuracy we will too. */
+        /* It's probably done to prevent errors on negative sizes. */
+        uint64_t remaining = 0x100;
+        if (size != 0x100 && (~size >= ~0xFFFFFFFFFFFFFEFFULL)) {
+            remaining = size;
+        }
+        memcpy(&g_imported_moduli[which][0], modulus, remaining);
+    }
+}
+
+static bool load_imported_rsa_keypair(unsigned int keyslot, unsigned int which) {
+    if (!g_is_modulus_verified[which]) {
+        return false;
+    }
+    set_rsa_keyslot(keyslot, g_imported_moduli[which], 0x100, g_imported_exponents[which], 0x100);
+    return true;
+}
+
+static void test_rsa_modulus_public(unsigned int which, unsigned int keyslot, const uint8_t *modulus, uint64_t modulus_size, unsigned int (*callback)(void)) {
+    import_rsa_modulus(which, modulus, modulus_size);
+    set_rsa_keyslot(keyslot, modulus, modulus_size, g_rsa_public_key, 0x4);
+    se_exp_mod(keyslot, g_rsa_test_vector, 0x100, callback);
+}
+
+static void test_rsa_modulus_private(unsigned int which, unsigned int keyslot, unsigned int (*callback)(void)) {
+    uint8_t exponentiated_data[0x100];
+    se_get_exp_mod_output(exponentiated_data, sizeof(exponentiated_data));
+    set_rsa_keyslot(keyslot, g_imported_moduli[which], 0x100, g_imported_exponents[which], 0x100);
+    se_exp_mod(keyslot, exponentiated_data, 0x100, callback);
+}
+
+static void validate_rsa_result(unsigned int which) {
+    char result[0x100];
+    se_get_exp_mod_output(result, sizeof(result));
+    if (memcmp(result, g_rsa_test_vector, sizeof(result)) == 0) {
+        g_is_modulus_verified[which] = true;
+    }
+}
+
 static bool is_user_keyslot_valid(unsigned int keyslot) {
     switch (exosphere_get_target_firmware()) {
         case ATMOSPHERE_TARGET_FIRMWARE_100:
@@ -55,27 +136,45 @@ static bool is_user_keyslot_valid(unsigned int keyslot) {
         case ATMOSPHERE_TARGET_FIRMWARE_810:
         case ATMOSPHERE_TARGET_FIRMWARE_900:
         case ATMOSPHERE_TARGET_FIRMWARE_910:
+        case ATMOSPHERE_TARGET_FIRMWARE_1000:
         default:
             return keyslot <= 5;
     }
 }
 
-void set_exp_mod_done(bool done) {
-    g_exp_mod_done = done;
+void set_exp_mod_result(uint32_t result) {
+    g_exp_mod_result = result;
 }
 
-bool get_exp_mod_done(void) {
-    return g_exp_mod_done;
+uint32_t get_exp_mod_result(void) {
+    return g_exp_mod_result;
 }
 
 uint32_t exp_mod_done_handler(void) {
-    set_exp_mod_done(true);
+    set_exp_mod_result(0);
 
     se_trigger_interrupt();
 
     return 0;
 }
 
+static uint32_t test_exp_mod_done_handler(void) {
+    if (g_test_exp_mod_in_progress) {
+        g_test_exp_mod_in_progress = false;
+        test_rsa_modulus_private(g_test_exp_mod_usecase, g_test_exp_mod_keyslot, test_exp_mod_done_handler);
+    } else {
+        validate_rsa_result(g_test_exp_mod_usecase);
+        if (load_imported_rsa_keypair(g_test_exp_mod_keyslot, g_test_exp_mod_usecase)) {
+            se_exp_mod(g_test_exp_mod_keyslot, g_rsa_shared_data.storage_exp_mod.user_data, 0x100, exp_mod_done_handler);
+        } else {
+            set_exp_mod_result(2);
+            se_trigger_interrupt();
+        }
+    }
+
+    return 0;
+}
+
 uint32_t user_exp_mod(smc_args_t *args) {
     uint8_t modulus[0x100];
     uint8_t exponent[0x100];
@@ -108,7 +207,8 @@ uint32_t user_exp_mod(smc_args_t *args) {
         return 2;
     }
 
-    set_exp_mod_done(false);
+    set_exp_mod_result(3);
+
     /* Hardcode RSA keyslot 0. */
     set_rsa_keyslot(0, modulus, 0x100, exponent, exponent_size);
     se_exp_mod(0, input, 0x100, exp_mod_done_handler);
@@ -650,10 +750,21 @@ uint32_t user_secure_exp_mod(smc_args_t *args) {
         return 2;
     }
 
-    set_exp_mod_done(false);
+    set_exp_mod_result(3);
+
     /* Hardcode RSA keyslot 0. */
-    set_rsa_keyslot(0, modulus, 0x100, g_imported_exponents[exponent_id], 0x100);
-    se_exp_mod(0, input, 0x100, exp_mod_done_handler);
+    if (exosphere_get_target_firmware() < ATMOSPHERE_TARGET_FIRMWARE_1000) {
+        set_rsa_keyslot(0, modulus, 0x100, g_imported_exponents[exponent_id], 0x100);
+        se_exp_mod(0, input, 0x100, exp_mod_done_handler);
+    } else if (load_imported_rsa_keypair(0, exponent_id)) {
+        se_exp_mod(0, input, 0x100, exp_mod_done_handler);
+    } else {
+        memcpy(g_rsa_shared_data.storage_exp_mod.user_data, input, 0x100);
+        g_test_exp_mod_keyslot     = 0;
+        g_test_exp_mod_usecase     = exponent_id;
+        g_test_exp_mod_in_progress = true;
+        test_rsa_modulus_public(exponent_id, 0, modulus, 0x100, test_exp_mod_done_handler);
+    }
 
     return 0;
 }
@@ -700,7 +811,7 @@ uint32_t user_unwrap_rsa_oaep_wrapped_titlekey(smc_args_t *args) {
         return 2;
     }
 
-    set_exp_mod_done(false);
+    set_exp_mod_result(3);
 
     /* Expected label_hash occupies args->X[3] to args->X[6]. */
     tkey_set_expected_label_hash(&args->X[3]);
@@ -879,6 +990,7 @@ uint32_t user_decrypt_or_import_rsa_key(smc_args_t *args) {
     }
 
     unsigned int exponent_id;
+    bool import_modulus;
 
     switch (usecase) {
         case 0:
@@ -888,22 +1000,33 @@ uint32_t user_decrypt_or_import_rsa_key(smc_args_t *args) {
             return 0;
         case 1:
             exponent_id = 1;
+            import_modulus = false;
             break;
         case 2:
             exponent_id = 0;
+            import_modulus = true;
             break;
         case 3:
             exponent_id = 2;
+            import_modulus = false;
             break;
         case 4:
             exponent_id = 3;
+            import_modulus = true;
             break;
         default:
             generic_panic();
     }
 
-    /* Copy key to global. */
-    memcpy(g_imported_exponents[exponent_id], user_data, 0x100);
+    /* Modulus import isn't implemented on < 10.0.0. */
+    import_modulus &= (exosphere_get_target_firmware() >= ATMOSPHERE_TARGET_FIRMWARE_1000);
+
+    /* Import the key. */
+    import_rsa_exponent(exponent_id, user_data, 0x100);
+    if (import_modulus) {
+        import_rsa_modulus(exponent_id, user_data + 0x100, 0x100);
+        g_is_modulus_verified[exponent_id] = true;
+    }
     return 0;
 
 }

exosphere/src/smc_user.h

@@ -13,7 +13,7 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
- 
+
 #ifndef EXOSPHERE_SMC_USER_H
 #define EXOSPHERE_SMC_USER_H
 
@@ -41,7 +41,7 @@ uint32_t user_decrypt_or_import_rsa_key(smc_args_t *args);
 void set_crypt_aes_done(bool done);
 bool get_crypt_aes_done(void);
 
-void set_exp_mod_done(bool done);
-bool get_exp_mod_done(void);
+void set_exp_mod_result(uint32_t result);
+uint32_t get_exp_mod_result(void);
 
 #endif

exosphere/src/titlekey.c

@@ -13,7 +13,7 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
- 
+
 #include <stdint.h>
 #include <string.h>
 
@@ -25,14 +25,10 @@
 #include "masterkey.h"
 #include "se.h"
 
-static uint64_t g_tkey_expected_label_hash[4];
-static unsigned int g_tkey_master_key_rev = MASTERKEY_REVISION_MAX;
-static unsigned int g_tkey_type = 0;
-
 /* Set the expected db prefix. */
 void tkey_set_expected_label_hash(uint64_t *label_hash) {
     for (unsigned int i = 0; i < 4; i++) {
-        g_tkey_expected_label_hash[i] = label_hash[i];
+        g_rsa_shared_data.unwrap_titlekey.expected_label_hash[i] = label_hash[i];
     }
 }
 
@@ -40,7 +36,7 @@ void tkey_set_master_key_rev(unsigned int master_key_rev) {
     if (master_key_rev >= MASTERKEY_REVISION_MAX) {
         generic_panic();
     }
-    g_tkey_master_key_rev = master_key_rev;
+    g_rsa_shared_data.unwrap_titlekey.master_key_rev = master_key_rev;
 }
 
 static void tkey_validate_type(unsigned int type) {
@@ -51,7 +47,7 @@ static void tkey_validate_type(unsigned int type) {
 
 void tkey_set_type(unsigned int type) {
     tkey_validate_type(type);
-    g_tkey_type = type;
+    g_rsa_shared_data.unwrap_titlekey.type = type;
 }
 
 /* Reference for MGF1 can be found here: https://en.wikipedia.org/wiki/Mask_generation_function#MGF1 */
@@ -116,7 +112,7 @@ size_t tkey_rsa_oaep_unwrap(void *dst, size_t dst_size, void *src, size_t src_si
     uint8_t *db = message + 0x21;
 
     /* This will be passed to smc_unwrap_rsa_oaep_wrapped_titlekey. */
-    uint8_t *expected_label_hash = (uint8_t *)(&g_tkey_expected_label_hash[0]);
+    uint8_t *expected_label_hash = (uint8_t *)(&g_rsa_shared_data.unwrap_titlekey.expected_label_hash[0]);
 
     /* Unmask the salt. */
     calculate_mgf1_and_xor(salt, 0x20, db, 0xDF);
@@ -171,13 +167,13 @@ static const uint8_t titlekek_sources[TITLEKEY_TYPE_MAX+1][0x10] = {
 };
 
 void tkey_aes_unwrap(void *dst, size_t dst_size, const void *src, size_t src_size) {
-    if (g_tkey_master_key_rev >= MASTERKEY_REVISION_MAX || dst_size != 0x10 || src_size != 0x10) {
+    if (g_rsa_shared_data.unwrap_titlekey.master_key_rev >= MASTERKEY_REVISION_MAX || dst_size != 0x10 || src_size != 0x10) {
         generic_panic();
     }
-    
+
     /* Generate the appropriate titlekek into keyslot 9. */
-    unsigned int master_keyslot = mkey_get_keyslot(g_tkey_master_key_rev);
-    decrypt_data_into_keyslot(KEYSLOT_SWITCH_TEMPKEY, master_keyslot, titlekek_sources[g_tkey_type], 0x10);
+    unsigned int master_keyslot = mkey_get_keyslot(g_rsa_shared_data.unwrap_titlekey.master_key_rev);
+    decrypt_data_into_keyslot(KEYSLOT_SWITCH_TEMPKEY, master_keyslot, titlekek_sources[g_rsa_shared_data.unwrap_titlekey.type], 0x10);
 
     /* Unwrap the titlekey using the titlekek. */
     se_aes_ecb_decrypt_block(KEYSLOT_SWITCH_TEMPKEY, dst, 0x10, src, 0x10);

exosphere/src/titlekey.h

@@ -13,11 +13,12 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
- 
+
 #ifndef EXOSPHERE_TITLEKEY_H
 #define EXOSPHERE_TITLEKEY_H
 
 #include <stdint.h>
+#include "rsa_common.h"
 
 #define TITLEKEY_TYPE_MAX 0x1
 

exosphere/src/uart.c

@@ -21,12 +21,12 @@
 
 static inline void uart_wait_cycles(uint32_t baud, uint32_t num)
 {
-    wait((num * 1000000 + 16 * baud - 1) / (16 * baud));
+    udelay((num * 1000000 + 16 * baud - 1) / (16 * baud));
 }
 
 static inline void uart_wait_syms(uint32_t baud, uint32_t num)
 {
-    wait((num * 1000000 + baud - 1) / baud);
+    udelay((num * 1000000 + baud - 1) / baud);
 }
 
 void uart_config(UartDevice dev) {
@@ -34,28 +34,28 @@ void uart_config(UartDevice dev) {
 
     switch (dev) {
         case UART_A:
-            pinmux->uart1_tx = 0;
-            pinmux->uart1_rx = (PINMUX_INPUT | PINMUX_PULL_UP);
-            pinmux->uart1_rts = 0;
-            pinmux->uart1_cts = (PINMUX_INPUT | PINMUX_PULL_DOWN);
+            pinmux->uart1_tx = (0 | 0 | PINMUX_PULL_NONE | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart1_rx = (PINMUX_INPUT | PINMUX_TRISTATE | PINMUX_PULL_UP | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart1_rts = (0 | 0 | PINMUX_PULL_NONE | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart1_cts = (PINMUX_INPUT | PINMUX_TRISTATE | PINMUX_PULL_DOWN | PINMUX_SELECT_FUNCTION0);
             break;
         case UART_B:
-            pinmux->uart2_tx = 0;
-            pinmux->uart2_rx = (PINMUX_INPUT | PINMUX_PULL_UP);
-            pinmux->uart2_rts = 0;
-            pinmux->uart2_cts = (PINMUX_INPUT | PINMUX_PULL_DOWN);
+            pinmux->uart2_tx = (0 | 0 | PINMUX_PULL_NONE | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart2_rx = (PINMUX_INPUT | PINMUX_TRISTATE | PINMUX_PULL_NONE | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart2_rts = (0 | 0 | PINMUX_PULL_DOWN | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart2_cts = (PINMUX_INPUT | PINMUX_TRISTATE | PINMUX_PULL_NONE | PINMUX_SELECT_FUNCTION0);
             break;
         case UART_C:
-            pinmux->uart3_tx = 0;
-            pinmux->uart3_rx = (PINMUX_INPUT | PINMUX_PULL_UP);
-            pinmux->uart3_rts = 0;
-            pinmux->uart3_cts = (PINMUX_INPUT | PINMUX_PULL_DOWN);
+            pinmux->uart3_tx = (0 | 0 | PINMUX_PULL_NONE | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart3_rx = (PINMUX_INPUT | PINMUX_TRISTATE | PINMUX_PULL_NONE | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart3_rts = (0 | 0 | PINMUX_PULL_DOWN | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart3_cts = (PINMUX_INPUT | PINMUX_TRISTATE | PINMUX_PULL_NONE | PINMUX_SELECT_FUNCTION0);
             break;
         case UART_D:
-            pinmux->uart4_tx = 0;
-            pinmux->uart4_rx = (PINMUX_INPUT | PINMUX_PULL_UP);
-            pinmux->uart4_rts = 0;
-            pinmux->uart4_cts = (PINMUX_INPUT | PINMUX_PULL_DOWN);
+            pinmux->uart4_tx = (0 | 0 | PINMUX_PULL_DOWN | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart4_rx = (PINMUX_INPUT | PINMUX_TRISTATE | PINMUX_PULL_DOWN | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart4_rts = (0 | 0 | PINMUX_PULL_DOWN | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart4_cts = (PINMUX_INPUT | PINMUX_TRISTATE | PINMUX_PULL_DOWN | PINMUX_SELECT_FUNCTION0);
             break;
         case UART_E:
             /* Unused. */

exosphere/src/warmboot_init.c

@@ -13,7 +13,7 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
- 
+
 #include "utils.h"
 #include "memory_map.h"
 #include "mc.h"
@@ -65,7 +65,7 @@ void init_dma_controllers(unsigned int target_firmware) {
         MAKE_REG32(0x6000C038) = 0x0;
 
         /* MSELECT_CONFIG_0 |= WRAP_TO_INCR_SLAVE0(APC) | WRAP_TO_INCR_SLAVE1(PCIe) | WRAP_TO_INCR_SLAVE2(GPU) */
-        MAKE_REG32(0x50060000) |= 0x38000000;
+        MAKE_REG32(0x50060000) = (MAKE_REG32(0x50060000) & 0xC4FFFFFF) | 0x38000000;
 
         /* AHB_ARBITRATION_DISABLE_0 - Disables USB, USB2, and AHB-DMA from arbitration */
         MAKE_REG32(0x6000C004) = 0x40060;
@@ -99,7 +99,7 @@ void init_dma_controllers(unsigned int target_firmware) {
         MAKE_REG32(0x60020038) = 0;
 
         /* MSELECT_CONFIG_0 |= WRAP_TO_INCR_SLAVE0(APC) | WRAP_TO_INCR_SLAVE1(PCIe) | WRAP_TO_INCR_SLAVE2(GPU) */
-        MAKE_REG32(0x50060000) |= 0x38000000;
+        MAKE_REG32(0x50060000) |= (MAKE_REG32(0x50060000) & 0xC4FFFFFF) | 0x38000000;
 
         /* AHB_ARBITRATION_PRIORITY_CTRL_0 - Select high prio group with prio 7 */
         MAKE_REG32(0x6000C008) = 0xE0000001;
@@ -111,6 +111,14 @@ void init_dma_controllers(unsigned int target_firmware) {
 
 void _set_memory_registers_enable_mmu(const uintptr_t ttbr0) {
     static const uintptr_t vbar  = TZRAM_GET_SEGMENT_ADDRESS(TZRAM_SEGEMENT_ID_SECMON_EVT) + 0x800;
+    /*
+        - Non-cacheable load forwarding enabled
+        - Disable load-pass DMB.
+        - NOTE: This and this alone is done via inline asm, due to register argument limits.
+    */
+
+    static const uint64_t cpuactlr = 0x800000001000000ull;
+    __asm__ __volatile__("msr s3_1_c15_c2_0, %0" :: "r"(cpuactlr) : "memory", "cc");
 
     /*
         - Disable table walk descriptor access prefetch.
@@ -197,12 +205,12 @@ void warmboot_init(void) {
     */
     flush_dcache_all();
     invalidate_icache_all();
-    
+
     /* On warmboot (not cpu_on) only */
     if (VIRT_MC_SECURITY_CFG3 == 0) {
         init_dma_controllers(g_exosphere_target_firmware_for_init);
     }
-    
+
     /*identity_remap_tzram();*/
     /* Nintendo pointlessly fully invalidate the TLB & invalidate the data cache on the modified ranges here */
     if (g_exosphere_target_firmware_for_init < ATMOSPHERE_TARGET_FIRMWARE_500) {

fusee/fusee-primary/src/gpio.c

@@ -21,66 +21,58 @@
 #include "gpio.h"
 #include "utils.h"
 
-static volatile tegra_gpio_bank_t *gpio_get_bank(uint32_t pin) {
+/* Set GPIO's value. */
+static void gpio_register_set(uint32_t pin, bool do_set, uint32_t offset) {
     volatile tegra_gpio_t *gpio = gpio_get_regs();
-    uint32_t bank_number = (pin >> GPIO_BANK_SHIFT);
-
-    return &gpio->bank[bank_number];
-}
-
-static volatile uint32_t gpio_get_port(uint32_t pin) {
-    return ((pin >> GPIO_PORT_SHIFT) & GPIO_PORT_MASK);
-}
-
-static volatile uint32_t gpio_get_mask(uint32_t pin) {
-    uint32_t pin_number = (pin & GPIO_PIN_MASK);
-    return (1 << pin_number);
-}
-
-static void gpio_simple_register_set(uint32_t pin, bool should_be_set, uint32_t offset) {
+    
     /* Retrieve the register set that corresponds to the given pin and offset. */
-    uintptr_t cluster_addr = (uintptr_t)gpio_get_bank(pin) + offset;
-    uint32_t *cluster = (uint32_t *)cluster_addr;
+    volatile uint32_t *cluster = (uint32_t *)((uintptr_t)&gpio->bank[(pin >> GPIO_BANK_SHIFT)] + offset);
 
     /* Figure out the offset into the cluster, and the mask to be used. */
-    uint32_t port = gpio_get_port(pin);
-    uint32_t mask = gpio_get_mask(pin);
+    uint32_t port = ((pin >> GPIO_PORT_SHIFT) & GPIO_PORT_MASK);
+    uint32_t mask = (1 << (pin & GPIO_PIN_MASK));
 
     /* Set or clear the bit, as appropriate. */
-    if (should_be_set)
+    if (do_set)
         cluster[port] |= mask;
     else
         cluster[port] &= ~mask;
     
     /* Dummy read. */
-    (void)cluster[port];
+    cluster[port];
 }
 
-static bool gpio_simple_register_get(uint32_t pin, uint32_t offset) {
+/* Get GPIO's value. */
+static bool gpio_register_get(uint32_t pin, uint32_t offset) {
+    volatile tegra_gpio_t *gpio = gpio_get_regs();
+    
     /* Retrieve the register set that corresponds to the given pin and offset. */
-    uintptr_t cluster_addr = (uintptr_t)gpio_get_bank(pin) + offset;
-    uint32_t *cluster = (uint32_t *)cluster_addr;
+    volatile uint32_t *cluster = (uint32_t *)((uintptr_t)&gpio->bank[(pin >> GPIO_BANK_SHIFT)] + offset);
 
     /* Figure out the offset into the cluster, and the mask to be used. */
-    uint32_t port = gpio_get_port(pin);
-    uint32_t mask = gpio_get_mask(pin);
+    uint32_t port = ((pin >> GPIO_PORT_SHIFT) & GPIO_PORT_MASK);
+    uint32_t mask = (1 << (pin & GPIO_PIN_MASK));
 
     /* Convert the given value to a boolean. */
     return !!(cluster[port] & mask);
 }
 
+/* Configure GPIO's mode. */
 void gpio_configure_mode(uint32_t pin, uint32_t mode) {
-    gpio_simple_register_set(pin, mode == GPIO_MODE_GPIO, offsetof(tegra_gpio_bank_t, config));
+    gpio_register_set(pin, mode == GPIO_MODE_GPIO, offsetof(tegra_gpio_bank_t, config));
 }
 
+/* Configure GPIO's direction. */
 void gpio_configure_direction(uint32_t pin, uint32_t dir) {
-    gpio_simple_register_set(pin, dir == GPIO_DIRECTION_OUTPUT, offsetof(tegra_gpio_bank_t, direction)); 
+    gpio_register_set(pin, dir == GPIO_DIRECTION_OUTPUT, offsetof(tegra_gpio_bank_t, direction)); 
 }
 
+/* Write to GPIO. */
 void gpio_write(uint32_t pin, uint32_t value) {
-    gpio_simple_register_set(pin, value == GPIO_LEVEL_HIGH, offsetof(tegra_gpio_bank_t, out)); 
+    gpio_register_set(pin, value == GPIO_LEVEL_HIGH, offsetof(tegra_gpio_bank_t, out)); 
 }
 
+/* Read from GPIO. */
 uint32_t gpio_read(uint32_t pin) {
-    return gpio_simple_register_get(pin, offsetof(tegra_gpio_bank_t, in)); 
+    return gpio_register_get(pin, offsetof(tegra_gpio_bank_t, in)); 
 }

fusee/fusee-primary/src/panic.c

@@ -13,7 +13,7 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
- 
+
 #include <stdio.h>
 #include "panic.h"
 #include "di.h"
@@ -51,8 +51,10 @@ static const char *get_error_desc_str(uint32_t error_desc) {
 
 static void _check_and_display_atmosphere_fatal_error(void) {
     /* Check for valid magic. */
-    if (ATMOSPHERE_FATAL_ERROR_CONTEXT->magic != ATMOSPHERE_REBOOT_TO_FATAL_MAGIC &&
-        ATMOSPHERE_FATAL_ERROR_CONTEXT->magic != ATMOSPHERE_REBOOT_TO_FATAL_MAGIC_0) {
+    if (ATMOSPHERE_FATAL_ERROR_CONTEXT->magic != ATMOSPHERE_REBOOT_TO_FATAL_MAGIC   &&
+        ATMOSPHERE_FATAL_ERROR_CONTEXT->magic != ATMOSPHERE_REBOOT_TO_FATAL_MAGIC_1 &&
+        ATMOSPHERE_FATAL_ERROR_CONTEXT->magic != ATMOSPHERE_REBOOT_TO_FATAL_MAGIC_0)
+    {
         return;
     }
 
@@ -69,10 +71,10 @@ static void _check_and_display_atmosphere_fatal_error(void) {
         /* Turn on the backlight after initializing the lfb */
         /* to avoid flickering. */
         display_backlight(true);
-        
+
         /* Override the global logging level. */
         log_set_log_level(SCREEN_LOG_LEVEL_ERROR);
-    
+
         /* Copy fatal error context to the stack. */
         atmosphere_fatal_error_ctx ctx = *(ATMOSPHERE_FATAL_ERROR_CONTEXT);
 
@@ -103,7 +105,7 @@ static void _check_and_display_atmosphere_fatal_error(void) {
 void check_and_display_panic(void) {
     /* Handle a panic sent via a stratosphere module. */
     _check_and_display_atmosphere_fatal_error();
-    
+
     /* We also handle our own panics. */
     bool has_panic = ((APBDEV_PMC_RST_STATUS_0 != 0) || (g_panic_code != 0));
     uint32_t code = (g_panic_code == 0) ? APBDEV_PMC_SCRATCH200_0 : g_panic_code;
@@ -152,10 +154,10 @@ void check_and_display_panic(void) {
 
         /* Initialize the display. */
         display_init();
-        
+
         /* Fill the screen. */
         display_color_screen(color);
-        
+
         /* Wait for button and reboot. */
         wait_for_button_and_reboot();
     } else {

fusee/fusee-primary/src/panic.h

@@ -13,7 +13,7 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
- 
+
 #ifndef FUSEE_PANIC_H
 #define FUSEE_PANIC_H
 
@@ -30,6 +30,7 @@
 
 #define AMS_FATAL_ERROR_MAX_STACKTRACE 0x20
 #define AMS_FATAL_ERROR_MAX_STACKDUMP 0x100
+#define AMS_FATAL_ERROR_TLS_SIZE      0x100
 
 /* Atmosphere reboot-to-fatal-error. */
 typedef struct {
@@ -57,10 +58,13 @@ typedef struct {
     uint64_t stack_dump_size;
     uint64_t stack_trace[AMS_FATAL_ERROR_MAX_STACKTRACE];
     uint8_t stack_dump[AMS_FATAL_ERROR_MAX_STACKDUMP];
+    uint8_t tls[AMS_FATAL_ERROR_TLS_SIZE];
 } atmosphere_fatal_error_ctx;
 
+/* "AFE2" */
+#define ATMOSPHERE_REBOOT_TO_FATAL_MAGIC   0x32454641
 /* "AFE1" */
-#define ATMOSPHERE_REBOOT_TO_FATAL_MAGIC 0x31454641
+#define ATMOSPHERE_REBOOT_TO_FATAL_MAGIC_1 0x31454641
 /* "AFE0" */
 #define ATMOSPHERE_REBOOT_TO_FATAL_MAGIC_0 0x30454641
 

fusee/fusee-primary/src/sdmmc/sdmmc_core.c

@@ -615,7 +615,7 @@ static void sdmmc_autocal_run(sdmmc_t *sdmmc, SdmmcBusVoltage voltage)
     while ((sdmmc->regs->auto_cal_status & SDMMC_AUTOCAL_ACTIVE)) {
         /* Ensure we haven't timed out. */
         if (get_time_since(timebase) > SDMMC_AUTOCAL_TIMEOUT) {
-            sdmmc_error(sdmmc, "Auto-calibration timed out!");
+            sdmmc_warn(sdmmc, "Auto-calibration timed out!");
             
             /* Force a register read to refresh the clock control value. */
             sdmmc_get_sd_clock_control(sdmmc);
@@ -1314,9 +1314,9 @@ static int sdmmc_wait_busy(sdmmc_t *sdmmc)
 
 static void sdmmc_intr_enable(sdmmc_t *sdmmc)
 {
-    /* Set all error bits and enable the relevant interrupts. */
-    sdmmc->regs->int_enable |= 0x017F0000;
+    /* Enable the relevant interrupts and set all error bits. */
     sdmmc->regs->int_enable |= (TEGRA_MMC_NORINTSTSEN_CMD_COMPLETE | TEGRA_MMC_NORINTSTSEN_XFER_COMPLETE | TEGRA_MMC_NORINTSTSEN_DMA_INTERRUPT);
+    sdmmc->regs->int_enable |= 0x017F0000;
     
     /* Refresh status. */
     sdmmc->regs->int_status = sdmmc->regs->int_status;
@@ -1324,34 +1324,35 @@ static void sdmmc_intr_enable(sdmmc_t *sdmmc)
 
 static void sdmmc_intr_disable(sdmmc_t *sdmmc)
 {
-    /* Clear all error bits and the interrupts. */
+    /* Clear all error bits and disable the relevant interrupts. */
     sdmmc->regs->int_enable &= ~(0x017F0000);
     sdmmc->regs->int_enable &= ~(TEGRA_MMC_NORINTSTSEN_CMD_COMPLETE | TEGRA_MMC_NORINTSTSEN_XFER_COMPLETE | TEGRA_MMC_NORINTSTSEN_DMA_INTERRUPT);
-    
-    /* Refresh status. */
-    sdmmc->regs->int_status = sdmmc->regs->int_status;
 }
 
-static bool sdmmc_intr_check_status(sdmmc_t *sdmmc, uint16_t status_mask)
+static int sdmmc_intr_check(sdmmc_t *sdmmc, uint16_t *status_out, uint16_t status_mask)
 {
-    bool is_masked = (sdmmc->regs->int_status & status_mask);
+    uint32_t int_status = sdmmc->regs->int_status;
     
-    /* Mask status. */
-    if (is_masked)
-        sdmmc->regs->int_status &= status_mask;
+    sdmmc_debug(sdmmc, "INTSTS: %08X", int_status);
     
-    return is_masked;
-}
-   
-static bool sdmmc_intr_check_error(sdmmc_t *sdmmc)
-{
-    bool is_error = (sdmmc->regs->int_status & TEGRA_MMC_NORINTSTS_ERR_INTERRUPT);
+    /* Return the status, if necessary. */
+    if (status_out)
+        *status_out = (int_status & 0xFFFF);
     
-    /* Refresh status. */
-    if (is_error)
-        sdmmc->regs->int_status = sdmmc->regs->int_status;
+    if (int_status & TEGRA_MMC_NORINTSTS_ERR_INTERRUPT)
+    {
+        /* Acknowledge error by refreshing status. */
+        sdmmc->regs->int_status = int_status;
+        return -1;
+    }
+    else if (int_status & status_mask)
+    {
+        /* Mask the status. */
+        sdmmc->regs->int_status = (int_status & status_mask);
+        return 1;
+    }
     
-    return is_error;
+    return 0;
 }
 
 static int sdmmc_dma_init(sdmmc_t *sdmmc, sdmmc_request_t *req)
@@ -1442,15 +1443,23 @@ static int sdmmc_dma_update(sdmmc_t *sdmmc)
         /* Watch over the DMA transfer. */
         while (!is_timeout) 
         {
+            /* Check interrupts. */
+            uint16_t intr_status = 0;
+            int intr_res = sdmmc_intr_check(sdmmc, &intr_status, TEGRA_MMC_NORINTSTS_XFER_COMPLETE | TEGRA_MMC_NORINTSTS_DMA_INTERRUPT);
+            
             /* An error has been raised. Reset. */
-            if (sdmmc_intr_check_error(sdmmc))
+            if (intr_res < 0)
             {
                 sdmmc_do_sw_reset(sdmmc);
                 return 0;
             }
             
+            /* Transfer is over. */
+            if (intr_status & TEGRA_MMC_NORINTSTS_XFER_COMPLETE)
+                return 1;
+            
             /* We have a DMA interrupt. Restart the transfer where it was interrupted. */
-            if (sdmmc_intr_check_status(sdmmc, TEGRA_MMC_NORINTSTS_DMA_INTERRUPT))
+            if (intr_status & TEGRA_MMC_NORINTSTS_DMA_INTERRUPT)
             {
                 if (sdmmc->use_adma)
                 {
@@ -1467,10 +1476,6 @@ static int sdmmc_dma_update(sdmmc_t *sdmmc)
                 sdmmc->next_dma_addr += 0x80000;
             }
             
-            /* Transfer is over. */
-            if (sdmmc_intr_check_status(sdmmc, TEGRA_MMC_NORINTSTS_XFER_COMPLETE))
-                return 1;
-            
             /* Keep checking if timeout expired. */
             is_timeout = (get_time_since(timebase) > 2000000);            
         }
@@ -1526,12 +1531,15 @@ static int sdmmc_wait_for_cmd(sdmmc_t *sdmmc)
 
     /* Wait for CMD to finish. */
     while (!is_err && !is_timeout) {
+        /* Check interrupts. */
+        int intr_res = sdmmc_intr_check(sdmmc, 0, TEGRA_MMC_NORINTSTS_CMD_COMPLETE);
+        
         /* Command is done. */
-        if (sdmmc_intr_check_status(sdmmc, TEGRA_MMC_NORINTSTS_CMD_COMPLETE))
+        if (intr_res > 0)
             return 1;
         
         /* Check for any raised errors. */
-        is_err = sdmmc_intr_check_error(sdmmc);
+        is_err = (intr_res < 0);
         
         /* Keep checking if timeout expired. */
         is_timeout = (get_time_since(timebase) > 2000000);
@@ -1642,6 +1650,7 @@ int sdmmc_send_cmd(sdmmc_t *sdmmc, sdmmc_command_t *cmd, sdmmc_request_t *req, u
         is_dma = true;
         dma_blkcnt = sdmmc_dma_init(sdmmc, req);
         
+        /* Abort in case initialization failed. */
         if (!dma_blkcnt)
         {
             sdmmc_error(sdmmc, "Failed to initialize the DMA transfer!");
@@ -1669,12 +1678,14 @@ int sdmmc_send_cmd(sdmmc_t *sdmmc, sdmmc_command_t *cmd, sdmmc_request_t *req, u
         /* Save response, if necessary. */
         sdmmc_save_response(sdmmc, cmd->flags);
         
-        /* Process the DMA request. */
+        /* Update the DMA request. */
         if (req)
         {
+            /* Disable interrupts and abort in case updating failed. */
             if (!sdmmc_dma_update(sdmmc))
             {
-                sdmmc_error(sdmmc, "Failed to process the DMA transfer!");
+                sdmmc_warn(sdmmc, "Failed to update the DMA transfer!");
+                sdmmc_intr_disable(sdmmc);
                 return 0;
             }
             
@@ -1840,7 +1851,7 @@ static int sdmmc_send_tuning(sdmmc_t *sdmmc, uint32_t opcode)
     while (!is_timeout)
     {
         /* Buffer Read Ready was asserted. */
-        if (sdmmc_intr_check_status(sdmmc, TEGRA_MMC_NORINTSTSEN_BUFFER_READ_READY))
+        if (sdmmc_intr_check(sdmmc, 0, TEGRA_MMC_NORINTSTSEN_BUFFER_READ_READY) > 0)
         {
             /* Manually disable the Buffer Read Ready interrupt. */
             sdmmc->regs->int_enable &= ~(TEGRA_MMC_NORINTSTSEN_BUFFER_READ_READY);

fusee/fusee-primary/src/uart.c

@@ -34,28 +34,28 @@ void uart_config(UartDevice dev) {
 
     switch (dev) {
         case UART_A:
-            pinmux->uart1_tx = 0;
-            pinmux->uart1_rx = (PINMUX_INPUT | PINMUX_PULL_UP);
-            pinmux->uart1_rts = 0;
-            pinmux->uart1_cts = (PINMUX_INPUT | PINMUX_PULL_DOWN);
+            pinmux->uart1_tx = (0 | 0 | PINMUX_PULL_NONE | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart1_rx = (PINMUX_INPUT | PINMUX_TRISTATE | PINMUX_PULL_UP | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart1_rts = (0 | 0 | PINMUX_PULL_NONE | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart1_cts = (PINMUX_INPUT | PINMUX_TRISTATE | PINMUX_PULL_DOWN | PINMUX_SELECT_FUNCTION0);
             break;
         case UART_B:
-            pinmux->uart2_tx = 0;
-            pinmux->uart2_rx = (PINMUX_INPUT | PINMUX_PULL_UP);
-            pinmux->uart2_rts = 0;
-            pinmux->uart2_cts = (PINMUX_INPUT | PINMUX_PULL_DOWN);
+            pinmux->uart2_tx = (0 | 0 | PINMUX_PULL_NONE | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart2_rx = (PINMUX_INPUT | PINMUX_TRISTATE | PINMUX_PULL_NONE | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart2_rts = (0 | 0 | PINMUX_PULL_DOWN | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart2_cts = (PINMUX_INPUT | PINMUX_TRISTATE | PINMUX_PULL_NONE | PINMUX_SELECT_FUNCTION0);
             break;
         case UART_C:
-            pinmux->uart3_tx = 0;
-            pinmux->uart3_rx = (PINMUX_INPUT | PINMUX_PULL_UP);
-            pinmux->uart3_rts = 0;
-            pinmux->uart3_cts = (PINMUX_INPUT | PINMUX_PULL_DOWN);
+            pinmux->uart3_tx = (0 | 0 | PINMUX_PULL_NONE | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart3_rx = (PINMUX_INPUT | PINMUX_TRISTATE | PINMUX_PULL_NONE | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart3_rts = (0 | 0 | PINMUX_PULL_DOWN | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart3_cts = (PINMUX_INPUT | PINMUX_TRISTATE | PINMUX_PULL_NONE | PINMUX_SELECT_FUNCTION0);
             break;
         case UART_D:
-            pinmux->uart4_tx = 0;
-            pinmux->uart4_rx = (PINMUX_INPUT | PINMUX_PULL_UP);
-            pinmux->uart4_rts = 0;
-            pinmux->uart4_cts = (PINMUX_INPUT | PINMUX_PULL_DOWN);
+            pinmux->uart4_tx = (0 | 0 | PINMUX_PULL_DOWN | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart4_rx = (PINMUX_INPUT | PINMUX_TRISTATE | PINMUX_PULL_DOWN | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart4_rts = (0 | 0 | PINMUX_PULL_DOWN | PINMUX_SELECT_FUNCTION0);
+            pinmux->uart4_cts = (PINMUX_INPUT | PINMUX_TRISTATE | PINMUX_PULL_DOWN | PINMUX_SELECT_FUNCTION0);
             break;
         case UART_E:
             /* Unused. */

fusee/fusee-secondary/Makefile

@@ -84,7 +84,7 @@ ifneq ($(BUILD),$(notdir $(CURDIR)))
 export OUTPUT	:=	$(CURDIR)/$(TARGET)
 export TOPDIR	:=	$(CURDIR)
 
-export KIPDIRS	:= $(AMS)/stratosphere/loader $(AMS)/stratosphere/pm $(AMS)/stratosphere/sm $(AMS)/stratosphere/boot $(AMS)/stratosphere/spl $(AMS)/stratosphere/ams_mitm
+export KIPDIRS	:= $(AMS)/stratosphere/loader $(AMS)/stratosphere/ncm $(AMS)/stratosphere/pm $(AMS)/stratosphere/sm $(AMS)/stratosphere/boot $(AMS)/stratosphere/spl $(AMS)/stratosphere/ams_mitm
 export VPATH	:=	$(foreach dir,$(SOURCES),$(CURDIR)/$(dir)) \
 			$(foreach dir,$(DATA),$(CURDIR)/$(dir)) \
 			$(AMS)/exosphere $(AMS)/exosphere/lp0fw $(AMS)/exosphere/rebootstub \
@@ -96,7 +96,7 @@ export DEPSDIR	:=	$(CURDIR)/$(BUILD)
 CFILES		:=	$(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.c)))
 CPPFILES	:=	$(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.cpp)))
 SFILES		:=	$(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.s)))
-KIPFILES 	:=	loader.kip pm.kip sm.kip ams_mitm.kip spl.kip boot.kip
+KIPFILES 	:=	loader.kip ncm.kip pm.kip sm.kip ams_mitm.kip spl.kip boot.kip
 BINFILES	:=	$(foreach dir,$(DATA),$(notdir $(wildcard $(dir)/*.*))) fusee-primary.bin \
 				exosphere.bin lp0fw.bin rebootstub.bin thermosphere.bin splash_screen.bmp \
 				sept-primary.bin sept-secondary_00.enc sept-secondary_01.enc emummc.kip \
@@ -221,12 +221,12 @@ sept_secondary_01.enc.o	sept_secondary_01_enc.h:	sept-secondary_01.enc
 #---------------------------------------------------------------------------------
 	@echo $(notdir $<)
 	@$(_bin2o)
-	
+
 sept_secondary_dev_00.enc.o	sept_secondary_dev_00_enc.h:	sept-secondary_dev_00.enc
 #---------------------------------------------------------------------------------
 	@echo $(notdir $<)
 	@$(_bin2o)
-	
+
 sept_secondary_dev_01.enc.o	sept_secondary_dev_01_enc.h:	sept-secondary_dev_01.enc
 #---------------------------------------------------------------------------------
 	@echo $(notdir $<)

fusee/fusee-secondary/linker.ld

@@ -230,6 +230,8 @@ SECTIONS
     PROVIDE(__loader_kip_size__ = loader_kip_end - loader_kip);
     PROVIDE(__lp0fw_bin_start__ = lp0fw_bin - __start__);
     PROVIDE(__lp0fw_bin_size__ = lp0fw_bin_end - lp0fw_bin);
+    PROVIDE(__ncm_kip_start__ = ncm_kip - __start__);
+    PROVIDE(__ncm_kip_size__ = ncm_kip_end - ncm_kip);
     PROVIDE(__pm_kip_start__ = pm_kip - __start__);
     PROVIDE(__pm_kip_size__ = pm_kip_end - pm_kip);
     PROVIDE(__rebootstub_bin_start__ = rebootstub_bin - __start__);

fusee/fusee-secondary/src/emummc_cfg.h

@@ -82,6 +82,9 @@ typedef enum {
     FS_VER_9_1_0,
     FS_VER_9_1_0_EXFAT,
 
+    FS_VER_10_0_0,
+    FS_VER_10_0_0_EXFAT,
+
     FS_VER_MAX,
 } emummc_fs_ver_t;
 

fusee/fusee-secondary/src/gpio.c

@@ -21,66 +21,58 @@
 #include "gpio.h"
 #include "utils.h"
 
-static volatile tegra_gpio_bank_t *gpio_get_bank(uint32_t pin) {
+/* Set GPIO's value. */
+static void gpio_register_set(uint32_t pin, bool do_set, uint32_t offset) {
     volatile tegra_gpio_t *gpio = gpio_get_regs();
-    uint32_t bank_number = (pin >> GPIO_BANK_SHIFT);
-
-    return &gpio->bank[bank_number];
-}
-
-static volatile uint32_t gpio_get_port(uint32_t pin) {
-    return ((pin >> GPIO_PORT_SHIFT) & GPIO_PORT_MASK);
-}
-
-static volatile uint32_t gpio_get_mask(uint32_t pin) {
-    uint32_t pin_number = (pin & GPIO_PIN_MASK);
-    return (1 << pin_number);
-}
-
-static void gpio_simple_register_set(uint32_t pin, bool should_be_set, uint32_t offset) {
+    
     /* Retrieve the register set that corresponds to the given pin and offset. */
-    uintptr_t cluster_addr = (uintptr_t)gpio_get_bank(pin) + offset;
-    uint32_t *cluster = (uint32_t *)cluster_addr;
+    volatile uint32_t *cluster = (uint32_t *)((uintptr_t)&gpio->bank[(pin >> GPIO_BANK_SHIFT)] + offset);
 
     /* Figure out the offset into the cluster, and the mask to be used. */
-    uint32_t port = gpio_get_port(pin);
-    uint32_t mask = gpio_get_mask(pin);
+    uint32_t port = ((pin >> GPIO_PORT_SHIFT) & GPIO_PORT_MASK);
+    uint32_t mask = (1 << (pin & GPIO_PIN_MASK));
 
     /* Set or clear the bit, as appropriate. */
-    if (should_be_set)
+    if (do_set)
         cluster[port] |= mask;
     else
         cluster[port] &= ~mask;
     
     /* Dummy read. */
-    (void)cluster[port];
+    cluster[port];
 }
 
-static bool gpio_simple_register_get(uint32_t pin, uint32_t offset) {
+/* Get GPIO's value. */
+static bool gpio_register_get(uint32_t pin, uint32_t offset) {
+    volatile tegra_gpio_t *gpio = gpio_get_regs();
+    
     /* Retrieve the register set that corresponds to the given pin and offset. */
-    uintptr_t cluster_addr = (uintptr_t)gpio_get_bank(pin) + offset;
-    uint32_t *cluster = (uint32_t *)cluster_addr;
+    volatile uint32_t *cluster = (uint32_t *)((uintptr_t)&gpio->bank[(pin >> GPIO_BANK_SHIFT)] + offset);
 
     /* Figure out the offset into the cluster, and the mask to be used. */
-    uint32_t port = gpio_get_port(pin);
-    uint32_t mask = gpio_get_mask(pin);
+    uint32_t port = ((pin >> GPIO_PORT_SHIFT) & GPIO_PORT_MASK);
+    uint32_t mask = (1 << (pin & GPIO_PIN_MASK));
 
     /* Convert the given value to a boolean. */
     return !!(cluster[port] & mask);
 }
 
+/* Configure GPIO's mode. */
 void gpio_configure_mode(uint32_t pin, uint32_t mode) {
-    gpio_simple_register_set(pin, mode == GPIO_MODE_GPIO, offsetof(tegra_gpio_bank_t, config));
+    gpio_register_set(pin, mode == GPIO_MODE_GPIO, offsetof(tegra_gpio_bank_t, config));
 }
 
+/* Configure GPIO's direction. */
 void gpio_configure_direction(uint32_t pin, uint32_t dir) {
-    gpio_simple_register_set(pin, dir == GPIO_DIRECTION_OUTPUT, offsetof(tegra_gpio_bank_t, direction)); 
+    gpio_register_set(pin, dir == GPIO_DIRECTION_OUTPUT, offsetof(tegra_gpio_bank_t, direction)); 
 }
 
+/* Write to GPIO. */
 void gpio_write(uint32_t pin, uint32_t value) {
-    gpio_simple_register_set(pin, value == GPIO_LEVEL_HIGH, offsetof(tegra_gpio_bank_t, out)); 
+    gpio_register_set(pin, value == GPIO_LEVEL_HIGH, offsetof(tegra_gpio_bank_t, out)); 
 }
 
+/* Read from GPIO. */
 uint32_t gpio_read(uint32_t pin) {
-    return gpio_simple_register_get(pin, offsetof(tegra_gpio_bank_t, in)); 
+    return gpio_register_get(pin, offsetof(tegra_gpio_bank_t, in)); 
 }

fusee/fusee-secondary/src/ips.c

@@ -417,6 +417,9 @@ static const uint8_t g_fs_hashes[FS_VER_MAX][0x8] = {
 
     "\xB5\xE7\xA6\x4C\x6F\x5C\x4F\xE3", /* FS_VER_9_1_0 */
     "\xF1\x96\xD1\x44\xD0\x44\x45\xB6", /* FS_VER_9_1_0_EXFAT */
+
+    "\x3E\xEB\xD9\xB7\xBC\xD1\xB5\xE0", /* FS_VER_10_0_0 */
+    "\x81\x7E\xA2\xB0\xB7\x02\xC1\xF3", /* FS_VER_10_0_0_EXFAT */
 };
 
 kip1_header_t *apply_kip_ips_patches(kip1_header_t *kip, size_t kip_size, emummc_fs_ver_t *out_fs_ver) {

fusee/fusee-secondary/src/kernel_patches.c

@@ -486,20 +486,77 @@ static const instruction_t MAKE_KERNEL_PATCH_NAME(900, proc_id_send)[] = {0xA9BF
 static const uint8_t MAKE_KERNEL_PATTERN_NAME(900, proc_id_recv)[] = {0x68, 0x03, 0x40, 0xF9, 0x08, 0x1D, 0x40, 0xF9, 0xE0, 0x03, 0x1B, 0xAA, 0x00, 0x01, 0x3F, 0xD6, 0xE8, 0x03, 0x17, 0x2A, 0xF7, 0x0A, 0x00, 0x11, 0x08, 0xF5, 0x7E, 0xD3};
 static const instruction_t MAKE_KERNEL_PATCH_NAME(900, proc_id_recv)[] = {0xA9BF2FEA, 0xF9404BEB, 0x2A1703EA, 0xD37EF54A, 0xF86A696A, 0x92FFFFE9, 0x8A090148, 0xD2FFFFE9, 0x8A09014A, 0xD2FFFFC9, 0xEB09015F, 0x54000100, 0xA9BF27E8, 0xF9400368, 0xF9401D08, 0xAA1B03E0, 0xD63F0100, 0xA8C127E8, 0xAA0003E8, 0xA8C12FEA, 0xAA0803E0};
 
+/*
+    stp x10, x11, [sp, #-0x10]!
+    ldr x11, [sp, #0xC0]
+    mov w10, w22
+    lsl x10, x10, #2
+    ldr x10, [x11, x10]
+    mov x9, #0x0000ffffffffffff
+    and x8, x10, x9
+    mov x9, #0xffff000000000000
+    and x10, x10, x9
+    mov x9, #0xfffe000000000000
+    cmp x10, x9
+    beq #0x20
+
+    stp x8, x9, [sp, #-0x10]!
+    ldr x8, [x23]
+    ldr x8, [x8, #0x38]
+    mov x0, x23
+    blr x8
+    ldp x8, x9, [sp],#0x10
+    mov x8, x0
+
+    ldp x10, x11, [sp],#0x10
+    mov x0, x8
+*/
+static const uint8_t MAKE_KERNEL_PATTERN_NAME(1000, proc_id_send)[] = {0xE8, 0x02, 0x40, 0xF9, 0x08, 0x1D, 0x40, 0xF9, 0xE0, 0x03, 0x17, 0xAA, 0x00, 0x01, 0x3F, 0xD6, 0x08, 0x4B, 0x36, 0x8B, 0x09, 0xFC, 0x60, 0xD3, 0x00, 0x25, 0x00, 0x29};
+static const instruction_t MAKE_KERNEL_PATCH_NAME(1000, proc_id_send)[] = {0xA9BF2FEA, 0xF94063EB, 0x2A1603EA, 0xD37EF54A, 0xF86A696A, 0x92FFFFE9, 0x8A090148, 0xD2FFFFE9, 0x8A09014A, 0xD2FFFFC9, 0xEB09015F, 0x54000100, 0xA9BF27E8, 0xF94002E8, 0xF9401D08, 0xAA1703E0, 0xD63F0100, 0xA8C127E8, 0xAA0003E8, 0xA8C12FEA, 0xAA0803E0};
+/*
+    stp x10, x11, [sp, #-0x10]!
+    ldr x11, [sp, #0xC8]
+    mov w10, w26
+    lsl x10, x10, #2
+    ldr x10, [x11, x10]
+    mov x9, #0x0000ffffffffffff
+    and x8, x10, x9
+    mov x9, #0xffff000000000000
+    and x10, x10, x9
+    mov x9, #0xfffe000000000000
+    cmp x10, x9
+    beq #0x20
+
+    stp x8, x9, [sp, #-0x10]!
+    ldr x8, [x28]
+    ldr x8, [x8, #0x38]
+    mov x0, x28
+    blr x8
+    ldp x8, x9, [sp],#0x10
+    mov x8, x0
+
+    ldp x10, x11, [sp],#0x10
+    mov x0, x8
+*/
+static const uint8_t MAKE_KERNEL_PATTERN_NAME(1000, proc_id_recv)[] = {0x88, 0x03, 0x40, 0xF9, 0x08, 0x1D, 0x40, 0xF9, 0xE0, 0x03, 0x1C, 0xAA, 0x00, 0x01, 0x3F, 0xD6, 0xE8, 0x87, 0x40, 0xF9, 0x08, 0x49, 0x3A, 0x8B, 0x09, 0xFC, 0x60, 0xD3};
+static const instruction_t MAKE_KERNEL_PATCH_NAME(1000, proc_id_recv)[] = {0xA9BF2FEA, 0xF94067EB, 0x2A1A03EA, 0xD37EF54A, 0xF86A696A, 0x92FFFFE9, 0x8A090148, 0xD2FFFFE9, 0x8A09014A, 0xD2FFFFC9, 0xEB09015F, 0x54000100, 0xA9BF27E8, 0xF9400388, 0xF9401D08, 0xAA1C03E0, 0xD63F0100, 0xA8C127E8, 0xAA0003E8, 0xA8C12FEA, 0xAA0803E0};
+
+
 /* svcControlCodeMemory Patches */
 /* b.eq -> nop */
-static const instruction_t MAKE_KERNEL_PATCH_NAME(500, svc_control_codememory)[] = {MAKE_NOP};
-static const instruction_t MAKE_KERNEL_PATCH_NAME(600, svc_control_codememory)[] = {MAKE_NOP};
-static const instruction_t MAKE_KERNEL_PATCH_NAME(700, svc_control_codememory)[] = {MAKE_NOP};
-static const instruction_t MAKE_KERNEL_PATCH_NAME(800, svc_control_codememory)[] = {MAKE_NOP};
-static const instruction_t MAKE_KERNEL_PATCH_NAME(900, svc_control_codememory)[] = {MAKE_NOP};
+static const instruction_t MAKE_KERNEL_PATCH_NAME(500,  svc_control_codememory)[] = {MAKE_NOP};
+static const instruction_t MAKE_KERNEL_PATCH_NAME(600,  svc_control_codememory)[] = {MAKE_NOP};
+static const instruction_t MAKE_KERNEL_PATCH_NAME(700,  svc_control_codememory)[] = {MAKE_NOP};
+static const instruction_t MAKE_KERNEL_PATCH_NAME(800,  svc_control_codememory)[] = {MAKE_NOP};
+static const instruction_t MAKE_KERNEL_PATCH_NAME(900,  svc_control_codememory)[] = {MAKE_NOP};
+static const instruction_t MAKE_KERNEL_PATCH_NAME(1000, svc_control_codememory)[] = {MAKE_NOP};
 
-
-static const instruction_t MAKE_KERNEL_PATCH_NAME(500, system_memory_increase)[] = {0x52A3C008}; /* MOV W8,   #0x1E000000 */
-static const instruction_t MAKE_KERNEL_PATCH_NAME(600, system_memory_increase)[] = {0x52A3B008}; /* MOV W8,   #0x1D800000 */
-static const instruction_t MAKE_KERNEL_PATCH_NAME(700, system_memory_increase)[] = {0x52A3B008}; /* MOV W8,   #0x1D800000 */
-static const instruction_t MAKE_KERNEL_PATCH_NAME(800, system_memory_increase)[] = {0x52A3B013}; /* MOV W19,  #0x1D800000 */
-static const instruction_t MAKE_KERNEL_PATCH_NAME(900, system_memory_increase)[] = {0x52A3B013}; /* MOV W19,  #0x1D800000 */
+static const instruction_t MAKE_KERNEL_PATCH_NAME(500,  system_memory_increase)[] = {0x52A3C008}; /* MOV W8,   #0x1E000000 */
+static const instruction_t MAKE_KERNEL_PATCH_NAME(600,  system_memory_increase)[] = {0x52A3B008}; /* MOV W8,   #0x1D800000 */
+static const instruction_t MAKE_KERNEL_PATCH_NAME(700,  system_memory_increase)[] = {0x52A3B008}; /* MOV W8,   #0x1D800000 */
+static const instruction_t MAKE_KERNEL_PATCH_NAME(800,  system_memory_increase)[] = {0x52A3B013}; /* MOV W19,  #0x1D800000 */
+static const instruction_t MAKE_KERNEL_PATCH_NAME(900,  system_memory_increase)[] = {0x52A3B013}; /* MOV W19,  #0x1D800000 */
+static const instruction_t MAKE_KERNEL_PATCH_NAME(1000, system_memory_increase)[] = {0x52A3B013}; /* MOV W19,  #0x1D800000 */
 
 /* Hook Definitions. */
 static const kernel_patch_t g_kernel_patches_100[] = {
@@ -735,6 +792,35 @@ static const kernel_patch_t g_kernel_patches_900[] = {
     }
 };
 
+static const kernel_patch_t g_kernel_patches_1000[] = {
+    {   /* Send Message Process ID Patch. */
+        .pattern_size = 0x1C,
+        .pattern = MAKE_KERNEL_PATTERN_NAME(1000, proc_id_send),
+        .pattern_hook_offset = 0x0,
+        .payload_num_instructions = sizeof(MAKE_KERNEL_PATCH_NAME(1000, proc_id_send))/sizeof(instruction_t),
+        .branch_back_offset = 0x10,
+        .payload = MAKE_KERNEL_PATCH_NAME(1000, proc_id_send)
+    },
+    {   /* Receive Message Process ID Patch. */
+        .pattern_size = 0x1C,
+        .pattern = MAKE_KERNEL_PATTERN_NAME(1000, proc_id_recv),
+        .pattern_hook_offset = 0x0,
+        .payload_num_instructions = sizeof(MAKE_KERNEL_PATCH_NAME(1000, proc_id_recv))/sizeof(instruction_t),
+        .branch_back_offset = 0x10,
+        .payload = MAKE_KERNEL_PATCH_NAME(1000, proc_id_recv)
+    },
+    {   /* svcControlCodeMemory Patch. */
+        .payload_num_instructions = sizeof(MAKE_KERNEL_PATCH_NAME(1000, svc_control_codememory))/sizeof(instruction_t),
+        .payload = MAKE_KERNEL_PATCH_NAME(1000, svc_control_codememory),
+        .patch_offset = 0x45DAC,
+    },
+    {   /* System Memory Increase Patch. */
+        .payload_num_instructions = sizeof(MAKE_KERNEL_PATCH_NAME(1000, system_memory_increase))/sizeof(instruction_t),
+        .payload = MAKE_KERNEL_PATCH_NAME(1000, system_memory_increase),
+        .patch_offset = 0x66950,
+    }
+};
+
 #define KERNEL_PATCHES(vers) .num_patches = sizeof(g_kernel_patches_##vers)/sizeof(kernel_patch_t), .patches = g_kernel_patches_##vers,
 
 /* Kernel Infos. */
@@ -801,6 +887,25 @@ static const kernel_info_t g_kernel_infos[] = {
         .embedded_ini_ptr = 0x180,
         .free_code_space_offset = 0x65780,
         KERNEL_PATCHES(900)
+    },
+    {   /* 9.2.0. */
+        /* NOTE: 9.2.0 has identical kernel layout to 9.0.0, so patches may be reused. */
+        .hash = {0x66, 0xE7, 0x73, 0xE7, 0xF5, 0xCB, 0x9B, 0xB8, 0x66, 0xB7, 0xB1, 0x26, 0x23, 0x02, 0x76, 0xF2, 0xD1, 0x60, 0x3E, 0x09, 0x14, 0x19, 0xC2, 0x84, 0xFA, 0x5D, 0x0F, 0x16, 0xA3, 0x65, 0xFA, 0x17},
+        .hash_offset = 0x1C0,
+        .hash_size = 0x90000 - 0x1C0,
+        .embedded_ini_offset = 0x90000,
+        .embedded_ini_ptr = 0x180,
+        .free_code_space_offset = 0x65780,
+        KERNEL_PATCHES(900)
+    },
+    {   /* 10.0.0. */
+        .hash = {0x59, 0x31, 0xE6, 0x46, 0xF7, 0xAA, 0x15, 0x59, 0x78, 0xC7, 0xB3, 0xA5, 0xFA, 0x80, 0xE2, 0xC0, 0xCA, 0x6F, 0x31, 0x97, 0x3D, 0x86, 0x8A, 0x69, 0xF3, 0xBF, 0xE6, 0xE5, 0x61, 0xA7, 0x1B, 0x5B, },
+        .hash_offset = 0x1B8,
+        .hash_size = 0x93000 - 0x1B8,
+        .embedded_ini_offset = 0x93000,
+        .embedded_ini_ptr = 0x178,
+        .free_code_space_offset = 0x67790,
+        KERNEL_PATCHES(1000)
     }
 };
 
@@ -845,7 +950,7 @@ const kernel_info_t *get_kernel_info(void *kernel, size_t size) {
     return NULL;
 }
 
-void package2_patch_kernel(void *_kernel, size_t *kernel_size, bool is_sd_kernel, void **out_ini1) {
+void package2_patch_kernel(void *_kernel, size_t *kernel_size, bool is_sd_kernel, void **out_ini1, uint32_t target_firmware) {
     const kernel_info_t *kernel_info = get_kernel_info(_kernel, *kernel_size);
     *out_ini1 = NULL;
 
@@ -866,6 +971,12 @@ void package2_patch_kernel(void *_kernel, size_t *kernel_size, bool is_sd_kernel
         const uint32_t kernel_ldr_offset = *((volatile uint64_t *)((uintptr_t)_kernel + kernel_info->embedded_ini_ptr + 8));
         memcpy((void *)((uintptr_t)_kernel + kernel_ldr_offset), kernel_ldr_bin, kernel_ldr_bin_size);
 
+        /* Set target firmware for our kernel loader. */
+        uint32_t *kldr_u32 = (uint32_t *)((uintptr_t)_kernel + kernel_ldr_offset);
+        if (kldr_u32[1] == 0x30444C4D) {
+            kldr_u32[2] = target_firmware;
+        }
+
         /* Update size. */
         *kernel_size = kernel_ldr_offset + kernel_ldr_bin_size;
 

fusee/fusee-secondary/src/kernel_patches.h

@@ -19,6 +19,6 @@
 
 #include "utils.h"
 
-void package2_patch_kernel(void *kernel, size_t *kernel_size, bool is_sd_kernel, void **out_ini1);
+void package2_patch_kernel(void *kernel, size_t *kernel_size, bool is_sd_kernel, void **out_ini1, uint32_t target_firmware);
 
 #endif

fusee/fusee-secondary/src/key_derivation.c

@@ -160,6 +160,7 @@ int derive_nx_keydata(uint32_t target_firmware, const nx_keyblob_t *keyblobs, ui
                 desired_keyblob = MASTERKEY_REVISION_900;
                 /* Fallthrough */
             case ATMOSPHERE_TARGET_FIRMWARE_910:
+            case ATMOSPHERE_TARGET_FIRMWARE_1000:
                 desired_keyblob = MASTERKEY_REVISION_910_CURRENT;
                 break;
             default:

fusee/fusee-secondary/src/nxboot.c

@@ -184,6 +184,12 @@ static int stratosphere_ini_handler(void *user, const char *section, const char
             strat_cfg->has_nogc_config = true;
             sscanf(value, "%d", &tmp);
             strat_cfg->enable_nogc = tmp != 0;
+        } else if (strcmp(name, STRATOSPHERE_ENABLE_NCM_KEY) == 0) {
+            sscanf(value, "%d", &tmp);
+            strat_cfg->ncm_enabled = tmp != 0;
+            if (strat_cfg->ncm_enabled) {
+                stratosphere_enable_ncm();
+            }
         } else {
             return 0;
         }
@@ -226,6 +232,8 @@ static uint32_t nxboot_get_target_firmware(const void *package1loader) {
                 return ATMOSPHERE_TARGET_FIRMWARE_900;
             } else if (memcmp(package1loader_header->build_timestamp, "20191021", 8) == 0) {
                 return ATMOSPHERE_TARGET_FIRMWARE_910;
+            } else if (memcmp(package1loader_header->build_timestamp, "20200303", 8) == 0) {
+                return ATMOSPHERE_TARGET_FIRMWARE_1000;
             } else {
                 fatal_error("[NXBOOT] Unable to identify package1!\n");
             }

fusee/fusee-secondary/src/package2.c

@@ -87,7 +87,7 @@ void package2_rebuild_and_copy(package2_header_t *package2, uint32_t target_firm
     }
 
     /* Perform any patches we want to the NX kernel. */
-    package2_patch_kernel(kernel, &kernel_size, is_sd_kernel, (void *)&orig_ini1);
+    package2_patch_kernel(kernel, &kernel_size, is_sd_kernel, (void *)&orig_ini1, target_firmware);
 
     /* Ensure we know where embedded INI is if present, and we don't if not. */
     if ((target_firmware < ATMOSPHERE_TARGET_FIRMWARE_800 && orig_ini1 != NULL) ||
@@ -232,7 +232,7 @@ static bool package2_validate_metadata(package2_meta_t *metadata, uint8_t data[]
 
     /* Perform version checks. */
     /* We will be compatible with all package2s released before current, but not newer ones. */
-    if (metadata->version_max >= PACKAGE2_MINVER_THEORETICAL && metadata->version_min < PACKAGE2_MAXVER_910_CURRENT) {
+    if (metadata->version_max >= PACKAGE2_MINVER_THEORETICAL && metadata->version_min < PACKAGE2_MAXVER_1000_CURRENT) {
         return true;
     }
 

fusee/fusee-secondary/src/package2.h

@@ -39,7 +39,8 @@
 #define PACKAGE2_MAXVER_700_800 0xA
 #define PACKAGE2_MAXVER_810 0xB
 #define PACKAGE2_MAXVER_900 0xC
-#define PACKAGE2_MAXVER_910_CURRENT 0xD
+#define PACKAGE2_MAXVER_910_920 0xD
+#define PACKAGE2_MAXVER_1000_CURRENT 0xE
 
 #define PACKAGE2_MINVER_100 0x3
 #define PACKAGE2_MINVER_200 0x4
@@ -52,7 +53,8 @@
 #define PACKAGE2_MINVER_700_800 0xB
 #define PACKAGE2_MINVER_810 0xC
 #define PACKAGE2_MINVER_900 0xD
-#define PACKAGE2_MINVER_910_CURRENT 0xE
+#define PACKAGE2_MINVER_910_920 0xE
+#define PACKAGE2_MINVER_1000_CURRENT 0xF
 
 #define NX_BOOTLOADER_PACKAGE2_LOAD_ADDRESS ((void *)(0xA9800000ull))
 

fusee/fusee-secondary/src/sdmmc/sdmmc_core.c

@@ -615,7 +615,7 @@ static void sdmmc_autocal_run(sdmmc_t *sdmmc, SdmmcBusVoltage voltage)
     while ((sdmmc->regs->auto_cal_status & SDMMC_AUTOCAL_ACTIVE)) {
         /* Ensure we haven't timed out. */
         if (get_time_since(timebase) > SDMMC_AUTOCAL_TIMEOUT) {
-            sdmmc_error(sdmmc, "Auto-calibration timed out!");
+            sdmmc_warn(sdmmc, "Auto-calibration timed out!");
             
             /* Force a register read to refresh the clock control value. */
             sdmmc_get_sd_clock_control(sdmmc);
@@ -1314,9 +1314,9 @@ static int sdmmc_wait_busy(sdmmc_t *sdmmc)
 
 static void sdmmc_intr_enable(sdmmc_t *sdmmc)
 {
-    /* Set all error bits and enable the relevant interrupts. */
-    sdmmc->regs->int_enable |= 0x017F0000;
+    /* Enable the relevant interrupts and set all error bits. */
     sdmmc->regs->int_enable |= (TEGRA_MMC_NORINTSTSEN_CMD_COMPLETE | TEGRA_MMC_NORINTSTSEN_XFER_COMPLETE | TEGRA_MMC_NORINTSTSEN_DMA_INTERRUPT);
+    sdmmc->regs->int_enable |= 0x017F0000;
     
     /* Refresh status. */
     sdmmc->regs->int_status = sdmmc->regs->int_status;
@@ -1324,34 +1324,35 @@ static void sdmmc_intr_enable(sdmmc_t *sdmmc)
 
 static void sdmmc_intr_disable(sdmmc_t *sdmmc)
 {
-    /* Clear all error bits and the interrupts. */
+    /* Clear all error bits and disable the relevant interrupts. */
     sdmmc->regs->int_enable &= ~(0x017F0000);
     sdmmc->regs->int_enable &= ~(TEGRA_MMC_NORINTSTSEN_CMD_COMPLETE | TEGRA_MMC_NORINTSTSEN_XFER_COMPLETE | TEGRA_MMC_NORINTSTSEN_DMA_INTERRUPT);
-    
-    /* Refresh status. */
-    sdmmc->regs->int_status = sdmmc->regs->int_status;
 }
 
-static bool sdmmc_intr_check_status(sdmmc_t *sdmmc, uint16_t status_mask)
+static int sdmmc_intr_check(sdmmc_t *sdmmc, uint16_t *status_out, uint16_t status_mask)
 {
-    bool is_masked = (sdmmc->regs->int_status & status_mask);
+    uint32_t int_status = sdmmc->regs->int_status;
     
-    /* Mask status. */
-    if (is_masked)
-        sdmmc->regs->int_status &= status_mask;
+    sdmmc_debug(sdmmc, "INTSTS: %08X", int_status);
     
-    return is_masked;
-}
-   
-static bool sdmmc_intr_check_error(sdmmc_t *sdmmc)
-{
-    bool is_error = (sdmmc->regs->int_status & TEGRA_MMC_NORINTSTS_ERR_INTERRUPT);
+    /* Return the status, if necessary. */
+    if (status_out)
+        *status_out = (int_status & 0xFFFF);
     
-    /* Refresh status. */
-    if (is_error)
-        sdmmc->regs->int_status = sdmmc->regs->int_status;
+    if (int_status & TEGRA_MMC_NORINTSTS_ERR_INTERRUPT)
+    {
+        /* Acknowledge error by refreshing status. */
+        sdmmc->regs->int_status = int_status;
+        return -1;
+    }
+    else if (int_status & status_mask)
+    {
+        /* Mask the status. */
+        sdmmc->regs->int_status = (int_status & status_mask);
+        return 1;
+    }
     
-    return is_error;
+    return 0;
 }
 
 static int sdmmc_dma_init(sdmmc_t *sdmmc, sdmmc_request_t *req)
@@ -1442,15 +1443,23 @@ static int sdmmc_dma_update(sdmmc_t *sdmmc)
         /* Watch over the DMA transfer. */
         while (!is_timeout) 
         {
+            /* Check interrupts. */
+            uint16_t intr_status = 0;
+            int intr_res = sdmmc_intr_check(sdmmc, &intr_status, TEGRA_MMC_NORINTSTS_XFER_COMPLETE | TEGRA_MMC_NORINTSTS_DMA_INTERRUPT);
+            
             /* An error has been raised. Reset. */
-            if (sdmmc_intr_check_error(sdmmc))
+            if (intr_res < 0)
             {
                 sdmmc_do_sw_reset(sdmmc);
                 return 0;
             }
             
+            /* Transfer is over. */
+            if (intr_status & TEGRA_MMC_NORINTSTS_XFER_COMPLETE)
+                return 1;
+            
             /* We have a DMA interrupt. Restart the transfer where it was interrupted. */
-            if (sdmmc_intr_check_status(sdmmc, TEGRA_MMC_NORINTSTS_DMA_INTERRUPT))
+            if (intr_status & TEGRA_MMC_NORINTSTS_DMA_INTERRUPT)
             {
                 if (sdmmc->use_adma)
                 {
@@ -1467,10 +1476,6 @@ static int sdmmc_dma_update(sdmmc_t *sdmmc)
                 sdmmc->next_dma_addr += 0x80000;
             }
             
-            /* Transfer is over. */
-            if (sdmmc_intr_check_status(sdmmc, TEGRA_MMC_NORINTSTS_XFER_COMPLETE))
-                return 1;
-            
             /* Keep checking if timeout expired. */
             is_timeout = (get_time_since(timebase) > 2000000);            
         }
@@ -1526,12 +1531,15 @@ static int sdmmc_wait_for_cmd(sdmmc_t *sdmmc)
 
     /* Wait for CMD to finish. */
     while (!is_err && !is_timeout) {
+        /* Check interrupts. */
+        int intr_res = sdmmc_intr_check(sdmmc, 0, TEGRA_MMC_NORINTSTS_CMD_COMPLETE);
+        
         /* Command is done. */
-        if (sdmmc_intr_check_status(sdmmc, TEGRA_MMC_NORINTSTS_CMD_COMPLETE))
+        if (intr_res > 0)
             return 1;
         
         /* Check for any raised errors. */
-        is_err = sdmmc_intr_check_error(sdmmc);
+        is_err = (intr_res < 0);
         
         /* Keep checking if timeout expired. */
         is_timeout = (get_time_since(timebase) > 2000000);
@@ -1642,6 +1650,7 @@ int sdmmc_send_cmd(sdmmc_t *sdmmc, sdmmc_command_t *cmd, sdmmc_request_t *req, u
         is_dma = true;
         dma_blkcnt = sdmmc_dma_init(sdmmc, req);
         
+        /* Abort in case initialization failed. */
         if (!dma_blkcnt)
         {
             sdmmc_error(sdmmc, "Failed to initialize the DMA transfer!");
@@ -1669,12 +1678,14 @@ int sdmmc_send_cmd(sdmmc_t *sdmmc, sdmmc_command_t *cmd, sdmmc_request_t *req, u
         /* Save response, if necessary. */
         sdmmc_save_response(sdmmc, cmd->flags);
         
-        /* Process the DMA request. */
+        /* Update the DMA request. */
         if (req)
         {
+            /* Disable interrupts and abort in case updating failed. */
             if (!sdmmc_dma_update(sdmmc))
             {
-                sdmmc_error(sdmmc, "Failed to process the DMA transfer!");
+                sdmmc_warn(sdmmc, "Failed to update the DMA transfer!");
+                sdmmc_intr_disable(sdmmc);
                 return 0;
             }
             
@@ -1840,7 +1851,7 @@ static int sdmmc_send_tuning(sdmmc_t *sdmmc, uint32_t opcode)
     while (!is_timeout)
     {
         /* Buffer Read Ready was asserted. */
-        if (sdmmc_intr_check_status(sdmmc, TEGRA_MMC_NORINTSTSEN_BUFFER_READ_READY))
+        if (sdmmc_intr_check(sdmmc, 0, TEGRA_MMC_NORINTSTSEN_BUFFER_READ_READY) > 0)
         {
             /* Manually disable the Buffer Read Ready interrupt. */
             sdmmc->regs->int_enable &= ~(TEGRA_MMC_NORINTSTSEN_BUFFER_READ_READY);

fusee/fusee-secondary/src/start.s

@@ -97,11 +97,18 @@ _metadata:
 #define CONTENT_TYPE_KLD 9
 #define CONTENT_TYPE_KRN 10
 
+#define CONTENT_FLAG_NONE          (0 << 0)
+
+#define CONTENT_FLAG0_EXPERIMENTAL (1 << 0)
+
 _content_headers:
 /* ams_mitm content header */
 .word __ams_mitm_kip_start__
 .word __ams_mitm_kip_size__
-.word CONTENT_TYPE_KIP
+.byte CONTENT_TYPE_KIP
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
 .word 0xCCCCCCCC
 .asciz "ams_mitm"
 .align 5
@@ -109,7 +116,10 @@ _content_headers:
 /* boot content header */
 .word __boot_kip_start__
 .word __boot_kip_size__
-.word CONTENT_TYPE_KIP
+.byte CONTENT_TYPE_KIP
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
 .word 0xCCCCCCCC
 .asciz "boot"
 .align 5
@@ -117,7 +127,10 @@ _content_headers:
 /* exosphere content header */
 .word __exosphere_bin_start__
 .word __exosphere_bin_size__
-.word CONTENT_TYPE_EXO
+.byte CONTENT_TYPE_EXO
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
 .word 0xCCCCCCCC
 .asciz "exosphere"
 .align 5
@@ -125,7 +138,10 @@ _content_headers:
 /* fusee_primary content header */
 .word __fusee_primary_bin_start__
 .word __fusee_primary_bin_size__
-.word CONTENT_TYPE_FSP
+.byte CONTENT_TYPE_FSP
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
 .word 0xCCCCCCCC
 .asciz "fusee_primary"
 .align 5
@@ -133,15 +149,21 @@ _content_headers:
 /* loader content header */
 .word __loader_kip_start__
 .word __loader_kip_size__
-.word CONTENT_TYPE_KIP
+.byte CONTENT_TYPE_KIP
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
 .word 0xCCCCCCCC
-.asciz "loader"
+.asciz "Loader"
 .align 5
 
 /* lp0fw content header */
 .word __lp0fw_bin_start__
 .word __lp0fw_bin_size__
-.word CONTENT_TYPE_WBT
+.byte CONTENT_TYPE_WBT
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
 .word 0xCCCCCCCC
 .asciz "lp0fw"
 .align 5
@@ -149,15 +171,21 @@ _content_headers:
 /* pm content header */
 .word __pm_kip_start__
 .word __pm_kip_size__
-.word CONTENT_TYPE_KIP
+.byte CONTENT_TYPE_KIP
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
 .word 0xCCCCCCCC
-.asciz "pm"
+.asciz "ProcessManager"
 .align 5
 
 /* rebootstub content header */
 .word __rebootstub_bin_start__
 .word __rebootstub_bin_size__
-.word CONTENT_TYPE_RBT
+.byte CONTENT_TYPE_RBT
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
 .word 0xCCCCCCCC
 .asciz "rebootstub"
 .align 5
@@ -165,7 +193,10 @@ _content_headers:
 /* sept_primary content header */
 .word __sept_primary_bin_start__
 .word __sept_primary_bin_size__
-.word CONTENT_TYPE_SP1
+.byte CONTENT_TYPE_SP1
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
 .word 0xCCCCCCCC
 .asciz "sept_primary"
 .align 5
@@ -173,7 +204,10 @@ _content_headers:
 /* sept_secondary 00 content header */
 .word __sept_secondary_00_enc_start__
 .word __sept_secondary_00_enc_size__
-.word CONTENT_TYPE_SP2
+.byte CONTENT_TYPE_SP2
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
 .word 0xCCCCCCCC
 .asciz "septsecondary00"
 .align 5
@@ -181,7 +215,10 @@ _content_headers:
 /* sept_secondary 01 content header */
 .word __sept_secondary_01_enc_start__
 .word __sept_secondary_01_enc_size__
-.word CONTENT_TYPE_SP2
+.byte CONTENT_TYPE_SP2
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
 .word 0xCCCCCCCC
 .asciz "septsecondary01"
 .align 5
@@ -189,7 +226,10 @@ _content_headers:
 /* sm content header */
 .word __sm_kip_start__
 .word __sm_kip_size__
-.word CONTENT_TYPE_KIP
+.byte CONTENT_TYPE_KIP
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
 .word 0xCCCCCCCC
 .asciz "sm"
 .align 5
@@ -197,15 +237,32 @@ _content_headers:
 /* spl content header */
 .word __spl_kip_start__
 .word __spl_kip_size__
-.word CONTENT_TYPE_KIP
+.byte CONTENT_TYPE_KIP
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
 .word 0xCCCCCCCC
 .asciz "spl"
 .align 5
 
+/* ncm content header */
+.word __ncm_kip_start__
+.word __ncm_kip_size__
+.byte CONTENT_TYPE_KIP
+.byte CONTENT_FLAG0_EXPERIMENTAL
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
+.word 0xCCCCCCCC
+.asciz "NCM"
+.align 5
+
 /* emummc content header */
 .word __emummc_kip_start__
 .word __emummc_kip_size__
-.word CONTENT_TYPE_EMC
+.byte CONTENT_TYPE_EMC
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
 .word 0xCCCCCCCC
 .asciz "emummc"
 .align 5
@@ -213,7 +270,10 @@ _content_headers:
 /* kernel_ldr content header */
 .word __kernel_ldr_bin_start__
 .word __kernel_ldr_bin_size__
-.word CONTENT_TYPE_KLD
+.byte CONTENT_TYPE_KLD
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
 .word 0xCCCCCCCC
 .asciz "kernel_ldr"
 .align 5
@@ -221,7 +281,10 @@ _content_headers:
 /* splash_screen content header */
 .word __splash_screen_bmp_start__
 .word __splash_screen_bmp_size__
-.word CONTENT_TYPE_BMP
+.byte CONTENT_TYPE_BMP
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
+.byte CONTENT_FLAG_NONE
 .word 0xCCCCCCCC
 .asciz "splash_screen"
 .align 5

fusee/fusee-secondary/src/stratosphere.c

@@ -13,7 +13,7 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
- 
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <stdint.h>
@@ -30,6 +30,7 @@
 #define u8 uint8_t
 #define u32 uint32_t
 #include "loader_kip.h"
+#include "ncm_kip.h"
 #include "pm_kip.h"
 #include "sm_kip.h"
 #include "ams_mitm_kip.h"
@@ -47,9 +48,10 @@ static bool g_stratosphere_pm_enabled = true;
 static bool g_stratosphere_ams_mitm_enabled = true;
 static bool g_stratosphere_spl_enabled = true;
 static bool g_stratosphere_boot_enabled = true;
+static bool g_stratosphere_ncm_enabled = false;
 
-extern const uint8_t loader_kip[], pm_kip[], sm_kip[], spl_kip[], boot_kip[], ams_mitm_kip[];
-extern const uint32_t loader_kip_size, pm_kip_size, sm_kip_size, spl_kip_size, boot_kip_size, ams_mitm_kip_size;
+extern const uint8_t loader_kip[], pm_kip[], sm_kip[], spl_kip[], boot_kip[], ncm_kip[], ams_mitm_kip[];
+extern const uint32_t loader_kip_size, pm_kip_size, sm_kip_size, spl_kip_size, boot_kip_size, ncm_kip_size, ams_mitm_kip_size;
 
 static emummc_fs_ver_t g_fs_ver = FS_VER_1_0_0;
 
@@ -57,6 +59,19 @@ emummc_fs_ver_t stratosphere_get_fs_version(void) {
     return g_fs_ver;
 }
 
+void stratosphere_enable_ncm(void) {
+    /* The Atmosphere team believes our implementation of NCM to be extremely accurate, */
+    /* and does not think it likely there is any possibility of undesirable behavior */
+    /* when using the NCM reimplementation. However, because NCM manages critical save games */
+    /* the implementation will default to off for some time, until the code has been thoroughly */
+    /* tested in practice. */
+
+    /* PLEASE NOTE: The default behavior will be NCM on in a future atmosphere release, */
+    /* and this opt-in functionality will be removed at that time. */
+
+    g_stratosphere_ncm_enabled = true;
+}
+
 /* GCC doesn't consider the size as const... we have to write it ourselves. */
 
 ini1_header_t *stratosphere_get_ini1(uint32_t target_firmware) {
@@ -68,38 +83,43 @@ ini1_header_t *stratosphere_get_ini1(uint32_t target_firmware) {
     }
 
     size_t size = sizeof(ini1_header_t);
-    
+
     /* Calculate our processes' sizes. */
     if (g_stratosphere_loader_enabled) {
         size += loader_kip_size;
         num_processes++;
     }
-    
+
     if (g_stratosphere_pm_enabled) {
         size += pm_kip_size;
         num_processes++;
     }
-    
+
     if (g_stratosphere_sm_enabled) {
         size += sm_kip_size;
         num_processes++;
     }
-    
+
     if (g_stratosphere_spl_enabled) {
         size += spl_kip_size;
         num_processes++;
     }
-    
+
     if (g_stratosphere_ams_mitm_enabled) {
         size += ams_mitm_kip_size;
         num_processes++;
     }
-    
+
+    if (g_stratosphere_ncm_enabled) {
+        size += ncm_kip_size;
+        num_processes++;
+    }
+
     if (g_stratosphere_boot_enabled) {
         size += boot_kip_size;
         num_processes++;
     }
-    
+
     g_stratosphere_ini1 = (ini1_header_t *)malloc(size);
 
     if (g_stratosphere_ini1 == NULL) {
@@ -134,6 +154,11 @@ ini1_header_t *stratosphere_get_ini1(uint32_t target_firmware) {
         data += spl_kip_size;
     }
 
+    if (g_stratosphere_ncm_enabled) {
+        memcpy(data, ncm_kip, ncm_kip_size);
+        data += ncm_kip_size;
+    }
+
     if (g_stratosphere_ams_mitm_enabled) {
         memcpy(data, ams_mitm_kip, ams_mitm_kip_size);
         data += ams_mitm_kip_size;
@@ -143,7 +168,7 @@ ini1_header_t *stratosphere_get_ini1(uint32_t target_firmware) {
         memcpy(data, boot_kip, boot_kip_size);
         data += boot_kip_size;
     }
-    
+
     return g_stratosphere_ini1;
 }
 
@@ -160,26 +185,26 @@ void stratosphere_free_ini1(void) {
 
 static void try_add_sd_kip(ini1_header_t *ini1, const char *kip_path) {
     size_t file_size = get_file_size(kip_path);
-    
+
     if (ini1->size + file_size > PACKAGE2_SIZE_MAX) {
         fatal_error("Failed to load %s: INI1 would be too large!\n", kip_path);
     }
-    
+
     kip1_header_t kip_header;
     if (read_from_file(&kip_header, sizeof(kip_header), kip_path) != sizeof(kip_header) || kip_header.magic != MAGIC_KIP1) {
         return;
     }
-    
+
     size_t kip_size = kip1_get_size_from_header(&kip_header);
     if (kip_size > file_size) {
         fatal_error("Failed to load %s: KIP size is corrupt!\n", kip_path);
     }
-    
+
     if (read_from_file(ini1->kip_data + ini1->size - sizeof(ini1_header_t), kip_size, kip_path) != kip_size) {
         /* TODO: is this error fatal? */
         return;
     }
-    
+
     ini1->size += kip_size;
     ini1->num_processes++;
 }
@@ -189,24 +214,24 @@ static kip1_header_t *inject_emummc_kip(kip1_header_t *fs_kip, kip1_header_t *em
     size_t fs_kip_size, emummc_kip_size;
     fs_kip = kip1_uncompress(fs_kip, &fs_kip_size);
     emummc_kip = kip1_uncompress(emummc_kip, &emummc_kip_size);
-    
+
     /* Allocate kip. */
     kip1_header_t *injected_kip = calloc(1, fs_kip_size + emummc_kip_size);
     if (injected_kip == NULL) {
         fatal_error("Failed to allocate memory for emummc kip injection!");
     }
     memcpy(injected_kip, fs_kip, sizeof(*fs_kip));
-    
+
     const size_t fs_contents_size = kip1_get_size_from_header(fs_kip) - sizeof(kip1_header_t);
-    
+
     const size_t emummc_data_size = emummc_kip->section_headers[3].out_offset + emummc_kip->section_headers[3].out_size;
     if (emummc_data_size & 0xFFF) {
         fatal_error("Invalid emummc kip!");
     }
-    
+
     /* Copy over capabilities. */
     memcpy(injected_kip->capabilities, emummc_kip->capabilities, sizeof(emummc_kip->capabilities));
-    
+
     /* Add extra cap for 1.0.0 */
     if (stratosphere_get_fs_version() == FS_VER_1_0_0) {
         for (size_t i = 0; i < 0x20; i++) {
@@ -217,14 +242,14 @@ static kip1_header_t *inject_emummc_kip(kip1_header_t *fs_kip, kip1_header_t *em
             }
         }
     }
-    
+
     /* Update sections. */
     injected_kip->section_headers[0].out_size += emummc_data_size;
     injected_kip->section_headers[0].compressed_size += emummc_data_size;
     for (size_t i = 1; i < 4; i++) {
         injected_kip->section_headers[i].out_offset += emummc_data_size;
     }
-    
+
     /* Copy data. */
     {
         size_t ofs = 0;
@@ -234,22 +259,22 @@ static kip1_header_t *inject_emummc_kip(kip1_header_t *fs_kip, kip1_header_t *em
         }
     }
     memcpy(injected_kip->data + emummc_data_size, fs_kip->data, fs_contents_size);
-    
+
     return injected_kip;
 }
 
-ini1_header_t *stratosphere_get_sd_files_ini1(void) {    
+ini1_header_t *stratosphere_get_sd_files_ini1(void) {
     if (g_sd_files_ini1 != NULL) {
         return g_sd_files_ini1;
     }
-    
+
     /* Allocate space. */
     g_sd_files_ini1 = (ini1_header_t *)malloc(PACKAGE2_SIZE_MAX);
     g_sd_files_ini1->magic = MAGIC_INI1;
     g_sd_files_ini1->size = sizeof(ini1_header_t);
     g_sd_files_ini1->num_processes = 0;
     g_sd_files_ini1->_0xC = 0;
-    
+
     /* Load all kips from /atmosphere/kips/<*>.kip or /atmosphere/kips/<*>/<*>.kip. */
     DIR *kips_dir = opendir("atmosphere/kips");
     struct dirent *ent;
@@ -258,15 +283,15 @@ ini1_header_t *stratosphere_get_sd_files_ini1(void) {
             if (strcmp(ent->d_name, ".") == 0 || strcmp(ent->d_name, "..") == 0) {
                 continue;
             }
-            
+
             char kip_path[0x301] = {0};
             snprintf(kip_path, 0x300, "atmosphere/kips/%s", ent->d_name);
-            
+
             struct stat kip_stat;
             if (stat(kip_path, &kip_stat) == -1) {
                 continue;
             }
-            
+
             if ((kip_stat.st_mode & S_IFMT) == S_IFREG) {
                 /* If file, add to ini1. */
                 try_add_sd_kip(g_sd_files_ini1, kip_path);
@@ -279,15 +304,15 @@ ini1_header_t *stratosphere_get_sd_files_ini1(void) {
                         if (strcmp(sub_ent->d_name, ".") == 0 || strcmp(sub_ent->d_name, "..") == 0) {
                             continue;
                         }
-                        
+
                         /* Reuse kip path variable. */
                         memset(kip_path, 0, sizeof(kip_path));
                         snprintf(kip_path, 0x300, "atmosphere/kips/%s/%s", ent->d_name, sub_ent->d_name);
-                        
+
                         if (stat(kip_path, &kip_stat) == -1) {
                             continue;
                         }
-                        
+
                         if ((kip_stat.st_mode & S_IFMT) == S_IFREG) {
                             /* If file, add to ini1. */
                             try_add_sd_kip(g_sd_files_ini1, kip_path);
@@ -299,7 +324,7 @@ ini1_header_t *stratosphere_get_sd_files_ini1(void) {
         }
         closedir(kips_dir);
     }
-    
+
     return g_sd_files_ini1;
 }
 
@@ -344,7 +369,7 @@ ini1_header_t *stratosphere_merge_inis(ini1_header_t **inis, size_t num_inis, vo
             }
 
             offset += kip1_get_size_from_header(current_kip);
-            
+
             bool already_loaded = false;
             for (uint32_t j = 0; j < merged->num_processes; j++) {
                 if (process_list[j] == current_kip->title_id) {
@@ -355,20 +380,20 @@ ini1_header_t *stratosphere_merge_inis(ini1_header_t **inis, size_t num_inis, vo
             if (already_loaded) {
                 continue;
             }
-            
+
             print(SCREEN_LOG_LEVEL_MANDATORY, "[NXBOOT]: Loading KIP %08x%08x...\n", (uint32_t)(current_kip->title_id >> 32), (uint32_t)current_kip->title_id);
 
             size_t current_kip_size = kip1_get_size_from_header(current_kip);
             if (current_kip_size > remaining_size) {
                 fatal_error("Not enough space for all the KIP1s!\n");
             }
-            
+
             kip1_header_t *patched_kip = apply_kip_ips_patches(current_kip, current_kip_size, &g_fs_ver);
-            
+
             if (current_kip->title_id == FS_TITLE_ID && emummc != NULL) {
                 patched_kip = inject_emummc_kip(patched_kip != NULL ? patched_kip : current_kip, (kip1_header_t *)emummc);
             }
-            
+
             if (patched_kip != NULL) {
                 size_t patched_kip_size = kip1_get_size_from_header(patched_kip);
                 if (patched_kip_size > remaining_size) {
@@ -377,9 +402,9 @@ ini1_header_t *stratosphere_merge_inis(ini1_header_t **inis, size_t num_inis, vo
                 memcpy(current_dst_kip, patched_kip, patched_kip_size);
                 remaining_size -= patched_kip_size;
                 current_dst_kip += patched_kip_size;
-                
+
                 free(patched_kip);
-            } else {   
+            } else {
                 memcpy(current_dst_kip, current_kip, current_kip_size);
                 remaining_size -= current_kip_size;
                 current_dst_kip += current_kip_size;

fusee/fusee-secondary/src/stratosphere.h

@@ -13,7 +13,7 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
- 
+
 #ifndef FUSEE_STRATOSPHERE_H
 #define FUSEE_STRATOSPHERE_H
 
@@ -30,6 +30,8 @@ ini1_header_t *stratosphere_get_ini1(uint32_t target_firmware);
 ini1_header_t *stratosphere_get_sd_files_ini1(void);
 void stratosphere_free_ini1(void);
 
+void stratosphere_enable_ncm(void);
+
 emummc_fs_ver_t stratosphere_get_fs_version(void);
 
 ini1_header_t *stratosphere_merge_inis(ini1_header_t **inis, unsigned int num_inis, void *emummc, size_t emummc_size);
@@ -37,8 +39,10 @@ ini1_header_t *stratosphere_merge_inis(ini1_header_t **inis, unsigned int num_in
 typedef struct {
     bool has_nogc_config;
     bool enable_nogc;
+    bool ncm_enabled;
 } stratosphere_cfg_t;
 
 #define STRATOSPHERE_NOGC_KEY "nogc"
+#define STRATOSPHERE_ENABLE_NCM_KEY "enable_ncm"
 
 #endif

libraries/.gitrepo

@@ -6,7 +6,7 @@
 [subrepo]
 	remote = https://github.com/Atmosphere-NX/Atmosphere-libs
 	branch = master
-	commit = 08c9b3cbf85471fe6adb5f42ba9f03357a5f633a
-	parent = 814c9d1cfb54c192bc1deb140ab476c9f3dd0edf
+	commit = c1fe12fcbd661ae47673af5f224804c92eebdbb5
+	parent = 98cc051387515977dcdede8ab0f1b09077f2f24b
 	method = merge
-	cmdver = 0.4.0
+	cmdver = 0.4.1

libraries/README.md

@@ -10,7 +10,8 @@ This software is licensed under the terms of the GPLv2, with exemptions for spec
 You can find a copy of the license in the [LICENSE file](LICENSE).
 
 Exemptions:
-* The [yuzu emulator project](https://github.com/yuzu-emu/yuzu) is exempt from GPLv2 licensing and may (at its option) instead license any source code authored for the atmosphere-libs project as GPLv2 or later.
+* The [yuzu Nintendo Switch emulator](https://github.com/yuzu-emu/yuzu) and the [Ryujinx Team and Contributors](https://github.com/orgs/Ryujinx) are exempt from GPLv2 licensing. They are permitted, each at their individual discretion, to instead license any source code authored for the atmosphere-libs project as either GPLv2 or later or the [MIT license](https://github.com/Atmosphere-NX/Atmosphere/blob/master/docs/licensing_exemptions/MIT_LICENSE). In doing so, they may alter, supplement, or entirely remove the copyright notice for each file they choose to relicense. Neither the Atmosphère project nor its individual contributors shall assert their moral rights against any of the aforementioned projects.
+* [Nintendo](https://github.com/Nintendo) is exempt from GPLv2 licensing and may (at its option) instead license any source code authored for the atmosphere-libs project under the Zero-Clause BSD license.
 
 Credits
 =====

libraries/config/arch/arm64/arch.mk

@@ -5,7 +5,7 @@ endif
 include $(DEVKITPRO)/devkitA64/base_rules
 
 export ATMOSPHERE_DEFINES  += -DATMOSPHERE_ARCH_ARM64
-export ATMOSPHERE_SETTINGS += -march=armv8-a -mtp=soft
+export ATMOSPHERE_SETTINGS += -march=armv8-a+crc+crypto -mtp=soft
 export ATMOSPHERE_CFLAGS   +=
 export ATMOSPHERE_CXXFLAGS +=
 export ATMOSPHERE_ASFLAGS  +=

libraries/config/board/nintendo/nx/board.mk

@@ -1,4 +1,4 @@
-export ATMOSPHERE_DEFINES  += -DATMOSPHERE_BOARD_NINTENDO_SWITCH -D__SWITCH__
+export ATMOSPHERE_DEFINES  += -DATMOSPHERE_BOARD_NINTENDO_NX -D__SWITCH__
 export ATMOSPHERE_SETTINGS +=
 export ATMOSPHERE_CFLAGS   +=
 export ATMOSPHERE_CXXFLAGS +=

libraries/config/common.mk

@@ -24,11 +24,11 @@ export ATMOSPHERE_ASFLAGS  :=
 
 ifeq ($(ATMOSPHERE_BOARD),nx-hac-001)
 export ATMOSPHERE_ARCH_DIR   := arch/arm64
-export ATMOSPHERE_BOARD_DIR  := board/nintendo/switch
+export ATMOSPHERE_BOARD_DIR  := board/nintendo/nx
 export ATMOSPHERE_OS_DIR     := os/horizon
 
 export ATMOSPHERE_ARCH_NAME  := arm64
-export ATMOSPHERE_BOARD_NAME := nintendo_switch
+export ATMOSPHERE_BOARD_NAME := nintendo_nx
 export ATMOSPHERE_OS_NAME    := horizon
 endif
 
@@ -76,14 +76,17 @@ TARGET       := $(notdir $(CURDIR))
 BUILD        := build
 DATA         := data
 INCLUDES     := include
-SOURCES      ?= $(foreach d,$(filter-out source/arch source/board,$(wildcard source)),$(call DIR_WILDCARD,$d) $d)
 
-ifneq ($(strip $(wildcard source/$(ATMOSPHERE_ARCH_DIR)/.*)),)
-SOURCES += $(call DIR_WILDCARD,source/$(ATMOSPHERE_ARCH_DIR))
-endif
-ifneq ($(strip $(wildcard source/$(ATMOSPHERE_BOARD_DIR)/.*)),)
-SOURCES += $(call DIR_WILDCARD,source/$(ATMOSPHERE_BOARD_DIR))
-endif
-ifneq ($(strip $(wildcard source/$(ATMOSPHERE_OS_DIR)/.*)),)
-SOURCES += $(call DIR_WILDCARD,source/$(ATMOSPHERE_OS_DIR))
-endif
+GENERAL_SOURCE_DIRS=$1 $(foreach d,$(filter-out $1/arch $1/board $1,$(wildcard $1/*)),$(if $(wildcard $d/.),$(call DIR_WILDCARD,$d) $d,))
+SPECIFIC_SOURCE_DIRS=$(if $(wildcard $1/$2/.*),$1/$2 $(call DIR_WILDCARD,$1/$2),)
+ALL_SOURCE_DIRS=$(call GENERAL_SOURCE_DIRS,$1) $(call SPECIFIC_SOURCE_DIRS,$1,$(ATMOSPHERE_ARCH_DIR)) $(call SPECIFIC_SOURCE_DIRS,$1,$(ATMOSPHERE_BOARD_DIR)) $(call SPECIFIC_SOURCE_DIRS,$1,$(ATMOSPHERE_OS_DIR))
+
+SOURCES      ?= $(call ALL_SOURCE_DIRS,source)
+
+#---------------------------------------------------------------------------------
+# Rules for compiling pre-compiled headers
+#---------------------------------------------------------------------------------
+%.gch: %.hpp
+	@echo $<
+	$(CXX) -w -x c++-header -MMD -MP -MF $(DEPSDIR)/$*.d $(CXXFLAGS) -c $< -o $@ $(ERROR_FILTER)
+	@cp $@ $(<).gch

libraries/config/templates/mesosphere.mk

@@ -7,12 +7,12 @@ include  $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/../common.mk
 # options for code generation
 #---------------------------------------------------------------------------------
 export DEFINES     := $(ATMOSPHERE_DEFINES) -DATMOSPHERE_IS_MESOSPHERE
-export SETTINGS    := $(ATMOSPHERE_SETTINGS) -O2 -mgeneral-regs-only -ffixed-x18 -Werror
+export SETTINGS    := $(ATMOSPHERE_SETTINGS) -O2 -mgeneral-regs-only -ffixed-x18 -Werror -fno-non-call-exceptions
 export CFLAGS      := $(ATMOSPHERE_CFLAGS) $(SETTINGS) $(DEFINES) $(INCLUDE)
-export CXXFLAGS    := $(CFLAGS) $(ATMOSPHERE_CXXFLAGS)
+export CXXFLAGS    := $(CFLAGS) $(ATMOSPHERE_CXXFLAGS) -fno-use-cxa-atexit
 export ASFLAGS     := $(ATMOSPHERE_ASFLAGS) $(SETTINGS) $(DEFINES)
 
-export LDFLAGS	=	-specs=$(TOPDIR)/kernel_ldr.specs -nostdlib -nostartfiles -g $(SETTINGS) -Wl,-Map,$(notdir $*.map)
+export LDFLAGS	=	-specs=$(TOPDIR)/$(notdir $(TOPDIR)).specs -nostdlib -nostartfiles -g $(SETTINGS) -Wl,-Map,$(notdir $*.map) -Wl,-z,relro,-z,now
 
 export CXXWRAPS := -Wl,--wrap,__cxa_pure_virtual \
 			-Wl,--wrap,__cxa_throw \
@@ -25,7 +25,6 @@ export CXXWRAPS := -Wl,--wrap,__cxa_pure_virtual \
 			-Wl,--wrap,__cxa_call_terminate \
 			-Wl,--wrap,__gxx_personality_v0 \
 			-Wl,--wrap,_Unwind_Resume \
-			-Wl,--wrap,_Unwind_Resume \
 			-Wl,--wrap,_ZSt19__throw_logic_errorPKc \
 			-Wl,--wrap,_ZSt20__throw_length_errorPKc \
 			-Wl,--wrap,_ZNSt11logic_errorC2EPKc

libraries/config/templates/stratosphere.mk

@@ -15,11 +15,11 @@ include $(DEVKITPRO)/libnx/switch_rules
 #---------------------------------------------------------------------------------
 # options for code generation
 #---------------------------------------------------------------------------------
-export DEFINES     := $(ATMOSPHERE_DEFINES) -DATMOSPHERE_IS_STRATOSPHERE
-export SETTINGS    := $(ATMOSPHERE_SETTINGS) -O2
-export CFLAGS      := $(ATMOSPHERE_CFLAGS) $(SETTINGS) $(DEFINES) $(INCLUDE)
-export CXXFLAGS    := $(CFLAGS) $(ATMOSPHERE_CXXFLAGS)
-export ASFLAGS     := $(ATMOSPHERE_ASFLAGS) $(SETTINGS) $(DEFINES)
+export DEFINES     = $(ATMOSPHERE_DEFINES) -DATMOSPHERE_IS_STRATOSPHERE -D_GNU_SOURCE
+export SETTINGS    = $(ATMOSPHERE_SETTINGS) -O2
+export CFLAGS      = $(ATMOSPHERE_CFLAGS) $(SETTINGS) $(DEFINES) $(INCLUDE)
+export CXXFLAGS    = $(CFLAGS) $(ATMOSPHERE_CXXFLAGS)
+export ASFLAGS     = $(ATMOSPHERE_ASFLAGS) $(SETTINGS) $(DEFINES)
 
 export CXXWRAPS := -Wl,--wrap,__cxa_pure_virtual \
 			-Wl,--wrap,__cxa_throw \
@@ -36,17 +36,17 @@ export CXXWRAPS := -Wl,--wrap,__cxa_pure_virtual \
 			-Wl,--wrap,_ZSt20__throw_length_errorPKc \
 			-Wl,--wrap,_ZNSt11logic_errorC2EPKc
 
-export LDFLAGS     := -specs=$(DEVKITPRO)/libnx/switch.specs $(SETTINGS) $(CXXWRAPS) -Wl,-Map,$(notdir $*.map)
+export LDFLAGS     = -specs=$(DEVKITPRO)/libnx/switch.specs $(SETTINGS) $(CXXWRAPS) -Wl,-Map,$(notdir $*.map)
 
-export LIBS	:= -lstratosphere -lnx
+export LIBS	= -lstratosphere -lnx
 
 #---------------------------------------------------------------------------------
 # list of directories containing libraries, this must be the top level containing
 # include and lib
 #---------------------------------------------------------------------------------
-export LIBDIRS	:= $(PORTLIBS) $(LIBNX) $(ATMOSPHERE_LIBRARIES_DIR)/libvapours $(ATMOSPHERE_LIBRARIES_DIR)/libstratosphere
+export LIBDIRS	= $(PORTLIBS) $(LIBNX) $(ATMOSPHERE_LIBRARIES_DIR)/libvapours $(ATMOSPHERE_LIBRARIES_DIR)/libstratosphere
 
 #---------------------------------------------------------------------------------
 # stratosphere sysmodules may (but usually do not) have an exefs source dir
 #---------------------------------------------------------------------------------
-export EXEFS_SRC := exefs_src
+export EXEFS_SRC = exefs_src

libraries/libmesosphere/Makefile

@@ -6,12 +6,17 @@ include  $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/../config/common.mk
 #---------------------------------------------------------------------------------
 # options for code generation
 #---------------------------------------------------------------------------------
+#PRECOMPILED_HEADERS := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/include/mesosphere.hpp
+PRECOMPILED_HEADERS :=
+
 DEFINES     := $(ATMOSPHERE_DEFINES) -DATMOSPHERE_IS_MESOSPHERE
-SETTINGS    := $(ATMOSPHERE_SETTINGS) -O2 -mgeneral-regs-only -ffixed-x18 -Werror
+SETTINGS    := $(ATMOSPHERE_SETTINGS) -O2 -mgeneral-regs-only -ffixed-x18 -Werror -fno-non-call-exceptions
 CFLAGS      := $(ATMOSPHERE_CFLAGS) $(SETTINGS) $(DEFINES) $(INCLUDE)
-CXXFLAGS    := $(CFLAGS) $(ATMOSPHERE_CXXFLAGS) -flto
+CXXFLAGS    := $(CFLAGS) $(ATMOSPHERE_CXXFLAGS) -fno-use-cxa-atexit -flto
 ASFLAGS     := $(ATMOSPHERE_ASFLAGS) $(SETTINGS)
 
+SOURCES     += $(call ALL_SOURCE_DIRS,../libvapours/source)
+
 LIBS        :=
 
 #---------------------------------------------------------------------------------
@@ -27,7 +32,7 @@ LIBDIRS	:= $(ATMOSPHERE_LIBRARIES_DIR)/libvapours
 ifneq ($(BUILD),$(notdir $(CURDIR)))
 #---------------------------------------------------------------------------------
 
-export VPATH	:=	$(foreach dir,$(SOURCES),$(CURDIR)/$(dir)) \
+export VPATH	:=	$(foreach dir,$(SOURCES),$(CURDIR)/$(dir)) $(CURDIR)/include \
 			$(foreach dir,$(DATA),$(CURDIR)/$(dir))
 
 CFILES      :=	$(foreach dir,$(SOURCES),$(filter-out $(notdir $(wildcard $(dir)/*.arch.*.c)) $(notdir $(wildcard $(dir)/*.board.*.c)) $(notdir $(wildcard $(dir)/*.os.*.c)), \
@@ -64,6 +69,7 @@ endif
 
 export OFILES_BIN	:=	$(addsuffix .o,$(BINFILES))
 export OFILES_SRC	:=	$(CPPFILES:.cpp=.o) $(CFILES:.c=.o) $(SFILES:.s=.o)
+export GCH_FILES    :=	$(foreach hdr,$(PRECOMPILED_HEADERS:.hpp=.gch),$(notdir $(hdr)))
 export OFILES 	:=	$(OFILES_BIN) $(OFILES_SRC)
 export HFILES_BIN	:=	$(addsuffix .h,$(subst .,_,$(BINFILES)))
 
@@ -105,13 +111,15 @@ clean:
 #---------------------------------------------------------------------------------
 else
 
-DEPENDS	:=	$(OFILES:.o=.d)
+DEPENDS	:=	$(OFILES:.o=.d) $(GCH_FILES:.gch=.d)
 
 #---------------------------------------------------------------------------------
 # main targets
 #---------------------------------------------------------------------------------
 $(OUTPUT)	:	$(OFILES)
 
+$(filter-out kern_svc_tables.o, $(OFILES))	:	$(GCH_FILES)
+
 $(OFILES_SRC)	: $(HFILES_BIN)
 
 #---------------------------------------------------------------------------------

libraries/libmesosphere/README.md

@@ -10,7 +10,8 @@ This software is licensed under the terms of the GPLv2, with exemptions for spec
 You can find a copy of the license in the [LICENSE file](LICENSE).
 
 Exemptions:
-* The [yuzu emulator project](https://github.com/yuzu-emu/yuzu) is exempt from GPLv2 licensing and may (at its option) instead license any source code authored for the libmesosphere project as GPLv2 or later.
+* The [yuzu Nintendo Switch emulator](https://github.com/yuzu-emu/yuzu) and the [Ryujinx Team and Contributors](https://github.com/orgs/Ryujinx) are exempt from GPLv2 licensing. They are permitted, each at their individual discretion, to instead license any source code authored for the libmesosphere project as either GPLv2 or later or the [MIT license](https://github.com/Atmosphere-NX/Atmosphere/blob/master/docs/licensing_exemptions/MIT_LICENSE). In doing so, they may alter, supplement, or entirely remove the copyright notice for each file they choose to relicense. Neither the Atmosphère project nor its individual contributors shall assert their moral rights against any of the aforementioned projects.
+* [Nintendo](https://github.com/Nintendo) is exempt from GPLv2 licensing and may (at its option) instead license any source code authored for the libmesosphere project under the Zero-Clause BSD license.
 
 Credits
 =====

libraries/libmesosphere/include/mesosphere.hpp

@@ -19,20 +19,70 @@
 #include <vapours.hpp>
 
 /* First, pull in core macros (panic, etc). */
-#include "mesosphere/kern_panic.hpp"
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_panic.hpp>
 
 /* Primitive types. */
-#include "mesosphere/kern_k_typed_address.hpp"
-#include "mesosphere/kern_initial_process.hpp"
+#include <mesosphere/kern_k_typed_address.hpp>
+#include <mesosphere/kern_initial_process.hpp>
+#include <mesosphere/kern_k_exception_context.hpp>
 
 /* Core pre-initialization includes. */
-#include "mesosphere/kern_select_cpu.hpp"
+#include <mesosphere/kern_select_cpu.hpp>
+#include <mesosphere/kern_select_system_control.hpp>
+#include <mesosphere/kern_k_target_system.hpp>
 
 /* Initialization headers. */
-#include "mesosphere/init/kern_init_elf.hpp"
-#include "mesosphere/init/kern_init_layout.hpp"
-#include "mesosphere/init/kern_init_page_table_select.hpp"
+#include <mesosphere/init/kern_init_elf.hpp>
+#include <mesosphere/init/kern_init_layout.hpp>
+#include <mesosphere/init/kern_init_slab_setup.hpp>
+#include <mesosphere/init/kern_init_page_table_select.hpp>
+#include <mesosphere/init/kern_init_arguments_select.hpp>
+#include <mesosphere/kern_k_memory_layout.hpp>
 
 /* Core functionality. */
-#include "mesosphere/kern_select_interrupts.hpp"
-#include "mesosphere/kern_select_k_system_control.hpp"
+#include <mesosphere/kern_select_interrupt_manager.hpp>
+#include <mesosphere/kern_k_spin_lock.hpp>
+#include <mesosphere/kern_k_memory_manager.hpp>
+#include <mesosphere/kern_k_interrupt_task_manager.hpp>
+#include <mesosphere/kern_k_core_local_region.hpp>
+#include <mesosphere/kern_k_slab_heap.hpp>
+#include <mesosphere/kern_k_light_lock.hpp>
+#include <mesosphere/kern_k_dpc_manager.hpp>
+#include <mesosphere/kern_kernel.hpp>
+#include <mesosphere/kern_k_page_table_manager.hpp>
+#include <mesosphere/kern_select_page_table.hpp>
+
+/* Miscellaneous objects. */
+#include <mesosphere/kern_k_shared_memory_info.hpp>
+#include <mesosphere/kern_k_event_info.hpp>
+
+/* Auto Objects. */
+#include <mesosphere/kern_k_auto_object.hpp>
+#include <mesosphere/kern_k_synchronization_object.hpp>
+#include <mesosphere/kern_k_readable_event.hpp>
+#include <mesosphere/kern_k_handle_table.hpp>
+#include <mesosphere/kern_k_event.hpp>
+#include <mesosphere/kern_k_interrupt_event.hpp>
+#include <mesosphere/kern_k_light_session.hpp>
+#include <mesosphere/kern_k_session.hpp>
+#include <mesosphere/kern_k_session_request.hpp>
+#include <mesosphere/kern_k_port.hpp>
+#include <mesosphere/kern_k_shared_memory.hpp>
+#include <mesosphere/kern_k_transfer_memory.hpp>
+#include <mesosphere/kern_k_code_memory.hpp>
+#include <mesosphere/kern_k_device_address_space.hpp>
+#include <mesosphere/kern_select_debug.hpp>
+#include <mesosphere/kern_k_process.hpp>
+#include <mesosphere/kern_k_resource_limit.hpp>
+#include <mesosphere/kern_k_synchronization.hpp>
+
+/* More Miscellaneous objects. */
+#include <mesosphere/kern_k_object_name.hpp>
+#include <mesosphere/kern_k_scoped_resource_reservation.hpp>
+
+/* Supervisor Calls. */
+#include <mesosphere/kern_svc.hpp>
+
+/* Main functionality. */
+#include <mesosphere/kern_main.hpp>

libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_arguments.hpp

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+
+namespace ams::kern::init {
+
+    struct KInitArguments {
+        u64 ttbr0;
+        u64 ttbr1;
+        u64 tcr;
+        u64 mair;
+        u64 cpuactlr;
+        u64 cpuectlr;
+        u64 sctlr;
+        u64 sp;
+        u64 entrypoint;
+        u64 argument;
+        u64 setup_function;
+    };
+
+}

libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp

@@ -15,173 +15,13 @@
  */
 #pragma once
 #include <vapours.hpp>
-#include <mesosphere/kern_panic.hpp>
+#include <mesosphere/kern_common.hpp>
 #include <mesosphere/kern_k_typed_address.hpp>
-#include "../kern_cpu.hpp"
-
-namespace ams::kern::init {
-
-    constexpr size_t PageSize              = 0x1000;
-    constexpr size_t L1BlockSize           = 0x40000000;
-    constexpr size_t L2BlockSize           = 0x200000;
-    constexpr size_t L2ContiguousBlockSize = 0x10 * L2BlockSize;
-    constexpr size_t L3BlockSize           = 0x1000;
-    constexpr size_t L3ContiguousBlockSize = 0x10 * L3BlockSize;
-
-    class PageTableEntry {
-        public:
-            enum Permission : u64 {
-                Permission_KernelRWX = ((0ul << 53) | (1ul << 54) | (0ul << 6)),
-                Permission_KernelRX  = ((0ul << 53) | (1ul << 54) | (2ul << 6)),
-                Permission_KernelR   = ((1ul << 53) | (1ul << 54) | (2ul << 6)),
-                Permission_KernelRW  = ((1ul << 53) | (1ul << 54) | (0ul << 6)),
-
-                Permission_UserRX    = ((1ul << 53) | (0ul << 54) | (3ul << 6)),
-                Permission_UserR     = ((1ul << 53) | (1ul << 54) | (3ul << 6)),
-                Permission_UserRW    = ((1ul << 53) | (1ul << 54) | (1ul << 6)),
-            };
-
-            enum Shareable : u64 {
-                Shareable_NonShareable   = (0 << 8),
-                Shareable_OuterShareable = (2 << 8),
-                Shareable_InnerShareable = (3 << 8),
-            };
-
-            /* Official attributes are: */
-            /*    0x00, 0x04, 0xFF, 0x44. 4-7 are unused. */
-            enum PageAttribute : u64 {
-                PageAttribute_Device_nGnRnE            = (0 << 2),
-                PageAttribute_Device_nGnRE             = (1 << 2),
-                PageAttribute_NormalMemory             = (2 << 2),
-                PageAttribute_NormalMemoryNotCacheable = (3 << 2),
-            };
-
-            enum AccessFlag : u64 {
-                AccessFlag_NotAccessed = (0 << 10),
-                AccessFlag_Accessed    = (1 << 10),
-            };
-        protected:
-            u64 attributes;
-        public:
-            /* Take in a raw attribute. */
-            constexpr ALWAYS_INLINE PageTableEntry(u64 attr) : attributes(attr) { /* ... */ }
-
-            /* Extend a previous attribute. */
-            constexpr ALWAYS_INLINE PageTableEntry(const PageTableEntry &rhs, u64 new_attr) : attributes(rhs.attributes | new_attr) { /* ... */ }
-
-            /* Construct a new attribute. */
-            constexpr ALWAYS_INLINE PageTableEntry(Permission perm, PageAttribute p_a, Shareable share)
-                : attributes(static_cast<u64>(perm) | static_cast<u64>(AccessFlag_Accessed) | static_cast<u64>(p_a) | static_cast<u64>(share))
-            {
-                /* ... */
-            }
-        protected:
-            constexpr ALWAYS_INLINE u64 GetBits(size_t offset, size_t count) const {
-                return (this->attributes >> offset) & ((1ul << count) - 1);
-            }
-
-            constexpr ALWAYS_INLINE u64 SelectBits(size_t offset, size_t count) const {
-                return this->attributes & (((1ul << count) - 1) << offset);
-            }
-        public:
-            constexpr ALWAYS_INLINE bool IsUserExecuteNever()        const { return this->GetBits(54, 1) != 0; }
-            constexpr ALWAYS_INLINE bool IsPrivilegedExecuteNever()  const { return this->GetBits(53, 1) != 0; }
-            constexpr ALWAYS_INLINE bool IsContiguous()              const { return this->GetBits(52, 1) != 0; }
-            constexpr ALWAYS_INLINE AccessFlag GetAccessFlag()       const { return static_cast<AccessFlag>(this->GetBits(10, 1)); }
-            constexpr ALWAYS_INLINE Shareable GetShareable()         const { return static_cast<Shareable>(this->GetBits(8, 2)); }
-            constexpr ALWAYS_INLINE PageAttribute GetPageAttribute() const { return static_cast<PageAttribute>(this->GetBits(2, 3)); }
-            constexpr ALWAYS_INLINE bool IsNonSecure()               const { return this->GetBits(5, 1) != 0; }
-            constexpr ALWAYS_INLINE bool IsBlock()                   const { return this->GetBits(0, 2) == 0x1; }
-            constexpr ALWAYS_INLINE bool IsTable()                   const { return this->GetBits(0, 2) == 0x3; }
-
-            /* Should not be called except by derived classes. */
-            constexpr ALWAYS_INLINE u64 GetRawAttributes() const {
-                return this->attributes;
-            }
-    };
-
-    static_assert(sizeof(PageTableEntry) == sizeof(u64));
-
-    constexpr PageTableEntry InvalidPageTableEntry = PageTableEntry(0);
-
-    constexpr size_t MaxPageTableEntries = PageSize / sizeof(PageTableEntry);
-
-    class L1PageTableEntry : public PageTableEntry {
-        public:
-            constexpr ALWAYS_INLINE L1PageTableEntry(KPhysicalAddress phys_addr, bool pxn)
-                : PageTableEntry((0x3ul << 60) | (static_cast<u64>(pxn) << 59) | GetInteger(phys_addr) | 0x3)
-            {
-                /* ... */
-            }
-
-            constexpr ALWAYS_INLINE L1PageTableEntry(KPhysicalAddress phys_addr, const PageTableEntry &attr, bool contig)
-                : PageTableEntry(attr, (static_cast<u64>(contig) << 52) | GetInteger(phys_addr) | 0x1)
-            {
-                /* ... */
-            }
-
-            constexpr ALWAYS_INLINE KPhysicalAddress GetBlock() const {
-                return this->SelectBits(30, 18);
-            }
-
-            constexpr ALWAYS_INLINE KPhysicalAddress GetTable() const {
-                return this->SelectBits(12, 36);
-            }
-
-            constexpr ALWAYS_INLINE bool IsCompatibleWithAttribute(const PageTableEntry &rhs, bool contig) const {
-                /* Check whether this has the same permission/etc as the desired attributes. */
-                return L1PageTableEntry(this->GetBlock(), rhs, contig).GetRawAttributes() == this->GetRawAttributes();
-            }
-    };
-
-    class L2PageTableEntry : public PageTableEntry {
-        public:
-            constexpr ALWAYS_INLINE L2PageTableEntry(KPhysicalAddress phys_addr, bool pxn)
-                : PageTableEntry((0x3ul << 60) | (static_cast<u64>(pxn) << 59) | GetInteger(phys_addr) | 0x3)
-            {
-                /* ... */
-            }
-
-            constexpr ALWAYS_INLINE L2PageTableEntry(KPhysicalAddress phys_addr, const PageTableEntry &attr, bool contig)
-                : PageTableEntry(attr, (static_cast<u64>(contig) << 52) | GetInteger(phys_addr) | 0x1)
-            {
-                /* ... */
-            }
-
-            constexpr ALWAYS_INLINE KPhysicalAddress GetBlock() const {
-                return this->SelectBits(21, 27);
-            }
-
-            constexpr ALWAYS_INLINE KPhysicalAddress GetTable() const {
-                return this->SelectBits(12, 36);
-            }
-
-            constexpr ALWAYS_INLINE bool IsCompatibleWithAttribute(const PageTableEntry &rhs, bool contig) const {
-                /* Check whether this has the same permission/etc as the desired attributes. */
-                return L2PageTableEntry(this->GetBlock(), rhs, contig).GetRawAttributes() == this->GetRawAttributes();
-            }
-    };
-
-    class L3PageTableEntry : public PageTableEntry {
-        public:
-            constexpr ALWAYS_INLINE L3PageTableEntry(KPhysicalAddress phys_addr, const PageTableEntry &attr, bool contig)
-                : PageTableEntry(attr, (static_cast<u64>(contig) << 52) | GetInteger(phys_addr) | 0x3)
-            {
-                /* ... */
-            }
-
-            constexpr ALWAYS_INLINE bool IsBlock() const { return this->GetBits(0, 2) == 0x3; }
-
-            constexpr ALWAYS_INLINE KPhysicalAddress GetBlock() const {
-                return this->SelectBits(12, 36);
-            }
-
-            constexpr ALWAYS_INLINE bool IsCompatibleWithAttribute(const PageTableEntry &rhs, bool contig) const {
-                /* Check whether this has the same permission/etc as the desired attributes. */
-                return L3PageTableEntry(this->GetBlock(), rhs, contig).GetRawAttributes() == this->GetRawAttributes();
-            }
-    };
+#include <mesosphere/kern_select_cpu.hpp>
+#include <mesosphere/arch/arm64/kern_k_page_table_entry.hpp>
+#include <mesosphere/kern_select_system_control.hpp>
 
+namespace ams::kern::arch::arm64::init {
 
     class KInitialPageTable {
         public:
@@ -190,10 +30,14 @@ namespace ams::kern::init {
                     virtual KPhysicalAddress Allocate()           { return Null<KPhysicalAddress>; }
                     virtual void Free(KPhysicalAddress phys_addr) { /* Nothing to do here. */ (void)(phys_addr); }
             };
+
+            struct NoClear{};
         private:
             KPhysicalAddress l1_table;
         public:
-            constexpr ALWAYS_INLINE KInitialPageTable(KPhysicalAddress l1) : l1_table(l1) {
+            constexpr ALWAYS_INLINE KInitialPageTable(KPhysicalAddress l1, NoClear) : l1_table(l1) { /* ... */ }
+
+            constexpr ALWAYS_INLINE KInitialPageTable(KPhysicalAddress l1) : KInitialPageTable(l1, NoClear{}) {
                 ClearNewPageTable(this->l1_table);
             }
 
@@ -221,12 +65,226 @@ namespace ams::kern::init {
                 /* The MMU is necessarily not yet turned on, if we are creating an initial page table. */
                 std::memset(reinterpret_cast<void *>(GetInteger(address)), 0, PageSize);
             }
+        private:
+            size_t NOINLINE GetBlockCount(KVirtualAddress virt_addr, size_t size, size_t block_size) {
+                const KVirtualAddress end_virt_addr = virt_addr + size;
+                size_t count = 0;
+                while (virt_addr < end_virt_addr) {
+                    L1PageTableEntry *l1_entry = GetL1Entry(this->l1_table, virt_addr);
+
+                    /* If an L1 block is mapped or we're empty, advance by L1BlockSize. */
+                    if (l1_entry->IsBlock() || l1_entry->IsEmpty()) {
+                        MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), L1BlockSize));
+                        MESOSPHERE_INIT_ABORT_UNLESS(static_cast<size_t>(end_virt_addr - virt_addr) >= L1BlockSize);
+                        virt_addr += L1BlockSize;
+                        if (l1_entry->IsBlock() && block_size == L1BlockSize) {
+                            count++;
+                        }
+                        continue;
+                    }
+
+                    /* Non empty and non-block must be table. */
+                    MESOSPHERE_INIT_ABORT_UNLESS(l1_entry->IsTable());
+
+                    /* Table, so check if we're mapped in L2. */
+                    L2PageTableEntry *l2_entry = GetL2Entry(l1_entry, virt_addr);
+
+                    if (l2_entry->IsBlock() || l2_entry->IsEmpty()) {
+                        const size_t advance_size = (l2_entry->IsBlock() && l2_entry->IsContiguous()) ? L2ContiguousBlockSize : L2BlockSize;
+                        MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), advance_size));
+                        MESOSPHERE_INIT_ABORT_UNLESS(static_cast<size_t>(end_virt_addr - virt_addr) >= advance_size);
+                        virt_addr += advance_size;
+                        if (l2_entry->IsBlock() && block_size == advance_size) {
+                            count++;
+                        }
+                        continue;
+                    }
+
+                    /* Non empty and non-block must be table. */
+                    MESOSPHERE_INIT_ABORT_UNLESS(l2_entry->IsTable());
+
+                    /* Table, so check if we're mapped in L3. */
+                    L3PageTableEntry *l3_entry = GetL3Entry(l2_entry, virt_addr);
+
+                    /* L3 must be block or empty. */
+                    MESOSPHERE_INIT_ABORT_UNLESS(l3_entry->IsBlock() || l3_entry->IsEmpty());
+
+                    const size_t advance_size = (l3_entry->IsBlock() && l3_entry->IsContiguous()) ? L3ContiguousBlockSize : L3BlockSize;
+                    MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), advance_size));
+                    MESOSPHERE_INIT_ABORT_UNLESS(static_cast<size_t>(end_virt_addr - virt_addr) >= advance_size);
+                    virt_addr += advance_size;
+                    if (l3_entry->IsBlock() && block_size == advance_size) {
+                        count++;
+                    }
+                }
+                return count;
+            }
+
+            KVirtualAddress NOINLINE GetBlockByIndex(KVirtualAddress virt_addr, size_t size, size_t block_size, size_t index) {
+                const KVirtualAddress end_virt_addr = virt_addr + size;
+                size_t count = 0;
+                while (virt_addr < end_virt_addr) {
+                    L1PageTableEntry *l1_entry = GetL1Entry(this->l1_table, virt_addr);
+
+                    /* If an L1 block is mapped or we're empty, advance by L1BlockSize. */
+                    if (l1_entry->IsBlock() || l1_entry->IsEmpty()) {
+                        MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), L1BlockSize));
+                        MESOSPHERE_INIT_ABORT_UNLESS(static_cast<size_t>(end_virt_addr - virt_addr) >= L1BlockSize);
+                        if (l1_entry->IsBlock() && block_size == L1BlockSize) {
+                            if ((count++) == index) {
+                                return virt_addr;
+                            }
+                        }
+                        virt_addr += L1BlockSize;
+                        continue;
+                    }
+
+                    /* Non empty and non-block must be table. */
+                    MESOSPHERE_INIT_ABORT_UNLESS(l1_entry->IsTable());
+
+                    /* Table, so check if we're mapped in L2. */
+                    L2PageTableEntry *l2_entry = GetL2Entry(l1_entry, virt_addr);
+
+                    if (l2_entry->IsBlock() || l2_entry->IsEmpty()) {
+                        const size_t advance_size = (l2_entry->IsBlock() && l2_entry->IsContiguous()) ? L2ContiguousBlockSize : L2BlockSize;
+                        MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), advance_size));
+                        MESOSPHERE_INIT_ABORT_UNLESS(static_cast<size_t>(end_virt_addr - virt_addr) >= advance_size);
+                        if (l2_entry->IsBlock() && block_size == advance_size) {
+                            if ((count++) == index) {
+                                return virt_addr;
+                            }
+                        }
+                        virt_addr += advance_size;
+                        continue;
+                    }
+
+                    /* Non empty and non-block must be table. */
+                    MESOSPHERE_INIT_ABORT_UNLESS(l2_entry->IsTable());
+
+                    /* Table, so check if we're mapped in L3. */
+                    L3PageTableEntry *l3_entry = GetL3Entry(l2_entry, virt_addr);
+
+                    /* L3 must be block or empty. */
+                    MESOSPHERE_INIT_ABORT_UNLESS(l3_entry->IsBlock() || l3_entry->IsEmpty());
+
+                    const size_t advance_size = (l3_entry->IsBlock() && l3_entry->IsContiguous()) ? L3ContiguousBlockSize : L3BlockSize;
+                    MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), advance_size));
+                    MESOSPHERE_INIT_ABORT_UNLESS(static_cast<size_t>(end_virt_addr - virt_addr) >= advance_size);
+                    if (l3_entry->IsBlock() && block_size == advance_size) {
+                        if ((count++) == index) {
+                            return virt_addr;
+                        }
+                    }
+                    virt_addr += advance_size;
+                }
+                return Null<KVirtualAddress>;
+            }
+
+            PageTableEntry *GetMappingEntry(KVirtualAddress virt_addr, size_t block_size) {
+                L1PageTableEntry *l1_entry = GetL1Entry(this->l1_table, virt_addr);
+
+                if (l1_entry->IsBlock()) {
+                    MESOSPHERE_INIT_ABORT_UNLESS(block_size == L1BlockSize);
+                    return l1_entry;
+                }
+
+                MESOSPHERE_INIT_ABORT_UNLESS(l1_entry->IsTable());
+
+                /* Table, so check if we're mapped in L2. */
+                L2PageTableEntry *l2_entry = GetL2Entry(l1_entry, virt_addr);
+
+                if (l2_entry->IsBlock()) {
+                    const size_t real_size = (l2_entry->IsContiguous()) ? L2ContiguousBlockSize : L2BlockSize;
+                    MESOSPHERE_INIT_ABORT_UNLESS(real_size == block_size);
+                    return l2_entry;
+                }
+
+                MESOSPHERE_INIT_ABORT_UNLESS(l2_entry->IsTable());
+
+                /* Table, so check if we're mapped in L3. */
+                L3PageTableEntry *l3_entry = GetL3Entry(l2_entry, virt_addr);
+
+                /* L3 must be block. */
+                MESOSPHERE_INIT_ABORT_UNLESS(l3_entry->IsBlock());
+
+                const size_t real_size = (l3_entry->IsContiguous()) ? L3ContiguousBlockSize : L3BlockSize;
+                MESOSPHERE_INIT_ABORT_UNLESS(real_size == block_size);
+                return l3_entry;
+            }
+
+            void NOINLINE SwapBlocks(KVirtualAddress src_virt_addr, KVirtualAddress dst_virt_addr, size_t block_size, bool do_copy) {
+                static_assert(L2ContiguousBlockSize / L2BlockSize == L3ContiguousBlockSize / L3BlockSize);
+                const bool contig = (block_size == L2ContiguousBlockSize || block_size == L3ContiguousBlockSize);
+                const size_t num_mappings = contig ? L2ContiguousBlockSize / L2BlockSize : 1;
+
+                /* Unmap the source. */
+                PageTableEntry *src_entry = this->GetMappingEntry(src_virt_addr, block_size);
+                const auto src_saved = *src_entry;
+                for (size_t i = 0; i < num_mappings; i++) {
+                    *src_entry = InvalidPageTableEntry;
+                }
+
+                /* Unmap the target. */
+                PageTableEntry *dst_entry = this->GetMappingEntry(dst_virt_addr, block_size);
+                const auto dst_saved = *dst_entry;
+                for (size_t i = 0; i < num_mappings; i++) {
+                    *dst_entry = InvalidPageTableEntry;
+                }
+
+                /* Invalidate the entire tlb. */
+                cpu::DataSynchronizationBarrierInnerShareable();
+                cpu::InvalidateEntireTlb();
+
+                /* Copy data, if we should. */
+                const u64 negative_block_size_for_mask = static_cast<u64>(-static_cast<s64>(block_size));
+                const u64 offset_mask                  = negative_block_size_for_mask & ((1ul << 36) - 1);
+                const KVirtualAddress copy_src_addr = KVirtualAddress(src_saved.GetRawAttributesUnsafeForSwap() & offset_mask);
+                const KVirtualAddress copy_dst_addr = KVirtualAddress(dst_saved.GetRawAttributesUnsafeForSwap() & offset_mask);
+                if (block_size && do_copy) {
+                    u8 tmp[0x100];
+                    for (size_t ofs = 0; ofs < block_size; ofs += sizeof(tmp)) {
+                        std::memcpy(tmp, GetVoidPointer(copy_src_addr + ofs), sizeof(tmp));
+                        std::memcpy(GetVoidPointer(copy_src_addr + ofs), GetVoidPointer(copy_dst_addr + ofs), sizeof(tmp));
+                        std::memcpy(GetVoidPointer(copy_dst_addr + ofs), tmp, sizeof(tmp));
+                    }
+                }
+
+                /* Swap the mappings. */
+                const u64 attr_preserve_mask  = (negative_block_size_for_mask | 0xFFFF000000000000ul) ^ ((1ul << 36) - 1);
+                const size_t shift_for_contig = contig ? 4 : 0;
+                size_t advanced_size = 0;
+                const u64 src_attr_val = src_saved.GetRawAttributesUnsafeForSwap() & attr_preserve_mask;
+                const u64 dst_attr_val = dst_saved.GetRawAttributesUnsafeForSwap() & attr_preserve_mask;
+                for (size_t i = 0; i < num_mappings; i++) {
+                    reinterpret_cast<u64 *>(src_entry)[i] = GetInteger(copy_dst_addr + (advanced_size >> shift_for_contig)) | src_attr_val;
+                    reinterpret_cast<u64 *>(dst_entry)[i] = GetInteger(copy_src_addr + (advanced_size >> shift_for_contig)) | dst_attr_val;
+                    advanced_size += block_size;
+                }
+
+                cpu::DataSynchronizationBarrierInnerShareable();
+            }
+
+            void NOINLINE PhysicallyRandomize(KVirtualAddress virt_addr, size_t size, size_t block_size, bool do_copy) {
+                const size_t block_count = this->GetBlockCount(virt_addr, size, block_size);
+                if (block_count > 1) {
+                    for (size_t cur_block = 0; cur_block < block_count; cur_block++) {
+                        const size_t target_block = KSystemControl::Init::GenerateRandomRange(cur_block, block_count - 1);
+                        if (cur_block != target_block) {
+                            const KVirtualAddress cur_virt_addr    = this->GetBlockByIndex(virt_addr, size, block_size, cur_block);
+                            const KVirtualAddress target_virt_addr = this->GetBlockByIndex(virt_addr, size, block_size, target_block);
+                            MESOSPHERE_INIT_ABORT_UNLESS(cur_virt_addr    != Null<KVirtualAddress>);
+                            MESOSPHERE_INIT_ABORT_UNLESS(target_virt_addr != Null<KVirtualAddress>);
+                            this->SwapBlocks(cur_virt_addr, target_virt_addr, block_size, do_copy);
+                        }
+                    }
+                }
+            }
         public:
             void NOINLINE Map(KVirtualAddress virt_addr, size_t size, KPhysicalAddress phys_addr, const PageTableEntry &attr, IPageAllocator &allocator) {
                 /* Ensure that addresses and sizes are page aligned. */
-                MESOSPHERE_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr),    PageSize));
-                MESOSPHERE_ABORT_UNLESS(util::IsAligned(GetInteger(phys_addr),    PageSize));
-                MESOSPHERE_ABORT_UNLESS(util::IsAligned(size,                     PageSize));
+                MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr),    PageSize));
+                MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(phys_addr),    PageSize));
+                MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(size,                     PageSize));
 
                 /* Iteratively map pages until the requested region is mapped. */
                 while (size > 0) {
@@ -309,10 +367,37 @@ namespace ams::kern::init {
                 }
             }
 
+            KPhysicalAddress GetPhysicalAddress(KVirtualAddress virt_addr) const {
+                /* Get the L1 entry. */
+                const L1PageTableEntry *l1_entry = GetL1Entry(this->l1_table, virt_addr);
+
+                if (l1_entry->IsBlock()) {
+                    return l1_entry->GetBlock() + (GetInteger(virt_addr) & (L1BlockSize - 1));
+                }
+
+                MESOSPHERE_INIT_ABORT_UNLESS(l1_entry->IsTable());
+
+                /* Get the L2 entry. */
+                const L2PageTableEntry *l2_entry = GetL2Entry(l1_entry, virt_addr);
+
+                if (l2_entry->IsBlock()) {
+                    return l2_entry->GetBlock() + (GetInteger(virt_addr) & (L2BlockSize - 1));
+                }
+
+                MESOSPHERE_INIT_ABORT_UNLESS(l2_entry->IsTable());
+
+                /* Get the L3 entry. */
+                const L3PageTableEntry *l3_entry = GetL3Entry(l2_entry, virt_addr);
+
+                MESOSPHERE_INIT_ABORT_UNLESS(l3_entry->IsBlock());
+
+                return l3_entry->GetBlock() + (GetInteger(virt_addr) & (L3BlockSize - 1));
+            }
+
             bool IsFree(KVirtualAddress virt_addr, size_t size) {
                 /* Ensure that addresses and sizes are page aligned. */
-                MESOSPHERE_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr),    PageSize));
-                MESOSPHERE_ABORT_UNLESS(util::IsAligned(size,                     PageSize));
+                MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr),    PageSize));
+                MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(size,                     PageSize));
 
                 const KVirtualAddress end_virt_addr = virt_addr + size;
                 while (virt_addr < end_virt_addr) {
@@ -360,8 +445,8 @@ namespace ams::kern::init {
                 cpu::DataSynchronizationBarrierInnerShareable();
 
                 /* Ensure that addresses and sizes are page aligned. */
-                MESOSPHERE_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr),    PageSize));
-                MESOSPHERE_ABORT_UNLESS(util::IsAligned(size,                     PageSize));
+                MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr),    PageSize));
+                MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(size,                     PageSize));
 
                 /* Iteratively reprotect pages until the requested region is reprotected. */
                 while (size > 0) {
@@ -371,9 +456,9 @@ namespace ams::kern::init {
                     if (l1_entry->IsBlock()) {
                         /* Ensure that we are allowed to have an L1 block here. */
                         const KPhysicalAddress block = l1_entry->GetBlock();
-                        MESOSPHERE_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), L1BlockSize));
-                        MESOSPHERE_ABORT_UNLESS(util::IsAligned(size, L1BlockSize));
-                        MESOSPHERE_ABORT_UNLESS(l1_entry->IsCompatibleWithAttribute(attr_before, false));
+                        MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), L1BlockSize));
+                        MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(size, L1BlockSize));
+                        MESOSPHERE_INIT_ABORT_UNLESS(l1_entry->IsCompatibleWithAttribute(attr_before, false));
 
                         /* Invalidate the existing L1 block. */
                         *static_cast<PageTableEntry *>(l1_entry) = InvalidPageTableEntry;
@@ -389,7 +474,7 @@ namespace ams::kern::init {
                     }
 
                     /* Not a block, so we must be a table. */
-                    MESOSPHERE_ABORT_UNLESS(l1_entry->IsTable());
+                    MESOSPHERE_INIT_ABORT_UNLESS(l1_entry->IsTable());
 
                     L2PageTableEntry *l2_entry = GetL2Entry(l1_entry, virt_addr);
                     if (l2_entry->IsBlock()) {
@@ -397,14 +482,14 @@ namespace ams::kern::init {
 
                         if (l2_entry->IsContiguous()) {
                             /* Ensure that we are allowed to have a contiguous L2 block here. */
-                            MESOSPHERE_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), L2ContiguousBlockSize));
-                            MESOSPHERE_ABORT_UNLESS(util::IsAligned(GetInteger(block),     L2ContiguousBlockSize));
-                            MESOSPHERE_ABORT_UNLESS(util::IsAligned(size,                  L2ContiguousBlockSize));
+                            MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), L2ContiguousBlockSize));
+                            MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(block),     L2ContiguousBlockSize));
+                            MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(size,                  L2ContiguousBlockSize));
 
                             /* Invalidate the existing contiguous L2 block. */
                             for (size_t i = 0; i < L2ContiguousBlockSize / L2BlockSize; i++) {
                                 /* Ensure that the entry is valid. */
-                                MESOSPHERE_ABORT_UNLESS(l2_entry[i].IsCompatibleWithAttribute(attr_before, true));
+                                MESOSPHERE_INIT_ABORT_UNLESS(l2_entry[i].IsCompatibleWithAttribute(attr_before, true));
                                 static_cast<PageTableEntry *>(l2_entry)[i] = InvalidPageTableEntry;
                             }
                             cpu::DataSynchronizationBarrierInnerShareable();
@@ -419,10 +504,10 @@ namespace ams::kern::init {
                             size      -= L2ContiguousBlockSize;
                         } else {
                             /* Ensure that we are allowed to have an L2 block here. */
-                            MESOSPHERE_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), L2BlockSize));
-                            MESOSPHERE_ABORT_UNLESS(util::IsAligned(GetInteger(block),     L2BlockSize));
-                            MESOSPHERE_ABORT_UNLESS(util::IsAligned(size,                  L2BlockSize));
-                            MESOSPHERE_ABORT_UNLESS(l2_entry->IsCompatibleWithAttribute(attr_before, false));
+                            MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), L2BlockSize));
+                            MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(block),     L2BlockSize));
+                            MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(size,                  L2BlockSize));
+                            MESOSPHERE_INIT_ABORT_UNLESS(l2_entry->IsCompatibleWithAttribute(attr_before, false));
 
                             /* Invalidate the existing L2 block. */
                             *static_cast<PageTableEntry *>(l2_entry) = InvalidPageTableEntry;
@@ -440,23 +525,23 @@ namespace ams::kern::init {
                     }
 
                     /* Not a block, so we must be a table. */
-                    MESOSPHERE_ABORT_UNLESS(l2_entry->IsTable());
+                    MESOSPHERE_INIT_ABORT_UNLESS(l2_entry->IsTable());
 
                     /* We must have a mapped l3 entry to reprotect. */
                     L3PageTableEntry *l3_entry = GetL3Entry(l2_entry, virt_addr);
-                    MESOSPHERE_ABORT_UNLESS(l3_entry->IsBlock());
+                    MESOSPHERE_INIT_ABORT_UNLESS(l3_entry->IsBlock());
                     const KPhysicalAddress block = l3_entry->GetBlock();
 
                     if (l3_entry->IsContiguous()) {
                         /* Ensure that we are allowed to have a contiguous L3 block here. */
-                        MESOSPHERE_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), L3ContiguousBlockSize));
-                        MESOSPHERE_ABORT_UNLESS(util::IsAligned(GetInteger(block),     L3ContiguousBlockSize));
-                        MESOSPHERE_ABORT_UNLESS(util::IsAligned(size,                  L3ContiguousBlockSize));
+                        MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), L3ContiguousBlockSize));
+                        MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(block),     L3ContiguousBlockSize));
+                        MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(size,                  L3ContiguousBlockSize));
 
                         /* Invalidate the existing contiguous L3 block. */
                         for (size_t i = 0; i < L3ContiguousBlockSize / L3BlockSize; i++) {
                             /* Ensure that the entry is valid. */
-                            MESOSPHERE_ABORT_UNLESS(l3_entry[i].IsCompatibleWithAttribute(attr_before, true));
+                            MESOSPHERE_INIT_ABORT_UNLESS(l3_entry[i].IsCompatibleWithAttribute(attr_before, true));
                             static_cast<PageTableEntry *>(l3_entry)[i] = InvalidPageTableEntry;
                         }
                         cpu::DataSynchronizationBarrierInnerShareable();
@@ -471,10 +556,10 @@ namespace ams::kern::init {
                         size      -= L3ContiguousBlockSize;
                     } else {
                         /* Ensure that we are allowed to have an L3 block here. */
-                        MESOSPHERE_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), L3BlockSize));
-                        MESOSPHERE_ABORT_UNLESS(util::IsAligned(GetInteger(block),     L3BlockSize));
-                        MESOSPHERE_ABORT_UNLESS(util::IsAligned(size,                  L3BlockSize));
-                        MESOSPHERE_ABORT_UNLESS(l3_entry->IsCompatibleWithAttribute(attr_before, false));
+                        MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), L3BlockSize));
+                        MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(block),     L3BlockSize));
+                        MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(size,                  L3BlockSize));
+                        MESOSPHERE_INIT_ABORT_UNLESS(l3_entry->IsCompatibleWithAttribute(attr_before, false));
 
                         /* Invalidate the existing L3 block. */
                         *static_cast<PageTableEntry *>(l3_entry) = InvalidPageTableEntry;
@@ -493,6 +578,67 @@ namespace ams::kern::init {
                 cpu::DataSynchronizationBarrierInnerShareable();
             }
 
+            void PhysicallyRandomize(KVirtualAddress virt_addr, size_t size, bool do_copy) {
+                this->PhysicallyRandomize(virt_addr, size, L1BlockSize, do_copy);
+                this->PhysicallyRandomize(virt_addr, size, L2ContiguousBlockSize, do_copy);
+                this->PhysicallyRandomize(virt_addr, size, L2BlockSize, do_copy);
+                this->PhysicallyRandomize(virt_addr, size, L3ContiguousBlockSize, do_copy);
+                this->PhysicallyRandomize(virt_addr, size, L3BlockSize, do_copy);
+            }
+
+    };
+
+    class KInitialPageAllocator : public KInitialPageTable::IPageAllocator {
+        public:
+            struct State {
+                uintptr_t next_address;
+                uintptr_t free_bitmap;
+            };
+        private:
+            State state;
+        public:
+            constexpr ALWAYS_INLINE KInitialPageAllocator() : state{} { /* ... */ }
+
+            ALWAYS_INLINE void Initialize(uintptr_t address) {
+                this->state.next_address = address + BITSIZEOF(this->state.free_bitmap) * PageSize;
+                this->state.free_bitmap  = ~uintptr_t();
+            }
+
+            ALWAYS_INLINE void InitializeFromState(uintptr_t state_val) {
+                if (kern::GetTargetFirmware() >= kern::TargetFirmware_10_0_0) {
+                    this->state = *reinterpret_cast<State *>(state_val);
+                } else {
+                    this->state.next_address = state_val;
+                    this->state.free_bitmap  = 0;
+                }
+            }
+
+            ALWAYS_INLINE void GetFinalState(State *out) {
+                *out = this->state;
+                this->state = {};
+            }
+        public:
+            virtual KPhysicalAddress Allocate() override {
+                MESOSPHERE_INIT_ABORT_UNLESS(this->state.next_address != Null<uintptr_t>);
+                uintptr_t allocated = this->state.next_address;
+                if (this->state.free_bitmap != 0) {
+                    u64 index;
+                    uintptr_t mask;
+                    do {
+                        index = KSystemControl::Init::GenerateRandomRange(0, BITSIZEOF(this->state.free_bitmap) - 1);
+                        mask  = (static_cast<uintptr_t>(1) << index);
+                    } while ((this->state.free_bitmap & mask) == 0);
+                    this->state.free_bitmap &= ~mask;
+                    allocated = this->state.next_address - ((BITSIZEOF(this->state.free_bitmap) - index) * PageSize);
+                } else {
+                    this->state.next_address += PageSize;
+                }
+
+                std::memset(reinterpret_cast<void *>(allocated), 0, PageSize);
+                return allocated;
+            }
+
+            /* No need to override free. The default does nothing, and so would we. */
     };
 
 }

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu.hpp

@@ -15,9 +15,27 @@
  */
 #pragma once
 #include <vapours.hpp>
-#include "kern_cpu_system_registers.hpp"
+#include <mesosphere/arch/arm64/kern_cpu_system_registers.hpp>
+#include <mesosphere/kern_select_userspace_memory_access.hpp>
 
-namespace ams::kern::arm64::cpu {
+namespace ams::kern::arch::arm64::cpu {
+
+#if defined(ATMOSPHERE_CPU_ARM_CORTEX_A57) || defined(ATMOSPHERE_CPU_ARM_CORTEX_A53)
+    constexpr inline size_t InstructionCacheLineSize = 0x40;
+    constexpr inline size_t DataCacheLineSize        = 0x40;
+    constexpr inline size_t NumPerformanceCounters   = 6;
+#else
+    #error "Unknown CPU for cache line sizes"
+#endif
+
+#if defined(ATMOSPHERE_BOARD_NINTENDO_NX)
+    constexpr inline size_t NumCores = 4;
+#else
+    #error "Unknown Board for cpu::NumCores"
+#endif
+
+    /* Initialization. */
+    NOINLINE void InitializeInterruptThreads(s32 core_id);
 
     /* Helpers for managing memory state. */
     ALWAYS_INLINE void DataSynchronizationBarrier() {
@@ -46,13 +64,154 @@ namespace ams::kern::arm64::cpu {
         EnsureInstructionConsistency();
     }
 
+    ALWAYS_INLINE void SwitchProcess(u64 ttbr, u32 proc_id) {
+        SetTtbr0El1(ttbr);
+        ContextIdRegisterAccessor(0).SetProcId(proc_id).Store();
+        InstructionMemoryBarrier();
+    }
+
+    /* Performance counter helpers. */
+    ALWAYS_INLINE u64 GetCycleCounter() {
+        return cpu::GetPmcCntrEl0();
+    }
+
+    ALWAYS_INLINE u32 GetPerformanceCounter(s32 n) {
+        u64 counter = 0;
+        if (n < static_cast<s32>(NumPerformanceCounters)) {
+            switch (n) {
+                case 0:
+                    counter = cpu::GetPmevCntr0El0();
+                    break;
+                case 1:
+                    counter = cpu::GetPmevCntr1El0();
+                    break;
+                case 2:
+                    counter = cpu::GetPmevCntr2El0();
+                    break;
+                case 3:
+                    counter = cpu::GetPmevCntr3El0();
+                    break;
+                case 4:
+                    counter = cpu::GetPmevCntr4El0();
+                    break;
+                case 5:
+                    counter = cpu::GetPmevCntr5El0();
+                    break;
+                default:
+                    break;
+            }
+        }
+        return static_cast<u32>(counter);
+    }
+
+    /* Helper for address access. */
+    ALWAYS_INLINE bool GetPhysicalAddressWritable(KPhysicalAddress *out, KVirtualAddress addr, bool privileged = false) {
+        const uintptr_t va = GetInteger(addr);
+
+        if (privileged) {
+            __asm__ __volatile__("at s1e1w, %[va]" :: [va]"r"(va) : "memory");
+        } else {
+            __asm__ __volatile__("at s1e0w, %[va]" :: [va]"r"(va) : "memory");
+        }
+        InstructionMemoryBarrier();
+
+        u64 par = GetParEl1();
+
+        if (par & 0x1) {
+            return false;
+        }
+
+        if (out) {
+            *out = KPhysicalAddress((par & 0xFFFFFFFFF000ull) | (va & 0xFFFull));
+        }
+        return true;
+    }
+
+    ALWAYS_INLINE bool GetPhysicalAddressReadable(KPhysicalAddress *out, KVirtualAddress addr, bool privileged = false) {
+        const uintptr_t va = GetInteger(addr);
+
+        if (privileged) {
+            __asm__ __volatile__("at s1e1r, %[va]" :: [va]"r"(va) : "memory");
+        } else {
+            __asm__ __volatile__("at s1e0r, %[va]" :: [va]"r"(va) : "memory");
+        }
+        InstructionMemoryBarrier();
+
+        u64 par = GetParEl1();
+
+        if (par & 0x1) {
+            return false;
+        }
+
+        if (out) {
+            *out = KPhysicalAddress((par & 0xFFFFFFFFF000ull) | (va & 0xFFFull));
+        }
+        return true;
+    }
+
+    /* Synchronization helpers. */
+    NOINLINE void SynchronizeAllCores();
+
     /* Cache management helpers. */
-    void FlushEntireDataCacheShared();
-    void FlushEntireDataCacheLocal();
+    void ClearPageToZeroImpl(void *);
+    void FlushEntireDataCacheSharedForInit();
+    void FlushEntireDataCacheLocalForInit();
+
+    void FlushEntireDataCache();
+
+    Result InvalidateDataCache(void *addr, size_t size);
+    Result StoreDataCache(const void *addr, size_t size);
+    Result FlushDataCache(const void *addr, size_t size);
+    Result InvalidateInstructionCache(void *addr, size_t size);
+
+    ALWAYS_INLINE void ClearPageToZero(void *page) {
+        MESOSPHERE_ASSERT(util::IsAligned(reinterpret_cast<uintptr_t>(page), PageSize));
+        MESOSPHERE_ASSERT(page != nullptr);
+        ClearPageToZeroImpl(page);
+    }
+
+    ALWAYS_INLINE void InvalidateTlbByAsid(u32 asid) {
+        const u64 value = (static_cast<u64>(asid) << 48);
+        __asm__ __volatile__("tlbi aside1is, %[value]" :: [value]"r"(static_cast<u64>(value) << 48) : "memory");
+        EnsureInstructionConsistency();
+    }
+
+    ALWAYS_INLINE void InvalidateTlbByAsidAndVa(u32 asid, KProcessAddress virt_addr) {
+        const u64 value = (static_cast<u64>(asid) << 48) | ((GetInteger(virt_addr) >> 12) & 0xFFFFFFFFFFFul);
+        __asm__ __volatile__("tlbi aside1is, %[value]" :: [value]"r"(value) : "memory");
+        EnsureInstructionConsistency();
+    }
 
     ALWAYS_INLINE void InvalidateEntireTlb() {
         __asm__ __volatile__("tlbi vmalle1is" ::: "memory");
         EnsureInstructionConsistency();
     }
 
+    ALWAYS_INLINE void InvalidateEntireTlbDataOnly() {
+        __asm__ __volatile__("tlbi vmalle1is" ::: "memory");
+        DataSynchronizationBarrier();
+    }
+
+    ALWAYS_INLINE void InvalidateTlbByVaDataOnly(KProcessAddress virt_addr) {
+        const u64 value = ((GetInteger(virt_addr) >> 12) & 0xFFFFFFFFFFFul);
+        __asm__ __volatile__("tlbi vaae1is, %[value]" :: [value]"r"(value) : "memory");
+        DataSynchronizationBarrier();
+    }
+
+    ALWAYS_INLINE uintptr_t GetCoreLocalRegionAddress() {
+        register uintptr_t x18 asm("x18");
+        __asm__ __volatile__("" : [x18]"=r"(x18));
+        return x18;
+    }
+
+    ALWAYS_INLINE void SetCoreLocalRegionAddress(uintptr_t value) {
+        register uintptr_t x18 asm("x18") = value;
+        __asm__ __volatile__("":: [x18]"r"(x18));
+        SetTpidrEl1(value);
+    }
+
+    ALWAYS_INLINE void SwitchThreadLocalRegion(uintptr_t tlr) {
+        cpu::SetTpidrRoEl0(tlr);
+    }
+
 }

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu_system_registers.hpp

@@ -16,7 +16,7 @@
 #pragma once
 #include <vapours.hpp>
 
-namespace ams::kern::arm64::cpu {
+namespace ams::kern::arch::arm64::cpu {
 
     #define MESOSPHERE_CPU_GET_SYSREG(name)                                            \
         ({                                                                             \
@@ -37,8 +37,14 @@ namespace ams::kern::arm64::cpu {
     MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(Ttbr0El1, ttbr0_el1)
     MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(Ttbr1El1, ttbr1_el1)
 
-    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(MairEl1, mair_el1)
     MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(TcrEl1, tcr_el1)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(MairEl1, mair_el1)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(TpidrEl1, tpidr_el1)
+
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(VbarEl1, vbar_el1)
+
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(FarEl1, far_el1)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(ParEl1, par_el1)
 
     MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(SctlrEl1, sctlr_el1)
 
@@ -46,21 +52,215 @@ namespace ams::kern::arm64::cpu {
     MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(CpuEctlrEl1, s3_1_c15_c2_1)
 
     MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(CsselrEl1, csselr_el1)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(CcsidrEl1, ccsidr_el1)
+
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(OslarEl1, oslar_el1)
+
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(TpidrRoEl0, tpidrro_el0)
+
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(EsrEl1, esr_el1)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(Afsr0El1, afsr0_el1)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(Afsr1El1, afsr1_el1)
+
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(PmUserEnrEl0, pmuserenr_el0)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(PmcCntrEl0,   pmccntr_el0)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(PmevCntr0El0, pmevcntr0_el0)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(PmevCntr1El0, pmevcntr1_el0)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(PmevCntr2El0, pmevcntr2_el0)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(PmevCntr3El0, pmevcntr3_el0)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(PmevCntr4El0, pmevcntr4_el0)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(PmevCntr5El0, pmevcntr5_el0)
+
+    #define FOR_I_IN_0_TO_15(HANDLER, ...)                                                                              \
+        HANDLER(0,  ## __VA_ARGS__) HANDLER(1,  ## __VA_ARGS__) HANDLER(2,  ## __VA_ARGS__) HANDLER(3,  ## __VA_ARGS__) \
+        HANDLER(4,  ## __VA_ARGS__) HANDLER(5,  ## __VA_ARGS__) HANDLER(6,  ## __VA_ARGS__) HANDLER(7,  ## __VA_ARGS__) \
+        HANDLER(8,  ## __VA_ARGS__) HANDLER(9,  ## __VA_ARGS__) HANDLER(10, ## __VA_ARGS__) HANDLER(11, ## __VA_ARGS__) \
+        HANDLER(12, ## __VA_ARGS__) HANDLER(13, ## __VA_ARGS__) HANDLER(14, ## __VA_ARGS__) HANDLER(15, ## __VA_ARGS__) \
+
+    #define MESOSPHERE_CPU_DEFINE_DBG_SYSREG_ACCESSORS(ID, ...)                     \
+        MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(DbgWcr##ID##El1, dbgwcr##ID##_el1)   \
+        MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(DbgWvr##ID##El1, dbgwvr##ID##_el1)   \
+        MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(DbgBcr##ID##El1, dbgbcr##ID##_el1)   \
+        MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(DbgBvr##ID##El1, dbgbvr##ID##_el1)
+
+    FOR_I_IN_0_TO_15(MESOSPHERE_CPU_DEFINE_DBG_SYSREG_ACCESSORS)
+
+    #undef MESOSPHERE_CPU_DEFINE_DBG_SYSREG_ACCESSORS
 
     /* Base class for register accessors. */
-    class GenericRegisterAccessor {
+    class GenericRegisterAccessorBase {
+        NON_COPYABLE(GenericRegisterAccessorBase);
+        NON_MOVEABLE(GenericRegisterAccessorBase);
         private:
             u64 value;
         public:
-            ALWAYS_INLINE GenericRegisterAccessor(u64 v) : value(v) { /* ... */ }
+            constexpr ALWAYS_INLINE GenericRegisterAccessorBase(u64 v) : value(v) { /* ... */ }
         protected:
+            constexpr ALWAYS_INLINE u64 GetValue() const {
+                return this->value;
+            }
+
             constexpr ALWAYS_INLINE u64 GetBits(size_t offset, size_t count) const {
                 return (this->value >> offset) & ((1ul << count) - 1);
             }
+
+            constexpr ALWAYS_INLINE void SetBits(size_t offset, size_t count, u64 value) {
+                const u64 mask = ((1ul << count) - 1) << offset;
+                this->value &= ~mask;
+                this->value |= (value & (mask >> offset)) << offset;
+            }
+
+            constexpr ALWAYS_INLINE void SetBitsDirect(size_t offset, size_t count, u64 value) {
+                const u64 mask = ((1ul << count) - 1) << offset;
+                this->value &= ~mask;
+                this->value |= (value & mask);
+            }
+
+            constexpr ALWAYS_INLINE void SetBit(size_t offset, bool enabled) {
+                const u64 mask = 1ul << offset;
+                if (enabled) {
+                    this->value |= mask;
+                } else {
+                    this->value &= ~mask;
+                }
+            }
     };
 
-    /* Special code for main id register. */
-    class MainIdRegisterAccessor : public GenericRegisterAccessor {
+    template<typename Derived>
+    class GenericRegisterAccessor : public GenericRegisterAccessorBase {
+        public:
+            constexpr ALWAYS_INLINE GenericRegisterAccessor(u64 v) : GenericRegisterAccessorBase(v) { /* ... */ }
+        protected:
+            ALWAYS_INLINE void Store() const {
+                static_cast<const Derived *>(this)->Store();
+            }
+    };
+
+    #define MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(name) class name##RegisterAccessor : public GenericRegisterAccessor<name##RegisterAccessor>
+
+    #define MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(accessor, reg_name)                                                  \
+        ALWAYS_INLINE accessor##RegisterAccessor() : GenericRegisterAccessor(MESOSPHERE_CPU_GET_SYSREG(reg_name)) { /* ... */ } \
+        constexpr ALWAYS_INLINE accessor##RegisterAccessor(u64 v) : GenericRegisterAccessor(v) { /* ... */ }                    \
+                                                                                                                                \
+        ALWAYS_INLINE void Store() { const u64 v = this->GetValue(); MESOSPHERE_CPU_SET_SYSREG(reg_name, v); }
+
+    /* Accessors. */
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(MemoryAccessIndirection) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(MemoryAccessIndirection, mair_el1)
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(TranslationControl) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(TranslationControl, tcr_el1)
+
+            constexpr ALWAYS_INLINE size_t GetT1Size() const {
+                const size_t shift_value = this->GetBits(16, 6);
+                return size_t(1) << (size_t(64) - shift_value);
+            }
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(ArchitecturalFeatureAccessControl) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(ArchitecturalFeatureAccessControl, cpacr_el1)
+
+            constexpr ALWAYS_INLINE decltype(auto) SetFpEnabled(bool en) {
+                if (en) {
+                    this->SetBits(20, 2, 0x3);
+                } else {
+                    this->SetBits(20, 2, 0x0);
+                }
+                return *this;
+            }
+
+            constexpr ALWAYS_INLINE bool IsFpEnabled() {
+                return this->GetBits(20, 2) != 0;
+            }
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(DebugFeature) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(DebugFeature, id_aa64dfr0_el1)
+
+            constexpr ALWAYS_INLINE size_t GetNumWatchpoints() const {
+                return this->GetBits(20, 4);
+            }
+
+            constexpr ALWAYS_INLINE size_t GetNumBreakpoints() const {
+                return this->GetBits(12, 4);
+            }
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(MonitorDebugSystemControl) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(MonitorDebugSystemControl, mdscr_el1)
+
+            constexpr ALWAYS_INLINE bool GetMde() const {
+                return this->GetBits(15, 1) != 0;
+            }
+
+            constexpr ALWAYS_INLINE size_t GetTdcc() const {
+                return this->GetBits(12, 1) != 0;
+            }
+
+            constexpr ALWAYS_INLINE decltype(auto) SetMde(bool set) {
+                this->SetBit(15, set);
+                return *this;
+            }
+
+            constexpr ALWAYS_INLINE decltype(auto) SetTdcc(bool set) {
+                this->SetBit(12, set);
+                return *this;
+            }
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(MultiprocessorAffinity) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(MultiprocessorAffinity, mpidr_el1)
+
+            constexpr ALWAYS_INLINE u64 GetAff0() const {
+                return this->GetBits(0, 8);
+            }
+
+            constexpr ALWAYS_INLINE u64 GetAff1() const {
+                return this->GetBits(8, 8);
+            }
+
+            constexpr ALWAYS_INLINE u64 GetAff2() const {
+                return this->GetBits(16, 8);
+            }
+
+            constexpr ALWAYS_INLINE u64 GetAff3() const {
+                return this->GetBits(32, 8);
+            }
+
+            constexpr ALWAYS_INLINE u64 GetCpuOnArgument() const {
+                constexpr u64 Mask = 0x000000FF00FFFF00ul;
+                return this->GetValue() & Mask;
+            }
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(ThreadId) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(ThreadId, tpidr_el1)
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(OsLockAccess) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(OsLockAccess, oslar_el1)
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(ContextId) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(ContextId, contextidr_el1)
+
+            constexpr ALWAYS_INLINE decltype(auto) SetProcId(u32 proc_id) {
+                this->SetBits(0, BITSIZEOF(proc_id), proc_id);
+                return *this;
+            }
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(MainId) {
         public:
             enum class Implementer {
                 ArmLimited = 0x41,
@@ -70,7 +270,7 @@ namespace ams::kern::arm64::cpu {
                 CortexA57  = 0xD07,
             };
         public:
-            ALWAYS_INLINE MainIdRegisterAccessor() : GenericRegisterAccessor(MESOSPHERE_CPU_GET_SYSREG(midr_el1)) { /* ... */ }
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(MainId, midr_el1)
         public:
             constexpr ALWAYS_INLINE Implementer GetImplementer() const {
                 return static_cast<Implementer>(this->GetBits(24, 8));
@@ -93,10 +293,69 @@ namespace ams::kern::arm64::cpu {
             }
     };
 
-    /* Accessors for cache registers. */
-    class CacheLineIdAccessor : public GenericRegisterAccessor {
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(SystemControl) {
         public:
-            ALWAYS_INLINE CacheLineIdAccessor() : GenericRegisterAccessor(MESOSPHERE_CPU_GET_SYSREG(clidr_el1)) { /* ... */ }
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(SystemControl, sctlr_el1)
+
+            constexpr ALWAYS_INLINE decltype(auto) SetWxn(bool en) {
+                this->SetBit(19, en);
+                return *this;
+            }
+    };
+
+    /* Accessors for timer registers. */
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(CounterTimerKernelControl) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(CounterTimerKernelControl, cntkctl_el1)
+
+            constexpr ALWAYS_INLINE decltype(auto) SetEl0PctEn(bool en) {
+                this->SetBit(0, en);
+                return *this;
+            }
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(CounterTimerPhysicalTimerControl) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(CounterTimerPhysicalTimerControl, cntp_ctl_el0)
+
+            constexpr ALWAYS_INLINE decltype(auto) SetEnable(bool en) {
+                this->SetBit(0, en);
+                return *this;
+            }
+
+            constexpr ALWAYS_INLINE decltype(auto) SetIMask(bool en) {
+                this->SetBit(1, en);
+                return *this;
+            }
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(CounterTimerPhysicalTimerCompareValue) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(CounterTimerPhysicalTimerCompareValue, cntp_cval_el0)
+
+            constexpr ALWAYS_INLINE u64 GetCompareValue() {
+                return this->GetValue();
+            }
+
+            constexpr ALWAYS_INLINE decltype(auto) SetCompareValue(u64 value) {
+                this->SetBits(0, BITSIZEOF(value), value);
+                return *this;
+            }
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(CounterTimerPhysicalCountValue) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(CounterTimerPhysicalCountValue, cntpct_el0)
+
+            constexpr ALWAYS_INLINE u64 GetCount() {
+                return this->GetValue();
+            }
+    };
+
+    /* Accessors for cache registers. */
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(CacheLineId) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(CacheLineId, clidr_el1)
         public:
             constexpr ALWAYS_INLINE int GetLevelsOfCoherency() const {
                 return static_cast<int>(this->GetBits(24, 3));
@@ -109,9 +368,9 @@ namespace ams::kern::arm64::cpu {
             /* TODO: Other bitfield accessors? */
     };
 
-    class CacheSizeIdAccessor : public GenericRegisterAccessor {
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(CacheSizeId) {
         public:
-            ALWAYS_INLINE CacheSizeIdAccessor() : GenericRegisterAccessor(MESOSPHERE_CPU_GET_SYSREG(ccsidr_el1)) { /* ... */ }
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(CacheSizeId, ccsidr_el1)
         public:
             constexpr ALWAYS_INLINE int GetNumberOfSets() const {
                 return static_cast<int>(this->GetBits(13, 15));
@@ -128,6 +387,9 @@ namespace ams::kern::arm64::cpu {
             /* TODO: Other bitfield accessors? */
     };
 
+    #undef  FOR_I_IN_0_TO_15
+    #undef  MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS
+    #undef  MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS
     #undef  MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS
     #undef  MESOSPHERE_CPU_GET_SYSREG
     #undef  MESOSPHERE_CPU_SET_SYSREG

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_debug.hpp

@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_select_cpu.hpp>
+#include <mesosphere/kern_k_debug_base.hpp>
+
+namespace ams::kern {
+
+    class KThread;
+    class KProcess;
+
+}
+
+namespace ams::kern::arch::arm64 {
+
+    class KDebug final : public KAutoObjectWithSlabHeapAndContainer<KDebug, KDebugBase> {
+        MESOSPHERE_AUTOOBJECT_TRAITS(KDebug, KSynchronizationObject);
+        public:
+            /* TODO: This is a placeholder definition. */
+    };
+
+}

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_exception_context.hpp

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+
+namespace ams::kern::arch::arm64 {
+
+    struct KExceptionContext {
+        u64 x[(30 - 0) + 1];
+        u64 sp;
+        u64 pc;
+        u64 psr;
+        u64 tpidr;
+        u64 reserved;
+    };
+    static_assert(sizeof(KExceptionContext) == 0x120);
+
+}

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_hardware_timer.hpp

@@ -0,0 +1,94 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_select_cpu.hpp>
+#include <mesosphere/kern_k_hardware_timer_base.hpp>
+
+namespace ams::kern::arch::arm64 {
+
+    namespace impl {
+
+        class KHardwareTimerInterruptTask;
+
+    }
+
+    class KHardwareTimer : public KHardwareTimerBase {
+        public:
+            constexpr KHardwareTimer() : KHardwareTimerBase() { /* ... */ }
+        public:
+            /* Public API. */
+            NOINLINE void Initialize(s32 core_id);
+            NOINLINE void Finalize();
+
+            static s64 GetTick() {
+                return GetCount();
+            }
+
+            void RegisterAbsoluteTask(KTimerTask *task, s64 task_time) {
+                KScopedDisableDispatch dd;
+                KScopedSpinLock lk(this->GetLock());
+
+                if (this->RegisterAbsoluteTaskImpl(task, task_time)) {
+                    SetCompareValue(task_time);
+                    EnableInterrupt();
+                }
+            }
+        private:
+            friend class impl::KHardwareTimerInterruptTask;
+            NOINLINE void DoInterruptTask();
+        private:
+            /* Hardware register accessors. */
+            static ALWAYS_INLINE void InitializeGlobalTimer() {
+                /* Set kernel control. */
+                cpu::CounterTimerKernelControlRegisterAccessor(0).SetEl0PctEn(true).Store();
+
+                /* Disable the physical timer. */
+                cpu::CounterTimerPhysicalTimerControlRegisterAccessor(0).SetEnable(false).SetIMask(false).Store();
+
+                /* Set the compare value to the maximum. */
+                cpu::CounterTimerPhysicalTimerCompareValueRegisterAccessor(0).SetCompareValue(std::numeric_limits<u64>::max()).Store();
+
+                /* Enable the physical timer, with interrupt masked. */
+                cpu::CounterTimerPhysicalTimerControlRegisterAccessor(0).SetEnable(true).SetIMask(true).Store();
+            }
+
+            static ALWAYS_INLINE void EnableInterrupt() {
+                cpu::CounterTimerPhysicalTimerControlRegisterAccessor(0).SetEnable(true).SetIMask(false).Store();
+            }
+
+            static ALWAYS_INLINE void DisableInterrupt() {
+                cpu::CounterTimerPhysicalTimerControlRegisterAccessor(0).SetEnable(true).SetIMask(true).Store();
+            }
+
+            static ALWAYS_INLINE void StopTimer() {
+                /* Set the compare value to the maximum. */
+                cpu::CounterTimerPhysicalTimerCompareValueRegisterAccessor(0).SetCompareValue(std::numeric_limits<u64>::max()).Store();
+
+                /* Disable the physical timer. */
+                cpu::CounterTimerPhysicalTimerControlRegisterAccessor(0).SetEnable(false).SetIMask(false).Store();
+            }
+
+            static ALWAYS_INLINE s64 GetCount() {
+                return cpu::CounterTimerPhysicalCountValueRegisterAccessor().GetCount();
+            }
+
+            static ALWAYS_INLINE void SetCompareValue(s64 value) {
+                cpu::CounterTimerPhysicalTimerCompareValueRegisterAccessor(0).SetCompareValue(static_cast<u64>(value)).Store();
+            }
+
+    };
+
+}

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_interrupt_controller.hpp

@@ -0,0 +1,275 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <vapours.hpp>
+#include <mesosphere/kern_select_cpu.hpp>
+#include <mesosphere/kern_k_typed_address.hpp>
+
+namespace ams::kern::arch::arm64 {
+
+    struct GicDistributor {
+        u32 ctlr;
+        u32 typer;
+        u32 iidr;
+        u32 reserved_0x0c;
+        u32 statusr;
+        u32 reserved_0x14[3];
+        u32 impldef_0x20[8];
+        u32 setspi_nsr;
+        u32 reserved_0x44;
+        u32 clrspi_nsr;
+        u32 reserved_0x4c;
+        u32 setspi_sr;
+        u32 reserved_0x54;
+        u32 clrspi_sr;
+        u32 reserved_0x5c[9];
+        u32 igroupr[32];
+        u32 isenabler[32];
+        u32 icenabler[32];
+        u32 ispendr[32];
+        u32 icpendr[32];
+        u32 isactiver[32];
+        u32 icactiver[32];
+        union {
+            u8  bytes[1020];
+            u32 words[255];
+        } ipriorityr;
+        u32 _0x7fc;
+        union {
+            u8  bytes[1020];
+            u32 words[255];
+        } itargetsr;
+        u32 _0xbfc;
+        u32 icfgr[64];
+        u32 igrpmodr[32];
+        u32 _0xd80[32];
+        u32 nsacr[64];
+        u32 sgir;
+        u32 _0xf04[3];
+        u32 cpendsgir[4];
+        u32 spendsgir[4];
+        u32 reserved_0xf30[52];
+
+        static constexpr size_t SgirCpuTargetListShift = 16;
+
+        enum SgirTargetListFilter : u32 {
+            SgirTargetListFilter_CpuTargetList = (0 << 24),
+            SgirTargetListFilter_Others        = (1 << 24),
+            SgirTargetListFilter_Self          = (2 << 24),
+            SgirTargetListFilter_Reserved      = (3 << 24),
+        };
+    };
+    static_assert(std::is_pod<GicDistributor>::value);
+    static_assert(sizeof(GicDistributor) == 0x1000);
+
+    struct GicCpuInterface {
+        u32 ctlr;
+        u32 pmr;
+        u32 bpr;
+        u32 iar;
+        u32 eoir;
+        u32 rpr;
+        u32 hppir;
+        u32 abpr;
+        u32 aiar;
+        u32 aeoir;
+        u32 ahppir;
+        u32 statusr;
+        u32 reserved_30[4];
+        u32 impldef_40[36];
+        u32 apr[4];
+        u32 nsapr[4];
+        u32 reserved_f0[3];
+        u32 iidr;
+        u32 reserved_100[960];
+        u32 dir;
+        u32 _0x1004[1023];
+    };
+    static_assert(std::is_pod<GicCpuInterface>::value);
+    static_assert(sizeof(GicCpuInterface) == 0x2000);
+
+    struct KInterruptController {
+        NON_COPYABLE(KInterruptController);
+        NON_MOVEABLE(KInterruptController);
+        public:
+            static constexpr s32 NumSoftwareInterrupts = 16;
+            static constexpr s32 NumLocalInterrupts = NumSoftwareInterrupts + 16;
+            static constexpr s32 NumGlobalInterrupts = 988;
+            static constexpr s32 NumInterrupts = NumLocalInterrupts + NumGlobalInterrupts;
+            static constexpr s32 NumPriorityLevels = 4;
+        public:
+            struct LocalState {
+                u32 local_isenabler[NumLocalInterrupts / 32];
+                u32 local_ipriorityr[NumLocalInterrupts / 4];
+                u32 local_targetsr[NumLocalInterrupts / 4];
+                u32 local_icfgr[NumLocalInterrupts / 16];
+            };
+
+            struct GlobalState {
+                u32 global_isenabler[NumGlobalInterrupts / 32];
+                u32 global_ipriorityr[NumGlobalInterrupts / 4];
+                u32 global_targetsr[NumGlobalInterrupts / 4];
+                u32 global_icfgr[NumGlobalInterrupts / 16];
+            };
+
+            enum PriorityLevel : u8 {
+                PriorityLevel_High  = 0,
+                PriorityLevel_Low   = NumPriorityLevels - 1,
+
+                PriorityLevel_Timer     = 1,
+                PriorityLevel_Scheduler = 2,
+            };
+        private:
+            static inline u32 s_mask[cpu::NumCores];
+        private:
+            volatile GicDistributor  *gicd;
+            volatile GicCpuInterface *gicc;
+        public:
+            constexpr KInterruptController() : gicd(nullptr), gicc(nullptr) { /* ... */ }
+
+            void Initialize(s32 core_id);
+            void Finalize(s32 core_id);
+        public:
+            u32 GetIrq() const {
+                return this->gicc->iar;
+            }
+
+            static constexpr s32 ConvertRawIrq(u32 irq) {
+                return (irq == 0x3FF) ? -1 : (irq & 0x3FF);
+            }
+
+            void Enable(s32 irq) const {
+                this->gicd->isenabler[irq / BITSIZEOF(u32)] = (1u << (irq % BITSIZEOF(u32)));
+            }
+
+            void Disable(s32 irq) const {
+                this->gicd->icenabler[irq / BITSIZEOF(u32)] = (1u << (irq % BITSIZEOF(u32)));
+            }
+
+            void Clear(s32 irq) const {
+                this->gicd->icpendr[irq / BITSIZEOF(u32)] = (1u << (irq % BITSIZEOF(u32)));
+            }
+
+            void SetTarget(s32 irq, s32 core_id) const {
+                this->gicd->itargetsr.bytes[irq] |= GetGicMask(core_id);
+            }
+
+            void ClearTarget(s32 irq, s32 core_id) const {
+                this->gicd->itargetsr.bytes[irq] &= ~GetGicMask(core_id);
+            }
+
+            void SetPriorityLevel(s32 irq, s32 level) const {
+                MESOSPHERE_ASSERT(PriorityLevel_High <= level && level <= PriorityLevel_Low);
+                this->gicd->ipriorityr.bytes[irq] = ToGicPriorityValue(level);
+            }
+
+            s32 GetPriorityLevel(s32 irq) const {
+                return FromGicPriorityValue(this->gicd->ipriorityr.bytes[irq]);
+            }
+
+            void SetPriorityLevel(s32 level) const {
+                MESOSPHERE_ASSERT(PriorityLevel_High <= level && level <= PriorityLevel_Low);
+                this->gicc->pmr = ToGicPriorityValue(level);
+            }
+
+            void SetEdge(s32 irq) const {
+                u32 cfg = this->gicd->icfgr[irq / (BITSIZEOF(u32) / 2)];
+                cfg &= ~(0x3 << (2 * (irq % (BITSIZEOF(u32) / 2))));
+                cfg |=  (0x2 << (2 * (irq % (BITSIZEOF(u32) / 2))));
+                this->gicd->icfgr[irq / (BITSIZEOF(u32) / 2)] = cfg;
+            }
+
+            void SetLevel(s32 irq) const {
+                u32 cfg = this->gicd->icfgr[irq / (BITSIZEOF(u32) / 2)];
+                cfg &= ~(0x3 << (2 * (irq % (BITSIZEOF(u32) / 2))));
+                cfg |=  (0x0 << (2 * (irq % (BITSIZEOF(u32) / 2))));
+                this->gicd->icfgr[irq / (BITSIZEOF(u32) / 2)] = cfg;
+            }
+
+            void SendInterProcessorInterrupt(s32 irq, u64 core_mask) {
+                MESOSPHERE_ASSERT(IsSoftware(irq));
+                this->gicd->sgir = GetCpuTargetListMask(irq, core_mask);
+            }
+
+            void SendInterProcessorInterrupt(s32 irq) {
+                MESOSPHERE_ASSERT(IsSoftware(irq));
+                this->gicd->sgir = GicDistributor::SgirTargetListFilter_Others | irq;
+            }
+
+            void EndOfInterrupt(u32 irq) const {
+                this->gicc->eoir = irq;
+            }
+
+            bool IsInterruptDefined(s32 irq) {
+                const s32 num_interrupts = std::min(32 + 32 * (this->gicd->typer & 0x1F), static_cast<u32>(NumInterrupts));
+                return (0 <= irq && irq < num_interrupts);
+            }
+
+            /* TODO: Implement more KInterruptController functionality. */
+        public:
+            static constexpr ALWAYS_INLINE bool IsSoftware(s32 id) {
+                MESOSPHERE_ASSERT(0 <= id && id < NumInterrupts);
+                return id < NumSoftwareInterrupts;
+            }
+
+            static constexpr ALWAYS_INLINE bool IsLocal(s32 id) {
+                MESOSPHERE_ASSERT(0 <= id && id < NumInterrupts);
+                return id < NumLocalInterrupts;
+            }
+
+            static constexpr ALWAYS_INLINE bool IsGlobal(s32 id) {
+                MESOSPHERE_ASSERT(0 <= id && id < NumInterrupts);
+                return NumLocalInterrupts <= id;
+            }
+
+            static constexpr size_t GetGlobalInterruptIndex(s32 id) {
+                MESOSPHERE_ASSERT(IsGlobal(id));
+                return id - NumLocalInterrupts;
+            }
+
+            static constexpr size_t GetLocalInterruptIndex(s32 id) {
+                MESOSPHERE_ASSERT(IsLocal(id));
+                return id;
+            }
+        private:
+            static constexpr size_t PriorityShift = BITSIZEOF(u8) - __builtin_ctz(NumPriorityLevels);
+            static_assert(PriorityShift < BITSIZEOF(u8));
+
+            static constexpr ALWAYS_INLINE u8 ToGicPriorityValue(s32 level) {
+                return (level << PriorityShift) | ((1 << PriorityShift) - 1);
+            }
+
+            static constexpr ALWAYS_INLINE s32 FromGicPriorityValue(u8 priority) {
+                return (priority >> PriorityShift) & (NumPriorityLevels - 1);
+            }
+
+            static constexpr ALWAYS_INLINE s32 GetCpuTargetListMask(s32 irq, u64 core_mask) {
+                MESOSPHERE_ASSERT(IsSoftware(irq));
+                MESOSPHERE_ASSERT(core_mask < (1ul << cpu::NumCores));
+                return GicDistributor::SgirTargetListFilter_CpuTargetList | irq | (static_cast<u16>(core_mask) << GicDistributor::SgirCpuTargetListShift);
+            }
+
+            static ALWAYS_INLINE s32 GetGicMask(s32 core_id) {
+                return s_mask[core_id];
+            }
+
+            ALWAYS_INLINE void SetGicMask(s32 core_id) const {
+                s_mask[core_id] = this->gicd->itargetsr.bytes[0];
+            }
+
+            NOINLINE void SetupInterruptLines(s32 core_id) const;
+    };
+}

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_interrupt_manager.hpp

@@ -0,0 +1,130 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <vapours.hpp>
+#include <mesosphere/kern_select_cpu.hpp>
+#include <mesosphere/kern_k_spin_lock.hpp>
+#include <mesosphere/kern_k_interrupt_task.hpp>
+#include <mesosphere/kern_select_interrupt_controller.hpp>
+
+namespace ams::kern::arch::arm64 {
+
+    class KInterruptManager {
+        NON_COPYABLE(KInterruptManager);
+        NON_MOVEABLE(KInterruptManager);
+        private:
+            struct KCoreLocalInterruptEntry {
+                KInterruptHandler *handler;
+                bool manually_cleared;
+                bool needs_clear;
+                u8 priority;
+
+                constexpr KCoreLocalInterruptEntry()
+                    : handler(nullptr), manually_cleared(false), needs_clear(false), priority(KInterruptController::PriorityLevel_Low)
+                {
+                    /* ... */
+                }
+            };
+
+            struct KGlobalInterruptEntry {
+                KInterruptHandler *handler;
+                bool manually_cleared;
+                bool needs_clear;
+
+                constexpr KGlobalInterruptEntry() : handler(nullptr), manually_cleared(false), needs_clear(false) { /* ... */ }
+            };
+        private:
+            static KSpinLock s_lock;
+            static std::array<KGlobalInterruptEntry, KInterruptController::NumGlobalInterrupts> s_global_interrupts;
+            static KInterruptController::GlobalState s_global_state;
+            static bool s_global_state_saved;
+        private:
+            KCoreLocalInterruptEntry core_local_interrupts[KInterruptController::NumLocalInterrupts];
+            KInterruptController interrupt_controller;
+            KInterruptController::LocalState local_state;
+            bool local_state_saved;
+        private:
+            static ALWAYS_INLINE KSpinLock &GetLock() { return s_lock; }
+            static ALWAYS_INLINE KGlobalInterruptEntry &GetGlobalInterruptEntry(s32 irq) { return s_global_interrupts[KInterruptController::GetGlobalInterruptIndex(irq)]; }
+            ALWAYS_INLINE KCoreLocalInterruptEntry &GetLocalInterruptEntry(s32 irq) { return this->core_local_interrupts[KInterruptController::GetLocalInterruptIndex(irq)]; }
+
+            bool OnHandleInterrupt();
+        public:
+            constexpr KInterruptManager() : core_local_interrupts(), interrupt_controller(), local_state(), local_state_saved(false) { /* ... */ }
+            NOINLINE void Initialize(s32 core_id);
+            NOINLINE void Finalize(s32 core_id);
+
+            bool IsInterruptDefined(s32 irq) {
+                return this->interrupt_controller.IsInterruptDefined(irq);
+            }
+
+            NOINLINE Result BindHandler(KInterruptHandler *handler, s32 irq, s32 core_id, s32 priority, bool manual_clear, bool level);
+            NOINLINE Result UnbindHandler(s32 irq, s32 core);
+
+            NOINLINE Result ClearInterrupt(s32 irq);
+            NOINLINE Result ClearInterrupt(s32 irq, s32 core_id);
+
+            ALWAYS_INLINE void SendInterProcessorInterrupt(s32 irq, u64 core_mask) {
+                this->interrupt_controller.SendInterProcessorInterrupt(irq, core_mask);
+            }
+
+            ALWAYS_INLINE void SendInterProcessorInterrupt(s32 irq) {
+                this->interrupt_controller.SendInterProcessorInterrupt(irq);
+            }
+
+            static void HandleInterrupt(bool user_mode);
+
+            /* Implement more KInterruptManager functionality. */
+        private:
+            Result BindGlobal(KInterruptHandler *handler, s32 irq, s32 core_id, s32 priority, bool manual_clear, bool level);
+            Result BindLocal(KInterruptHandler *handler, s32 irq, s32 priority, bool manual_clear);
+            Result UnbindGlobal(s32 irq);
+            Result UnbindLocal(s32 irq);
+            Result ClearGlobal(s32 irq);
+            Result ClearLocal(s32 irq);
+        public:
+            static ALWAYS_INLINE u32 DisableInterrupts() {
+                u64 intr_state;
+                __asm__ __volatile__("mrs %[intr_state], daif\n"
+                                     "msr daifset, #2"
+                                     : [intr_state]"=r"(intr_state)
+                                     :: "memory");
+                return intr_state;
+            }
+
+            static ALWAYS_INLINE u32 EnableInterrupts() {
+                u64 intr_state;
+                __asm__ __volatile__("mrs %[intr_state], daif\n"
+                                     "msr daifclr, #2"
+                                     : [intr_state]"=r"(intr_state)
+                                     :: "memory");
+                return intr_state;
+            }
+
+            static ALWAYS_INLINE void RestoreInterrupts(u32 intr_state) {
+                u64 cur_state;
+                __asm__ __volatile__("mrs %[cur_state], daif" : [cur_state]"=r"(cur_state));
+                __asm__ __volatile__("msr daif, %[intr_state]" :: [intr_state]"r"((cur_state & ~0x80ul) | (intr_state & 0x80)));
+            }
+
+            static ALWAYS_INLINE bool AreInterruptsEnabled() {
+                u64 intr_state;
+                __asm__ __volatile__("mrs %[intr_state], daif" : [intr_state]"=r"(intr_state));
+                return (intr_state & 0x80) == 0;
+            }
+    };
+
+}

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_interrupt_name.hpp

@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+
+namespace ams::kern::arch::arm64 {
+
+    namespace interrupt_name {
+
+        enum KInterruptName : s32 {
+            /* SGIs */
+            KInterruptName_ThreadTerminate        = 4,
+            KInterruptName_CacheOperation         = 5,
+            KInterruptName_Scheduler              = 6,
+
+            KInterruptName_PerformanceCounter     = 8,
+
+            /* PPIs */
+    #if defined(ATMOSPHERE_BOARD_NINTENDO_NX)
+            KInterruptName_VirtualMaintenance     = 25,
+            KInterruptName_HypervisorTimer        = 26,
+            KInterruptName_VirtualTimer           = 27,
+            KInterruptName_LegacyNFiq             = 38,
+            KInterruptName_SecurePhysicalTimer    = 29,
+            KInterruptName_NonSecurePhysicalTimer = 30,
+            KInterruptName_LegacyNIrq             = 31,
+    #endif
+
+    #if defined(ATMOSPHERE_BOARD_NINTENDO_NX)
+            KInterruptName_MemoryController       = 109,
+    #endif
+        };
+
+    };
+
+}

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table.hpp

@@ -0,0 +1,275 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_select_cpu.hpp>
+#include <mesosphere/kern_k_page_table_base.hpp>
+#include <mesosphere/kern_k_page_group.hpp>
+#include <mesosphere/kern_k_page_table_manager.hpp>
+
+namespace ams::kern::arch::arm64 {
+
+    class KPageTable : public KPageTableBase {
+        NON_COPYABLE(KPageTable);
+        NON_MOVEABLE(KPageTable);
+        public:
+            using TraversalEntry   = KPageTableImpl::TraversalEntry;
+            using TraversalContext = KPageTableImpl::TraversalContext;
+
+            enum BlockType {
+                BlockType_L3Block,
+                BlockType_L3ContiguousBlock,
+                BlockType_L2Block,
+
+#ifdef ATMOSPHERE_BOARD_NINTENDO_NX
+                BlockType_L2TegraSmmuBlock,
+#endif
+
+                BlockType_L2ContiguousBlock,
+                BlockType_L1Block,
+
+                BlockType_Count,
+            };
+            static_assert(L3BlockSize == PageSize);
+            static constexpr size_t ContiguousPageSize = L3ContiguousBlockSize;
+
+#ifdef ATMOSPHERE_BOARD_NINTENDO_NX
+            static constexpr size_t L2TegraSmmuBlockSize = 2 * L2BlockSize;
+#endif
+            static constexpr size_t BlockSizes[BlockType_Count] = {
+                [BlockType_L3Block]           = L3BlockSize,
+                [BlockType_L3ContiguousBlock] = L3ContiguousBlockSize,
+                [BlockType_L2Block]           = L2BlockSize,
+#ifdef ATMOSPHERE_BOARD_NINTENDO_NX
+                [BlockType_L2TegraSmmuBlock]  = L2TegraSmmuBlockSize,
+#endif
+                [BlockType_L2ContiguousBlock] = L2ContiguousBlockSize,
+                [BlockType_L1Block]           = L1BlockSize,
+            };
+
+            static constexpr BlockType GetMaxBlockType() {
+                return BlockType_L1Block;
+            }
+
+            static constexpr size_t GetBlockSize(BlockType type) {
+                return BlockSizes[type];
+            }
+
+            static constexpr BlockType GetBlockType(size_t size) {
+                switch (size) {
+                    case L3BlockSize:           return BlockType_L3Block;
+                    case L3ContiguousBlockSize: return BlockType_L3ContiguousBlock;
+                    case L2BlockSize:           return BlockType_L2Block;
+#ifdef ATMOSPHERE_BOARD_NINTENDO_NX
+                    case L2TegraSmmuBlockSize:  return BlockType_L2TegraSmmuBlock;
+#endif
+                    case L2ContiguousBlockSize: return BlockType_L2ContiguousBlock;
+                    case L1BlockSize:           return BlockType_L1Block;
+                    MESOSPHERE_UNREACHABLE_DEFAULT_CASE();
+                }
+            }
+
+            static constexpr size_t GetSmallerAlignment(size_t alignment) {
+                MESOSPHERE_ASSERT(alignment > L3BlockSize);
+                return KPageTable::GetBlockSize(static_cast<KPageTable::BlockType>(KPageTable::GetBlockType(alignment) - 1));
+            }
+
+            static constexpr size_t GetLargerAlignment(size_t alignment) {
+                MESOSPHERE_ASSERT(alignment < L1BlockSize);
+                return KPageTable::GetBlockSize(static_cast<KPageTable::BlockType>(KPageTable::GetBlockType(alignment) + 1));
+            }
+        private:
+            KPageTableManager *manager;
+            u64 ttbr;
+            u8 asid;
+        protected:
+            virtual Result Operate(PageLinkedList *page_list, KProcessAddress virt_addr, size_t num_pages, KPhysicalAddress phys_addr, bool is_pa_valid, const KPageProperties properties, OperationType operation, bool reuse_ll) override;
+            virtual Result Operate(PageLinkedList *page_list, KProcessAddress virt_addr, size_t num_pages, const KPageGroup &page_group, const KPageProperties properties, OperationType operation, bool reuse_ll) override;
+            virtual void   FinalizeUpdate(PageLinkedList *page_list) override;
+
+            KPageTableManager &GetPageTableManager() const { return *this->manager; }
+        private:
+            constexpr PageTableEntry GetEntryTemplate(const KPageProperties properties) const {
+                /* Set basic attributes. */
+                PageTableEntry entry;
+                entry.SetPrivilegedExecuteNever(true);
+                entry.SetAccessFlag(PageTableEntry::AccessFlag_Accessed);
+                entry.SetShareable(PageTableEntry::Shareable_InnerShareable);
+
+                if (!this->IsKernel()) {
+                    entry.SetGlobal(false);
+                }
+
+                /* Set page attribute. */
+                if (properties.io) {
+                    MESOSPHERE_ABORT_UNLESS(!properties.uncached);
+                    MESOSPHERE_ABORT_UNLESS((properties.perm & (KMemoryPermission_KernelExecute | KMemoryPermission_UserExecute)) == 0);
+
+                    entry.SetPageAttribute(PageTableEntry::PageAttribute_Device_nGnRnE)
+                         .SetUserExecuteNever(true);
+                } else if (properties.uncached) {
+                    MESOSPHERE_ABORT_UNLESS((properties.perm & (KMemoryPermission_KernelExecute | KMemoryPermission_UserExecute)) == 0);
+
+                    entry.SetPageAttribute(PageTableEntry::PageAttribute_NormalMemoryNotCacheable);
+                } else {
+                    entry.SetPageAttribute(PageTableEntry::PageAttribute_NormalMemory);
+                }
+
+                /* Set user execute never bit. */
+                if (properties.perm != KMemoryPermission_UserReadExecute) {
+                    MESOSPHERE_ABORT_UNLESS((properties.perm & (KMemoryPermission_KernelExecute | KMemoryPermission_UserExecute)) == 0);
+                    entry.SetUserExecuteNever(true);
+                }
+
+                /* Set can be contiguous. */
+                entry.SetContiguousAllowed(!properties.non_contiguous);
+
+                /* Set AP[1] based on perm. */
+                switch (properties.perm & KMemoryPermission_UserReadWrite) {
+                    case KMemoryPermission_UserReadWrite:
+                    case KMemoryPermission_UserRead:
+                        entry.SetUserAccessible(true);
+                        break;
+                    case KMemoryPermission_KernelReadWrite:
+                    case KMemoryPermission_KernelRead:
+                        entry.SetUserAccessible(false);
+                        break;
+                    MESOSPHERE_UNREACHABLE_DEFAULT_CASE();
+                }
+
+                /* Set AP[2] based on perm. */
+                switch (properties.perm & KMemoryPermission_UserReadWrite) {
+                    case KMemoryPermission_UserReadWrite:
+                    case KMemoryPermission_KernelReadWrite:
+                        entry.SetReadOnly(false);
+                        break;
+                    case KMemoryPermission_KernelRead:
+                    case KMemoryPermission_UserRead:
+                        entry.SetReadOnly(true);
+                        break;
+                    MESOSPHERE_UNREACHABLE_DEFAULT_CASE();
+                }
+
+                return entry;
+            }
+        public:
+            constexpr KPageTable() : KPageTableBase(), manager(), ttbr(), asid() { /* ... */ }
+
+            static NOINLINE void Initialize(s32 core_id);
+
+            ALWAYS_INLINE void Activate(u32 proc_id) {
+                cpu::DataSynchronizationBarrier();
+                cpu::SwitchProcess(this->ttbr, proc_id);
+            }
+
+            NOINLINE Result InitializeForKernel(void *table, KVirtualAddress start, KVirtualAddress end);
+            NOINLINE Result InitializeForProcess(u32 id, ams::svc::CreateProcessFlag as_type, bool enable_aslr, bool from_back, KMemoryManager::Pool pool, KProcessAddress code_address, size_t code_size, KMemoryBlockSlabManager *mem_block_slab_manager, KBlockInfoManager *block_info_manager, KPageTableManager *pt_manager);
+            Result Finalize();
+        private:
+            Result Map(KProcessAddress virt_addr, KPhysicalAddress phys_addr, size_t num_pages, PageTableEntry entry_template, PageLinkedList *page_list, bool reuse_ll);
+            Result Unmap(KProcessAddress virt_addr, size_t num_pages, PageLinkedList *page_list, bool force, bool reuse_ll);
+
+            Result Map(KProcessAddress virt_addr, KPhysicalAddress phys_addr, size_t num_pages, PageTableEntry entry_template, size_t page_size, PageLinkedList *page_list, bool reuse_ll) {
+                switch (page_size) {
+                    case L1BlockSize:
+#ifdef ATMOSPHERE_BOARD_NINTENDO_NX
+                    case L2TegraSmmuBlockSize:
+#endif
+                    case L2BlockSize:
+                    case L3BlockSize:
+                        break;
+                    case L2ContiguousBlockSize:
+                    case L3ContiguousBlockSize:
+                        entry_template.SetContiguous(true);
+                        break;
+                    MESOSPHERE_UNREACHABLE_DEFAULT_CASE();
+                }
+                return this->Map(virt_addr, phys_addr, num_pages, entry_template, page_list, reuse_ll);
+            }
+
+            Result MapContiguous(KProcessAddress virt_addr, KPhysicalAddress phys_addr, size_t num_pages, PageTableEntry entry_template, PageLinkedList *page_list, bool reuse_ll);
+            Result MapGroup(KProcessAddress virt_addr, const KPageGroup &pg, size_t num_pages, PageTableEntry entry_template, PageLinkedList *page_list, bool reuse_ll);
+
+            bool MergePages(KProcessAddress virt_addr, PageLinkedList *page_list);
+
+            ALWAYS_INLINE Result SeparatePagesImpl(KProcessAddress virt_addr, size_t block_size, PageLinkedList *page_list, bool reuse_ll);
+            Result SeparatePages(KProcessAddress virt_addr, size_t block_size, PageLinkedList *page_list, bool reuse_ll);
+
+            Result ChangePermissions(KProcessAddress virt_addr, size_t num_pages, PageTableEntry entry_template, bool refresh_mapping, PageLinkedList *page_list, bool reuse_ll);
+
+            static void PteDataSynchronizationBarrier() {
+                cpu::DataSynchronizationBarrierInnerShareable();
+            }
+
+            static void ClearPageTable(KVirtualAddress table) {
+                cpu::ClearPageToZero(GetVoidPointer(table));
+            }
+
+            void OnTableUpdated() const {
+                cpu::InvalidateTlbByAsid(this->asid);
+            }
+
+            void OnKernelTableUpdated() const {
+                cpu::InvalidateEntireTlbDataOnly();
+            }
+
+            void OnKernelTableSinglePageUpdated(KProcessAddress virt_addr) const {
+                cpu::InvalidateTlbByVaDataOnly(virt_addr);
+            }
+
+            void NoteUpdated() const {
+                cpu::DataSynchronizationBarrier();
+
+                if (this->IsKernel()) {
+                    this->OnKernelTableUpdated();
+                } else {
+                    this->OnTableUpdated();
+                }
+            }
+
+            void NoteSingleKernelPageUpdated(KProcessAddress virt_addr) const {
+                MESOSPHERE_ASSERT(this->IsKernel());
+
+                cpu::DataSynchronizationBarrier();
+                this->OnKernelTableSinglePageUpdated(virt_addr);
+            }
+
+            KVirtualAddress AllocatePageTable(PageLinkedList *page_list, bool reuse_ll) const {
+                KVirtualAddress table = this->GetPageTableManager().Allocate();
+
+                if (table == Null<KVirtualAddress>) {
+                    if (reuse_ll && page_list->Peek()) {
+                        table = KVirtualAddress(reinterpret_cast<uintptr_t>(page_list->Pop()));
+                    } else {
+                        return Null<KVirtualAddress>;
+                    }
+                }
+
+                ClearPageTable(table);
+
+                MESOSPHERE_ASSERT(this->GetPageTableManager().GetRefCount(table) == 0);
+
+                return table;
+            }
+
+            void FreePageTable(PageLinkedList *page_list, KVirtualAddress table) const {
+                MESOSPHERE_ASSERT(this->GetPageTableManager().IsInPageTableHeap(table));
+                MESOSPHERE_ASSERT(this->GetPageTableManager().GetRefCount(table) == 0);
+                page_list->Push(table);
+            }
+    };
+
+}

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table_entry.hpp

@@ -0,0 +1,299 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_select_cpu.hpp>
+#include <mesosphere/kern_k_typed_address.hpp>
+
+namespace ams::kern::arch::arm64 {
+
+    constexpr size_t L1BlockSize           = 1_GB;
+    constexpr size_t L1ContiguousBlockSize = 0x10 * L1BlockSize;
+    constexpr size_t L2BlockSize           = 2_MB;
+    constexpr size_t L2ContiguousBlockSize = 0x10 * L2BlockSize;
+    constexpr size_t L3BlockSize           = PageSize;
+    constexpr size_t L3ContiguousBlockSize = 0x10 * L3BlockSize;
+
+    class PageTableEntry {
+        public:
+            struct InvalidTag{};
+
+            enum ExtensionTag : u64 {
+                ExtensionTag_IsValidBit  = (1ul << 56),
+                ExtensionTag_IsValid     = (ExtensionTag_IsValidBit | (1ul << 0)),
+                ExtensionTag_IsBlockMask = (ExtensionTag_IsValidBit | (1ul << 1)),
+            };
+
+            enum Permission : u64 {
+                Permission_KernelRWX = ((0ul << 53) | (1ul << 54) | (0ul << 6)),
+                Permission_KernelRX  = ((0ul << 53) | (1ul << 54) | (2ul << 6)),
+                Permission_KernelR   = ((1ul << 53) | (1ul << 54) | (2ul << 6)),
+                Permission_KernelRW  = ((1ul << 53) | (1ul << 54) | (0ul << 6)),
+
+                Permission_UserRX    = ((1ul << 53) | (0ul << 54) | (3ul << 6)),
+                Permission_UserR     = ((1ul << 53) | (1ul << 54) | (3ul << 6)),
+                Permission_UserRW    = ((1ul << 53) | (1ul << 54) | (1ul << 6)),
+            };
+
+            enum Shareable : u64 {
+                Shareable_NonShareable   = (0 << 8),
+                Shareable_OuterShareable = (2 << 8),
+                Shareable_InnerShareable = (3 << 8),
+            };
+
+            /* Official attributes are: */
+            /*    0x00, 0x04, 0xFF, 0x44. 4-7 are unused. */
+            enum PageAttribute : u64 {
+                PageAttribute_Device_nGnRnE            = (0 << 2),
+                PageAttribute_Device_nGnRE             = (1 << 2),
+                PageAttribute_NormalMemory             = (2 << 2),
+                PageAttribute_NormalMemoryNotCacheable = (3 << 2),
+            };
+
+            enum AccessFlag : u64 {
+                AccessFlag_NotAccessed = (0 << 10),
+                AccessFlag_Accessed    = (1 << 10),
+            };
+
+            enum Type : u64 {
+                Type_None    = 0x0,
+                Type_L1Block = 0x1,
+                Type_L1Table = 0x3,
+                Type_L2Block = 0x1,
+                Type_L2Table = 0x3,
+                Type_L3Block = 0x3,
+            };
+
+            enum ContigType : u64 {
+                ContigType_NotContiguous = (0x0ul << 52),
+                ContigType_Contiguous    = (0x1ul << 52),
+            };
+        protected:
+            u64 attributes;
+        public:
+            /* Take in a raw attribute. */
+            constexpr ALWAYS_INLINE PageTableEntry() : attributes() { /* ... */ }
+            constexpr ALWAYS_INLINE PageTableEntry(u64 attr) : attributes(attr) { /* ... */ }
+
+            constexpr ALWAYS_INLINE PageTableEntry(InvalidTag) : attributes(0) { /* ... */ }
+
+            /* Extend a previous attribute. */
+            constexpr ALWAYS_INLINE PageTableEntry(const PageTableEntry &rhs, u64 new_attr) : attributes(rhs.attributes | new_attr) { /* ... */ }
+
+            /* Construct a new attribute. */
+            constexpr ALWAYS_INLINE PageTableEntry(Permission perm, PageAttribute p_a, Shareable share)
+                : attributes(static_cast<u64>(perm) | static_cast<u64>(AccessFlag_Accessed) | static_cast<u64>(p_a) | static_cast<u64>(share) | static_cast<u64>(ExtensionTag_IsValid))
+            {
+                /* ... */
+            }
+        protected:
+            constexpr ALWAYS_INLINE u64 GetBits(size_t offset, size_t count) const {
+                return (this->attributes >> offset) & ((1ul << count) - 1);
+            }
+
+            constexpr ALWAYS_INLINE u64 SelectBits(size_t offset, size_t count) const {
+                return this->attributes & (((1ul << count) - 1) << offset);
+            }
+
+            constexpr ALWAYS_INLINE void SetBits(size_t offset, size_t count, u64 value) {
+                const u64 mask = ((1ul << count) - 1) << offset;
+                this->attributes &= ~mask;
+                this->attributes |= (value & (mask >> offset)) << offset;
+            }
+
+            constexpr ALWAYS_INLINE void SetBitsDirect(size_t offset, size_t count, u64 value) {
+                const u64 mask = ((1ul << count) - 1) << offset;
+                this->attributes &= ~mask;
+                this->attributes |= (value & mask);
+            }
+
+            constexpr ALWAYS_INLINE void SetBit(size_t offset, bool enabled) {
+                const u64 mask = 1ul << offset;
+                if (enabled) {
+                    this->attributes |= mask;
+                } else {
+                    this->attributes &= ~mask;
+                }
+            }
+        public:
+            constexpr ALWAYS_INLINE bool IsContiguousAllowed()       const { return this->GetBits(55, 1) != 0; }
+            constexpr ALWAYS_INLINE bool IsUserExecuteNever()        const { return this->GetBits(54, 1) != 0; }
+            constexpr ALWAYS_INLINE bool IsPrivilegedExecuteNever()  const { return this->GetBits(53, 1) != 0; }
+            constexpr ALWAYS_INLINE bool IsContiguous()              const { return this->GetBits(52, 1) != 0; }
+            constexpr ALWAYS_INLINE bool IsGlobal()                  const { return this->GetBits(11, 1) == 0; }
+            constexpr ALWAYS_INLINE AccessFlag GetAccessFlag()       const { return static_cast<AccessFlag>(this->GetBits(10, 1)); }
+            constexpr ALWAYS_INLINE Shareable GetShareable()         const { return static_cast<Shareable>(this->GetBits(8, 2)); }
+            constexpr ALWAYS_INLINE PageAttribute GetPageAttribute() const { return static_cast<PageAttribute>(this->GetBits(2, 3)); }
+            constexpr ALWAYS_INLINE bool IsReadOnly()                const { return this->GetBits(7, 1) != 0; }
+            constexpr ALWAYS_INLINE bool IsUserAccessible()          const { return this->GetBits(6, 1) != 0; }
+            constexpr ALWAYS_INLINE bool IsNonSecure()               const { return this->GetBits(5, 1) != 0; }
+            constexpr ALWAYS_INLINE bool IsBlock()                   const { return (this->attributes & ExtensionTag_IsBlockMask) == ExtensionTag_IsValidBit; }
+            constexpr ALWAYS_INLINE bool IsTable()                   const { return this->GetBits(0, 2) == 0x3; }
+            constexpr ALWAYS_INLINE bool IsEmpty()                   const { return this->GetBits(0, 2) == 0x0; }
+
+            constexpr ALWAYS_INLINE decltype(auto) SetContiguousAllowed(bool en)      { this->SetBit(55, !en); return *this; }
+            constexpr ALWAYS_INLINE decltype(auto) SetUserExecuteNever(bool en)       { this->SetBit(54, en); return *this; }
+            constexpr ALWAYS_INLINE decltype(auto) SetPrivilegedExecuteNever(bool en) { this->SetBit(53, en); return *this; }
+            constexpr ALWAYS_INLINE decltype(auto) SetContiguous(bool en)             { this->SetBit(52, en); return *this; }
+            constexpr ALWAYS_INLINE decltype(auto) SetGlobal(bool en)                 { this->SetBit(11, !en); return *this; }
+            constexpr ALWAYS_INLINE decltype(auto) SetAccessFlag(AccessFlag f)        { this->SetBitsDirect(10, 1, f); return *this; }
+            constexpr ALWAYS_INLINE decltype(auto) SetShareable(Shareable s)          { this->SetBitsDirect(8, 2, s); return *this; }
+            constexpr ALWAYS_INLINE decltype(auto) SetReadOnly(bool en)               { this->SetBit(7, en); return *this; }
+            constexpr ALWAYS_INLINE decltype(auto) SetUserAccessible(bool en)         { this->SetBit(6, en); return *this; }
+            constexpr ALWAYS_INLINE decltype(auto) SetPageAttribute(PageAttribute a)  { this->SetBitsDirect(2, 3, a); return *this; }
+
+            constexpr ALWAYS_INLINE u64 GetEntryTemplate() const {
+                constexpr u64 Mask = (0xFFF0000000000FFFul & ~u64(0x3ul | (0x1ul << 52)));
+                return this->attributes & Mask;
+            }
+
+            constexpr ALWAYS_INLINE bool Is(u64 attr) const {
+                return this->attributes == attr;
+            }
+
+            constexpr ALWAYS_INLINE u64 GetRawAttributesUnsafeForSwap() const {
+                return this->attributes;
+            }
+
+        protected:
+            constexpr ALWAYS_INLINE u64 GetRawAttributes() const {
+                return this->attributes;
+            }
+    };
+
+    static_assert(sizeof(PageTableEntry) == sizeof(u64));
+
+    constexpr inline PageTableEntry InvalidPageTableEntry = PageTableEntry(PageTableEntry::InvalidTag{});
+
+    constexpr inline size_t MaxPageTableEntries = PageSize / sizeof(PageTableEntry);
+
+    class L1PageTableEntry : public PageTableEntry {
+        public:
+            constexpr ALWAYS_INLINE L1PageTableEntry(InvalidTag) : PageTableEntry(InvalidTag{}) { /* ... */ }
+
+            constexpr ALWAYS_INLINE L1PageTableEntry(KPhysicalAddress phys_addr, bool pxn)
+                : PageTableEntry((0x3ul << 60) | (static_cast<u64>(pxn) << 59) | GetInteger(phys_addr) | 0x3)
+            {
+                /* ... */
+            }
+
+            constexpr ALWAYS_INLINE L1PageTableEntry(KPhysicalAddress phys_addr, bool is_kernel, bool pxn)
+                : PageTableEntry(((is_kernel ? 0x3ul : 0) << 60) | (static_cast<u64>(pxn) << 59) | GetInteger(phys_addr) | 0x3)
+            {
+                /* ... */
+            }
+
+            constexpr ALWAYS_INLINE L1PageTableEntry(KPhysicalAddress phys_addr, const PageTableEntry &attr, bool contig)
+                : PageTableEntry(attr, (static_cast<u64>(contig) << 52) | GetInteger(phys_addr) | PageTableEntry::ExtensionTag_IsValid)
+            {
+                /* ... */
+            }
+
+            constexpr ALWAYS_INLINE KPhysicalAddress GetBlock() const {
+                return this->SelectBits(30, 18);
+            }
+
+            constexpr ALWAYS_INLINE KPhysicalAddress GetTable() const {
+                return this->SelectBits(12, 36);
+            }
+
+            constexpr ALWAYS_INLINE bool GetTable(KPhysicalAddress &out) const {
+                if (this->IsTable()) {
+                    out = this->GetTable();
+                    return true;
+                } else {
+                    return false;
+                }
+            }
+
+            constexpr ALWAYS_INLINE bool IsCompatibleWithAttribute(const PageTableEntry &rhs, bool contig) const {
+                /* Check whether this has the same permission/etc as the desired attributes. */
+                return L1PageTableEntry(this->GetBlock(), rhs, contig).GetRawAttributes() == this->GetRawAttributes();
+            }
+    };
+
+    class L2PageTableEntry : public PageTableEntry {
+        public:
+            constexpr ALWAYS_INLINE L2PageTableEntry(InvalidTag) : PageTableEntry(InvalidTag{}) { /* ... */ }
+
+            constexpr ALWAYS_INLINE L2PageTableEntry(KPhysicalAddress phys_addr, bool pxn)
+                : PageTableEntry((0x3ul << 60) | (static_cast<u64>(pxn) << 59) | GetInteger(phys_addr) | 0x3)
+            {
+                /* ... */
+            }
+
+            constexpr ALWAYS_INLINE L2PageTableEntry(KPhysicalAddress phys_addr, bool is_kernel, bool pxn)
+                : PageTableEntry(((is_kernel ? 0x3ul : 0) << 60) | (static_cast<u64>(pxn) << 59) | GetInteger(phys_addr) | 0x3)
+            {
+                /* ... */
+            }
+
+            constexpr ALWAYS_INLINE L2PageTableEntry(KPhysicalAddress phys_addr, const PageTableEntry &attr, bool contig)
+                : PageTableEntry(attr, (static_cast<u64>(contig) << 52) | GetInteger(phys_addr) | PageTableEntry::ExtensionTag_IsValid)
+            {
+                /* ... */
+            }
+
+            constexpr ALWAYS_INLINE KPhysicalAddress GetBlock() const {
+                return this->SelectBits(21, 27);
+            }
+
+            constexpr ALWAYS_INLINE KPhysicalAddress GetTable() const {
+                return this->SelectBits(12, 36);
+            }
+
+            constexpr ALWAYS_INLINE bool GetTable(KPhysicalAddress &out) const {
+                if (this->IsTable()) {
+                    out = this->GetTable();
+                    return true;
+                } else {
+                    return false;
+                }
+            }
+
+            constexpr ALWAYS_INLINE bool IsCompatibleWithAttribute(const PageTableEntry &rhs, bool contig) const {
+                /* Check whether this has the same permission/etc as the desired attributes. */
+                return L2PageTableEntry(this->GetBlock(), rhs, contig).GetRawAttributes() == this->GetRawAttributes();
+            }
+    };
+
+    class L3PageTableEntry : public PageTableEntry {
+        public:
+            constexpr ALWAYS_INLINE L3PageTableEntry(InvalidTag) : PageTableEntry(InvalidTag{}) { /* ... */ }
+
+            constexpr ALWAYS_INLINE L3PageTableEntry(KPhysicalAddress phys_addr, const PageTableEntry &attr, bool contig)
+                : PageTableEntry(attr, (static_cast<u64>(contig) << 52) | GetInteger(phys_addr) | 0x2 | PageTableEntry::ExtensionTag_IsValid)
+            {
+                /* ... */
+            }
+
+            constexpr ALWAYS_INLINE bool IsBlock() const { return (GetRawAttributes() & ExtensionTag_IsBlockMask) == ExtensionTag_IsBlockMask; }
+
+            constexpr ALWAYS_INLINE KPhysicalAddress GetBlock() const {
+                return this->SelectBits(12, 36);
+            }
+
+            constexpr ALWAYS_INLINE bool IsCompatibleWithAttribute(const PageTableEntry &rhs, bool contig) const {
+                /* Check whether this has the same permission/etc as the desired attributes. */
+                return L3PageTableEntry(this->GetBlock(), rhs, contig).GetRawAttributes() == this->GetRawAttributes();
+            }
+    };
+
+    constexpr inline L1PageTableEntry InvalidL1PageTableEntry = L1PageTableEntry(PageTableEntry::InvalidTag{});
+    constexpr inline L2PageTableEntry InvalidL2PageTableEntry = L2PageTableEntry(PageTableEntry::InvalidTag{});
+    constexpr inline L3PageTableEntry InvalidL3PageTableEntry = L3PageTableEntry(PageTableEntry::InvalidTag{});
+
+}

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table_impl.hpp

@@ -0,0 +1,115 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_select_cpu.hpp>
+#include <mesosphere/kern_k_typed_address.hpp>
+#include <mesosphere/kern_k_memory_layout.hpp>
+#include <mesosphere/arch/arm64/kern_k_page_table_entry.hpp>
+
+namespace ams::kern::arch::arm64 {
+
+    class KPageTableImpl {
+        NON_COPYABLE(KPageTableImpl);
+        NON_MOVEABLE(KPageTableImpl);
+        public:
+            struct TraversalEntry {
+                KPhysicalAddress phys_addr;
+                size_t block_size;
+            };
+
+            struct TraversalContext {
+                const L1PageTableEntry *l1_entry;
+                const L2PageTableEntry *l2_entry;
+                const L3PageTableEntry *l3_entry;
+            };
+        private:
+            static constexpr size_t PageBits  = __builtin_ctzll(PageSize);
+            static constexpr size_t NumLevels = 3;
+            static constexpr size_t LevelBits = 9;
+            static_assert(NumLevels > 0);
+
+            template<size_t Offset, size_t Count>
+            static constexpr ALWAYS_INLINE u64 GetBits(u64 value) {
+                return (value >> Offset) & ((1ul << Count) - 1);
+            }
+
+            template<size_t Offset, size_t Count>
+            constexpr ALWAYS_INLINE u64 SelectBits(u64 value) {
+                return value & (((1ul << Count) - 1) << Offset);
+            }
+
+            static constexpr ALWAYS_INLINE uintptr_t GetL0Index(KProcessAddress addr) { return GetBits<PageBits + LevelBits * (NumLevels - 0), LevelBits>(GetInteger(addr)); }
+            static constexpr ALWAYS_INLINE uintptr_t GetL1Index(KProcessAddress addr) { return GetBits<PageBits + LevelBits * (NumLevels - 1), LevelBits>(GetInteger(addr)); }
+            static constexpr ALWAYS_INLINE uintptr_t GetL2Index(KProcessAddress addr) { return GetBits<PageBits + LevelBits * (NumLevels - 2), LevelBits>(GetInteger(addr)); }
+            static constexpr ALWAYS_INLINE uintptr_t GetL3Index(KProcessAddress addr) { return GetBits<PageBits + LevelBits * (NumLevels - 3), LevelBits>(GetInteger(addr)); }
+
+            static constexpr ALWAYS_INLINE uintptr_t GetL1Offset(KProcessAddress addr) { return GetBits<0, PageBits + LevelBits * (NumLevels - 1)>(GetInteger(addr)); }
+            static constexpr ALWAYS_INLINE uintptr_t GetL2Offset(KProcessAddress addr) { return GetBits<0, PageBits + LevelBits * (NumLevels - 2)>(GetInteger(addr)); }
+            static constexpr ALWAYS_INLINE uintptr_t GetL3Offset(KProcessAddress addr) { return GetBits<0, PageBits + LevelBits * (NumLevels - 3)>(GetInteger(addr)); }
+            static constexpr ALWAYS_INLINE uintptr_t GetContiguousL1Offset(KProcessAddress addr) { return GetBits<0, PageBits + LevelBits * (NumLevels - 1) + 4>(GetInteger(addr)); }
+            static constexpr ALWAYS_INLINE uintptr_t GetContiguousL2Offset(KProcessAddress addr) { return GetBits<0, PageBits + LevelBits * (NumLevels - 2) + 4>(GetInteger(addr)); }
+            static constexpr ALWAYS_INLINE uintptr_t GetContiguousL3Offset(KProcessAddress addr) { return GetBits<0, PageBits + LevelBits * (NumLevels - 3) + 4>(GetInteger(addr)); }
+
+            static ALWAYS_INLINE KVirtualAddress GetPageTableVirtualAddress(KPhysicalAddress addr) {
+                return KMemoryLayout::GetLinearVirtualAddress(addr);
+            }
+
+            ALWAYS_INLINE bool ExtractL1Entry(TraversalEntry *out_entry, TraversalContext *out_context, const L1PageTableEntry *l1_entry, KProcessAddress virt_addr) const;
+            ALWAYS_INLINE bool ExtractL2Entry(TraversalEntry *out_entry, TraversalContext *out_context, const L2PageTableEntry *l2_entry, KProcessAddress virt_addr) const;
+            ALWAYS_INLINE bool ExtractL3Entry(TraversalEntry *out_entry, TraversalContext *out_context, const L3PageTableEntry *l3_entry, KProcessAddress virt_addr) const;
+        private:
+            L1PageTableEntry *table;
+            bool is_kernel;
+            u32  num_entries;
+        public:
+            ALWAYS_INLINE KVirtualAddress GetTableEntry(KVirtualAddress table, size_t index) const {
+                return table + index * sizeof(PageTableEntry);
+            }
+
+            ALWAYS_INLINE L1PageTableEntry *GetL1Entry(KProcessAddress address) const {
+                return GetPointer<L1PageTableEntry>(GetTableEntry(KVirtualAddress(this->table), GetL1Index(address) & (this->num_entries - 1)));
+            }
+
+            ALWAYS_INLINE L2PageTableEntry *GetL2EntryFromTable(KVirtualAddress table, KProcessAddress address) const {
+                return GetPointer<L2PageTableEntry>(GetTableEntry(table, GetL2Index(address)));
+            }
+
+            ALWAYS_INLINE L2PageTableEntry *GetL2Entry(const L1PageTableEntry *entry, KProcessAddress address) const {
+                return GetL2EntryFromTable(KMemoryLayout::GetLinearVirtualAddress(entry->GetTable()), address);
+            }
+
+            ALWAYS_INLINE L3PageTableEntry *GetL3EntryFromTable(KVirtualAddress table, KProcessAddress address) const {
+                return GetPointer<L3PageTableEntry>(GetTableEntry(table, GetL3Index(address)));
+            }
+
+            ALWAYS_INLINE L3PageTableEntry *GetL3Entry(const L2PageTableEntry *entry, KProcessAddress address) const {
+                return GetL3EntryFromTable(KMemoryLayout::GetLinearVirtualAddress(entry->GetTable()), address);
+            }
+        public:
+            constexpr KPageTableImpl() : table(), is_kernel(), num_entries() { /* ... */ }
+
+            NOINLINE void InitializeForKernel(void *tb, KVirtualAddress start, KVirtualAddress end);
+            NOINLINE void InitializeForProcess(void *tb, KVirtualAddress start, KVirtualAddress end);
+            L1PageTableEntry *Finalize();
+
+            bool BeginTraversal(TraversalEntry *out_entry, TraversalContext *out_context, KProcessAddress address) const;
+            bool ContinueTraversal(TraversalEntry *out_entry, TraversalContext *context) const;
+
+            bool GetPhysicalAddress(KPhysicalAddress *out, KProcessAddress virt_addr) const;
+    };
+
+}

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_process_page_table.hpp

@@ -0,0 +1,122 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/arch/arm64/kern_k_page_table.hpp>
+
+namespace ams::kern::arch::arm64 {
+
+    class KProcessPageTable {
+        private:
+            KPageTable page_table;
+        public:
+            constexpr KProcessPageTable() : page_table() { /* ... */ }
+
+            void Activate(u64 id) {
+                /* Activate the page table with the specified contextidr. */
+                this->page_table.Activate(id);
+            }
+
+            Result Initialize(u32 id, ams::svc::CreateProcessFlag as_type, bool enable_aslr, bool from_back, KMemoryManager::Pool pool, KProcessAddress code_address, size_t code_size, KMemoryBlockSlabManager *mem_block_slab_manager, KBlockInfoManager *block_info_manager, KPageTableManager *pt_manager) {
+                return this->page_table.InitializeForProcess(id, as_type, enable_aslr, from_back, pool, code_address, code_size, mem_block_slab_manager, block_info_manager, pt_manager);
+            }
+
+            void Finalize() { this->page_table.Finalize(); }
+
+            Result SetMemoryPermission(KProcessAddress addr, size_t size, ams::svc::MemoryPermission perm) {
+                return this->page_table.SetMemoryPermission(addr, size, perm);
+            }
+
+            Result SetProcessMemoryPermission(KProcessAddress addr, size_t size, ams::svc::MemoryPermission perm) {
+                return this->page_table.SetProcessMemoryPermission(addr, size, perm);
+            }
+
+            Result SetHeapSize(KProcessAddress *out, size_t size) {
+                return this->page_table.SetHeapSize(out, size);
+            }
+
+            Result SetMaxHeapSize(size_t size) {
+                return this->page_table.SetMaxHeapSize(size);
+            }
+
+            Result QueryInfo(KMemoryInfo *out_info, ams::svc::PageInfo *out_page_info, KProcessAddress addr) const {
+                return this->page_table.QueryInfo(out_info, out_page_info, addr);
+            }
+
+            Result MapIo(KPhysicalAddress phys_addr, size_t size, KMemoryPermission perm) {
+                return this->page_table.MapIo(phys_addr, size, perm);
+            }
+
+            Result MapStatic(KPhysicalAddress phys_addr, size_t size, KMemoryPermission perm) {
+                return this->page_table.MapStatic(phys_addr, size, perm);
+            }
+
+            Result MapRegion(KMemoryRegionType region_type, KMemoryPermission perm) {
+                return this->page_table.MapRegion(region_type, perm);
+            }
+
+            Result MapPageGroup(KProcessAddress addr, const KPageGroup &pg, KMemoryState state, KMemoryPermission perm) {
+                return this->page_table.MapPageGroup(addr, pg, state, perm);
+            }
+
+            Result MapPages(KProcessAddress *out_addr, size_t num_pages, size_t alignment, KPhysicalAddress phys_addr, KMemoryState state, KMemoryPermission perm) {
+                return this->page_table.MapPages(out_addr, num_pages, alignment, phys_addr, state, perm);
+            }
+
+            Result MapPages(KProcessAddress *out_addr, size_t num_pages, KMemoryState state, KMemoryPermission perm) {
+                return this->page_table.MapPages(out_addr, num_pages, state, perm);
+            }
+
+            Result UnmapPages(KProcessAddress addr, size_t num_pages, KMemoryState state) {
+                return this->page_table.UnmapPages(addr, num_pages, state);
+            }
+
+            Result MakeAndOpenPageGroup(KPageGroup *out, KProcessAddress address, size_t num_pages, u32 state_mask, u32 state, u32 perm_mask, u32 perm, u32 attr_mask, u32 attr) {
+                return this->page_table.MakeAndOpenPageGroup(out, address, num_pages, state_mask, state, perm_mask, perm, attr_mask, attr);
+            }
+
+            bool GetPhysicalAddress(KPhysicalAddress *out, KProcessAddress address) const {
+                return this->page_table.GetPhysicalAddress(out, address);
+            }
+
+            bool Contains(KProcessAddress addr, size_t size) const { return this->page_table.Contains(addr, size); }
+            bool CanContain(KProcessAddress addr, size_t size, KMemoryState state) const { return this->page_table.CanContain(addr, size, state); }
+
+            KProcessAddress GetAddressSpaceStart()    const { return this->page_table.GetAddressSpaceStart(); }
+            KProcessAddress GetHeapRegionStart()      const { return this->page_table.GetHeapRegionStart(); }
+            KProcessAddress GetAliasRegionStart()     const { return this->page_table.GetAliasRegionStart(); }
+            KProcessAddress GetStackRegionStart()     const { return this->page_table.GetStackRegionStart(); }
+            KProcessAddress GetKernelMapRegionStart() const { return this->page_table.GetKernelMapRegionStart(); }
+            KProcessAddress GetAliasCodeRegionStart() const { return this->page_table.GetAliasCodeRegionStart(); }
+
+            size_t GetAddressSpaceSize()    const { return this->page_table.GetAddressSpaceSize(); }
+            size_t GetHeapRegionSize()      const { return this->page_table.GetHeapRegionSize(); }
+            size_t GetAliasRegionSize()     const { return this->page_table.GetAliasRegionSize(); }
+            size_t GetStackRegionSize()     const { return this->page_table.GetStackRegionSize(); }
+            size_t GetKernelMapRegionSize() const { return this->page_table.GetKernelMapRegionSize(); }
+            size_t GetAliasCodeRegionSize() const { return this->page_table.GetAliasCodeRegionSize(); }
+
+            KPhysicalAddress GetHeapPhysicalAddress(KVirtualAddress address) const {
+                /* TODO: Better way to convert address type? */
+                return this->page_table.GetHeapPhysicalAddress(address);
+            }
+
+            KBlockInfoManager *GetBlockInfoManager() {
+                return this->page_table.GetBlockInfoManager();
+            }
+    };
+
+}

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_spin_lock.hpp

@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <vapours.hpp>
+#include <mesosphere/kern_select_cpu.hpp>
+
+namespace ams::kern::arch::arm64 {
+
+    class KNotAlignedSpinLock {
+        private:
+            u32 packed_tickets;
+        public:
+            constexpr KNotAlignedSpinLock() : packed_tickets(0) { /* ... */ }
+
+            void Lock() {
+                u32 tmp0, tmp1;
+
+                __asm__ __volatile__(
+                    "    prfm   pstl1keep, %[packed_tickets]\n"
+                    "1:\n"
+                    "    ldaxr  %w[tmp0], %[packed_tickets]\n"
+                    "    add    %w[tmp0], %w[tmp0], #0x10000\n"
+                    "    stxr   %w[tmp1], %w[tmp0], %[packed_tickets]\n"
+                    "    cbnz   %w[tmp1], 1b\n"
+                    "    \n"
+                    "    and    %w[tmp1], %w[tmp0], #0xFFFF\n"
+                    "    cmp    %w[tmp1], %w[tmp0], lsr #16\n"
+                    "    b.eq   done"
+                    "    sevl\n"
+                    "2:\n"
+                    "    wfe\n"
+                    "    ldaxrh %w[tmp1], %[packed_tickets]\n"
+                    "    cmp    %w[tmp1], %w[tmp0], lsr #16\n"
+                    "    b.ne   2b\n"
+                    "done:\n"
+                    : [tmp0]"=&r"(tmp0), [tmp1]"=&r"(tmp1), [packed_tickets]"+Q"(this->packed_tickets)
+                    :
+                    : "cc", "memory"
+                );
+            }
+
+            void Unlock() {
+                const u32 value = this->packed_tickets + 1;
+                __asm__ __volatile__(
+                    "    stlrh %w[value], %[packed_tickets]\n"
+                    : [packed_tickets]"+Q"(this->packed_tickets)
+                    : [value]"r"(value)
+                    : "memory"
+                );
+            }
+    };
+    static_assert(sizeof(KNotAlignedSpinLock) == sizeof(u32));
+
+    class KAlignedSpinLock {
+        private:
+            alignas(cpu::DataCacheLineSize) u16 current_ticket;
+            alignas(cpu::DataCacheLineSize) u16 next_ticket;
+        public:
+            constexpr KAlignedSpinLock() : current_ticket(0), next_ticket(0) { /* ... */ }
+
+            void Lock() {
+                u32 tmp0, tmp1, got_lock;
+
+                __asm__ __volatile__(
+                    "    prfm   pstl1keep, %[next_ticket]\n"
+                    "1:\n"
+                    "    ldaxrh %w[tmp0], %[next_ticket]\n"
+                    "    add    %w[tmp1], %w[tmp0], #0x1\n"
+                    "    stxrh  %w[got_lock], %w[tmp1], %[next_ticket]\n"
+                    "    cbnz   %w[got_lock], 1b\n"
+                    "    \n"
+                    "    sevl\n"
+                    "2:\n"
+                    "    wfe\n"
+                    "    ldaxrh %w[tmp1], %[current_ticket]\n"
+                    "    cmp    %w[tmp1], %w[tmp0]\n"
+                    "    b.ne   2b\n"
+                    : [tmp0]"=&r"(tmp0), [tmp1]"=&r"(tmp1), [got_lock]"=&r"(got_lock), [next_ticket]"+Q"(this->next_ticket)
+                    : [current_ticket]"Q"(this->current_ticket)
+                    : "cc", "memory"
+                );
+            }
+
+            void Unlock() {
+                const u32 value = this->current_ticket + 1;
+                __asm__ __volatile__(
+                    "    stlrh %w[value], %[current_ticket]\n"
+                    : [current_ticket]"+Q"(this->current_ticket)
+                    : [value]"r"(value)
+                    : "memory"
+                );
+            }
+    };
+    static_assert(sizeof(KAlignedSpinLock) == 2 * cpu::DataCacheLineSize);
+
+    using KSpinLock = KAlignedSpinLock;
+
+}

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_supervisor_page_table.hpp

@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_select_cpu.hpp>
+#include <mesosphere/arch/arm64/kern_k_page_table.hpp>
+
+namespace ams::kern::arch::arm64 {
+
+    class KSupervisorPageTable {
+        private:
+            KPageTable page_table;
+            u64 ttbr0[cpu::NumCores];
+        public:
+            constexpr KSupervisorPageTable() : page_table(), ttbr0() { /* ... */ }
+
+            NOINLINE void Initialize(s32 core_id);
+
+            void Activate() {
+                /* Activate, using process id = 0xFFFFFFFF */
+                this->page_table.Activate(0xFFFFFFFF);
+            }
+
+            void ActivateForInit() {
+                this->Activate();
+
+                /* Invalidate entire TLB. */
+                cpu::InvalidateEntireTlb();
+            }
+
+            void Finalize(s32 core_id);
+
+            Result MapPages(KProcessAddress *out_addr, size_t num_pages, size_t alignment, KPhysicalAddress phys_addr, KProcessAddress region_start, size_t region_num_pages, KMemoryState state, KMemoryPermission perm) {
+                return this->page_table.MapPages(out_addr, num_pages, alignment, phys_addr, region_start, region_num_pages, state, perm);
+            }
+
+            Result UnmapPages(KProcessAddress address, size_t num_pages, KMemoryState state) {
+                return this->page_table.UnmapPages(address, num_pages, state);
+            }
+
+            Result MapPageGroup(KProcessAddress *out_addr, const KPageGroup &pg, KProcessAddress region_start, size_t region_num_pages, KMemoryState state, KMemoryPermission perm) {
+                return this->page_table.MapPageGroup(out_addr, pg, region_start, region_num_pages, state, perm);
+            }
+
+            Result UnmapPageGroup(KProcessAddress address, const KPageGroup &pg, KMemoryState state) {
+                return this->page_table.UnmapPageGroup(address, pg, state);
+            }
+
+            bool GetPhysicalAddress(KPhysicalAddress *out, KProcessAddress address) const {
+                return this->page_table.GetPhysicalAddress(out, address);
+            }
+    };
+
+}

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_thread_context.hpp

@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_k_typed_address.hpp>
+
+namespace ams::kern {
+
+    class KThread;
+
+}
+
+namespace ams::kern::arch::arm64 {
+
+    class KThreadContext {
+        public:
+            static constexpr size_t NumCalleeSavedRegisters = (29 - 19) + 1;
+            static constexpr size_t NumFpuRegisters = 32;
+        private:
+            union {
+                u64 registers[NumCalleeSavedRegisters];
+                struct {
+                    u64 x19;
+                    u64 x20;
+                    u64 x21;
+                    u64 x22;
+                    u64 x23;
+                    u64 x24;
+                    u64 x25;
+                    u64 x26;
+                    u64 x27;
+                    u64 x28;
+                    u64 x29;
+                };
+            } callee_saved;
+            u64 lr;
+            u64 sp;
+            u64 cpacr;
+            u64 fpcr;
+            u64 fpsr;
+            alignas(0x10) u128 fpu_registers[NumFpuRegisters];
+            bool locked;
+        private:
+            static void RestoreFpuRegisters64(const KThreadContext &);
+            static void RestoreFpuRegisters32(const KThreadContext &);
+        public:
+            constexpr explicit KThreadContext() : callee_saved(), lr(), sp(), cpacr(), fpcr(), fpsr(), fpu_registers(), locked() { /* ... */ }
+
+            Result Initialize(KVirtualAddress u_pc, KVirtualAddress k_sp, KVirtualAddress u_sp, uintptr_t arg, bool is_user, bool is_64_bit, bool is_main);
+            Result Finalize();
+
+            void SetArguments(uintptr_t arg0, uintptr_t arg1);
+
+            static void FpuContextSwitchHandler(KThread *thread);
+
+            /* TODO: More methods (especially FPU management) */
+    };
+
+}

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_userspace_memory_access.hpp

@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+
+namespace ams::kern::arch::arm64 {
+
+    void UserspaceAccessFunctionAreaBegin();
+
+    class UserspaceAccess {
+        public:
+            static bool CopyMemoryFromUser(void *dst, const void *src, size_t size);
+            static bool CopyMemoryFromUserAligned32Bit(void *dst, const void *src, size_t size);
+            static bool CopyMemoryFromUserAligned64Bit(void *dst, const void *src, size_t size);
+            static bool CopyMemoryFromUserSize32Bit(void *dst, const void *src);
+            static s32  CopyStringFromUser(void *dst, const void *src, size_t size);
+
+            static bool CopyMemoryToUser(void *dst, const void *src, size_t size);
+            static bool CopyMemoryToUserAligned32Bit(void *dst, const void *src, size_t size);
+            static bool CopyMemoryToUserAligned64Bit(void *dst, const void *src, size_t size);
+            static bool CopyMemoryToUserSize32Bit(void *dst, const void *src);
+            static s32  CopyStringToUser(void *dst, const void *src, size_t size);
+
+            static bool ClearMemory(void *dst, size_t size);
+            static bool ClearMemoryAligned32Bit(void *dst, size_t size);
+            static bool ClearMemoryAligned64Bit(void *dst, size_t size);
+            static bool ClearMemorySize32Bit(void *dst);
+
+            static bool StoreDataCache(uintptr_t start, uintptr_t end);
+            static bool FlushDataCache(uintptr_t start, uintptr_t end);
+            static bool InvalidateDataCache(uintptr_t start, uintptr_t end);
+            static bool InvalidateInstructionCache(uintptr_t start, uintptr_t end);
+
+            static bool ReadIoMemory32Bit(void *dst, const void *src, size_t size);
+            static bool ReadIoMemory16Bit(void *dst, const void *src, size_t size);
+            static bool ReadIoMemory8Bit(void *dst, const void *src, size_t size);
+            static bool WriteIoMemory32Bit(void *dst, const void *src, size_t size);
+            static bool WriteIoMemory16Bit(void *dst, const void *src, size_t size);
+            static bool WriteIoMemory8Bit(void *dst, const void *src, size_t size);
+    };
+
+
+    void UserspaceAccessFunctionAreaEnd();
+
+}

libraries/libmesosphere/include/mesosphere/board/nintendo/nx/kern_k_device_page_table.hpp

@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_k_page_group.hpp>
+#include <mesosphere/kern_k_memory_manager.hpp>
+#include <mesosphere/kern_select_page_table.hpp>
+
+namespace ams::kern::board::nintendo::nx {
+
+    using KDeviceVirtualAddress = u64;
+
+    class KDevicePageTable {
+        private:
+            static constexpr size_t TableCount = 4;
+        private:
+            KVirtualAddress tables[TableCount];
+            u8 table_asids[TableCount];
+            u64 attached_device;
+            u32 attached_value;
+            u32 detached_value;
+            u32 hs_attached_value;
+            u32 hs_detached_value;
+        private:
+            static ALWAYS_INLINE KVirtualAddress GetHeapVirtualAddress(KPhysicalAddress addr) {
+                return KPageTable::GetHeapVirtualAddress(addr);
+            }
+
+            static ALWAYS_INLINE KPhysicalAddress GetHeapPhysicalAddress(KVirtualAddress addr) {
+                return KPageTable::GetHeapPhysicalAddress(addr);
+            }
+
+            static ALWAYS_INLINE KVirtualAddress GetPageTableVirtualAddress(KPhysicalAddress addr) {
+                return KPageTable::GetPageTableVirtualAddress(addr);
+            }
+
+            static ALWAYS_INLINE KPhysicalAddress GetPageTablePhysicalAddress(KVirtualAddress addr) {
+                return KPageTable::GetPageTablePhysicalAddress(addr);
+            }
+        public:
+            constexpr KDevicePageTable() : tables(), table_asids(), attached_device(), attached_value(), detached_value(), hs_attached_value(), hs_detached_value() { /* ... */ }
+
+            static void Initialize();
+    };
+
+}

libraries/libmesosphere/include/mesosphere/board/nintendo/nx/kern_k_system_control.hpp

@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+
+namespace ams::kern::board::nintendo::nx {
+
+    class KSystemControl {
+        public:
+            class Init {
+                public:
+                    /* Initialization. */
+                    static size_t GetIntendedMemorySize();
+                    static KPhysicalAddress GetKernelPhysicalBaseAddress(uintptr_t base_address);
+                    static bool ShouldIncreaseThreadResourceLimit();
+                    static void CpuOn(u64 core_id, uintptr_t entrypoint, uintptr_t arg);
+                    static size_t GetApplicationPoolSize();
+                    static size_t GetAppletPoolSize();
+                    static size_t GetMinimumNonSecureSystemPoolSize();
+
+                    /* Randomness. */
+                    static void GenerateRandomBytes(void *dst, size_t size);
+                    static u64  GenerateRandomRange(u64 min, u64 max);
+            };
+        public:
+            /* Initialization. */
+            static NOINLINE void InitializePhase1();
+            static NOINLINE void InitializePhase2();
+            static NOINLINE u32 GetInitialProcessBinaryPool();
+
+            /* Randomness. */
+            static void GenerateRandomBytes(void *dst, size_t size);
+            static u64  GenerateRandomRange(u64 min, u64 max);
+
+            /* Privileged Access. */
+            static void ReadWriteRegisterPrivileged(u32 *out, ams::svc::PhysicalAddress address, u32 mask, u32 value);
+            static void ReadWriteRegister(u32 *out, ams::svc::PhysicalAddress address, u32 mask, u32 value);
+
+            static ALWAYS_INLINE u32 ReadRegisterPrivileged(ams::svc::PhysicalAddress address) {
+                u32 v;
+                ReadWriteRegisterPrivileged(std::addressof(v), address, 0x00000000u, 0);
+                return v;
+            }
+
+            static ALWAYS_INLINE void WriteRegisterPrivileged(ams::svc::PhysicalAddress address, u32 value) {
+                u32 v;
+                ReadWriteRegisterPrivileged(std::addressof(v), address, 0xFFFFFFFFu, value);
+            }
+
+            /* Power management. */
+            static void SleepSystem();
+            static NORETURN void StopSystem();
+
+            /* User access. */
+            static void CallSecureMonitorFromUser(ams::svc::lp64::SecureMonitorArguments *args);
+    };
+
+}

libraries/libmesosphere/include/mesosphere/init/kern_init_arguments_select.hpp

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_k_typed_address.hpp>
+
+#ifdef ATMOSPHERE_ARCH_ARM64
+    #include <mesosphere/arch/arm64/init/kern_k_init_arguments.hpp>
+#else
+    #error "Unknown architecture for KInitArguments"
+#endif
+
+namespace ams::kern::init {
+
+    KPhysicalAddress GetInitArgumentsAddress(s32 core_id);
+    void SetInitArguments(s32 core_id, KPhysicalAddress address, uintptr_t arg);
+    void StoreInitArguments();
+
+}

libraries/libmesosphere/include/mesosphere/init/kern_init_elf.hpp

@@ -17,23 +17,21 @@
 #include <vapours.hpp>
 
 #ifdef ATMOSPHERE_ARCH_ARM64
-    #include "kern_init_elf64.hpp"
+    #include <mesosphere/init/kern_init_elf64.hpp>
+
+    namespace ams::kern::init::Elf {
+        using namespace ams::kern::init::Elf::Elf64;
+
+        enum RelocationType {
+            R_ARCHITECTURE_RELATIVE = 0x403, /* Real name R_AARCH64_RELATIVE */
+        };
+    }
 #else
     #error "Unknown Architecture"
 #endif
 
 namespace ams::kern::init::Elf {
 
-    #ifdef ATMOSPHERE_ARCH_ARM64
-        using namespace ams::kern::init::Elf::Elf64;
-
-        enum RelocationType {
-            R_ARCHITECTURE_RELATIVE = 0x403, /* Real name R_AARCH64_RELATIVE */
-        };
-    #else
-        #error "Unknown Architecture"
-    #endif
-
     /* API to apply relocations or call init array. */
     void ApplyRelocations(uintptr_t base_address, const Dyn *dynamic);
     void CallInitArrayFuncs(uintptr_t init_array_start, uintptr_t init_array_end);

libraries/libmesosphere/include/mesosphere/init/kern_init_layout.hpp

@@ -27,7 +27,7 @@ namespace ams::kern::init {
         u32 rw_end_offset;
         u32 bss_offset;
         u32 bss_end_offset;
-        u32 ini_end_offset;
+        u32 ini_load_offset;
         u32 dynamic_offset;
         u32 init_array_offset;
         u32 init_array_end_offset;

libraries/libmesosphere/include/mesosphere/init/kern_init_page_table_select.hpp

@@ -14,9 +14,16 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 #pragma once
+#include <mesosphere/kern_common.hpp>
 
 #ifdef ATMOSPHERE_ARCH_ARM64
-    #include "../arch/arm64/init/kern_k_init_page_table.hpp"
+    #include <mesosphere/arch/arm64/init/kern_k_init_page_table.hpp>
+
+    namespace ams::kern::init {
+        using ams::kern::arch::arm64::PageTableEntry;
+        using ams::kern::arch::arm64::init::KInitialPageTable;
+        using ams::kern::arch::arm64::init::KInitialPageAllocator;
+    }
 #else
     #error "Unknown architecture for KInitialPageTable"
 #endif

libraries/libmesosphere/include/mesosphere/init/kern_init_slab_setup.hpp

@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <vapours.hpp>
+#include <mesosphere/kern_k_slab_heap.hpp>
+
+namespace ams::kern::init {
+
+    struct KSlabResourceCounts {
+        size_t num_KProcess;
+        size_t num_KThread;
+        size_t num_KEvent;
+        size_t num_KInterruptEvent;
+        size_t num_KPort;
+        size_t num_KSharedMemory;
+        size_t num_KTransferMemory;
+        size_t num_KCodeMemory;
+        size_t num_KDeviceAddressSpace;
+        size_t num_KSession;
+        size_t num_KLightSession;
+        size_t num_KObjectName;
+        size_t num_KResourceLimit;
+        size_t num_KDebug;
+    };
+
+    NOINLINE void InitializeSlabResourceCounts();
+    const KSlabResourceCounts &GetSlabResourceCounts();
+
+    size_t CalculateTotalSlabHeapSize();
+    NOINLINE void InitializeKPageBufferSlabHeap();
+    NOINLINE void InitializeSlabHeaps();
+
+}

libraries/libmesosphere/include/mesosphere/kern_common.hpp

@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <vapours.hpp>
+
+namespace ams::kern {
+
+    constexpr size_t PageSize = 4_KB;
+
+    enum TargetFirmware : u32 {
+        TargetFirmware_1_0_0  = ATMOSPHERE_TARGET_FIRMWARE_100,
+        TargetFirmware_2_0_0  = ATMOSPHERE_TARGET_FIRMWARE_200,
+        TargetFirmware_3_0_0  = ATMOSPHERE_TARGET_FIRMWARE_300,
+        TargetFirmware_4_0_0  = ATMOSPHERE_TARGET_FIRMWARE_400,
+        TargetFirmware_5_0_0  = ATMOSPHERE_TARGET_FIRMWARE_500,
+        TargetFirmware_6_0_0  = ATMOSPHERE_TARGET_FIRMWARE_600,
+        TargetFirmware_6_2_0  = ATMOSPHERE_TARGET_FIRMWARE_620,
+        TargetFirmware_7_0_0  = ATMOSPHERE_TARGET_FIRMWARE_700,
+        TargetFirmware_8_0_0  = ATMOSPHERE_TARGET_FIRMWARE_800,
+        TargetFirmware_8_1_0  = ATMOSPHERE_TARGET_FIRMWARE_810,
+        TargetFirmware_9_0_0  = ATMOSPHERE_TARGET_FIRMWARE_900,
+        TargetFirmware_9_1_0  = ATMOSPHERE_TARGET_FIRMWARE_910,
+        TargetFirmware_10_0_0 = ATMOSPHERE_TARGET_FIRMWARE_1000,
+    };
+
+    TargetFirmware GetTargetFirmware();
+
+}
+
+#if 1 || defined(AMS_BUILD_FOR_AUDITING)
+#define MESOSPHERE_BUILD_FOR_AUDITING
+#endif
+
+#if defined(MESOSPHERE_BUILD_FOR_AUDITING) || defined(AMS_BUILD_FOR_DEBUGGING)
+#define MESOSPHERE_BUILD_FOR_DEBUGGING
+#endif
+
+#ifdef  MESOSPHERE_BUILD_FOR_DEBUGGING
+#define MESOSPHERE_ENABLE_ASSERTIONS
+#define MESOSPHERE_ENABLE_DEBUG_PRINT
+#endif
+
+#include <mesosphere/svc/kern_svc_results.hpp>

libraries/libmesosphere/include/mesosphere/kern_debug_log.hpp

@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+
+namespace ams::kern {
+
+    class KDebugLog {
+        private:
+            static NOINLINE void VSNPrintf(char *dst, const size_t dst_size, const char *format, ::std::va_list vl);
+        public:
+            static NOINLINE void Initialize();
+
+            static NOINLINE void Printf(const char *format, ...) __attribute__((format(printf, 1, 2)));
+            static NOINLINE void VPrintf(const char *format, ::std::va_list vl);
+    };
+
+}
+
+#ifndef MESOSPHERE_DEBUG_LOG_SELECTED
+
+    #ifdef ATMOSPHERE_BOARD_NINTENDO_NX
+        #define MESOSPHERE_DEBUG_LOG_USE_UART_C
+    #else
+        #error "Unknown board for Default Debug Log Source"
+    #endif
+
+    #define MESOSPHERE_DEBUG_LOG_SELECTED
+
+#endif
+
+#define MESOSPHERE_RELEASE_LOG(fmt, ...) ::ams::kern::KDebugLog::Printf((fmt), ## __VA_ARGS__)
+#define MESOSPHERE_RELEASE_VLOG(fmt, vl) ::ams::kern::KDebugLog::VPrintf((fmt), (vl))
+
+#ifdef MESOSPHERE_ENABLE_DEBUG_PRINT
+#define MESOSPHERE_LOG(fmt, ...) MESOSPHERE_RELEASE_LOG((fmt), ## __VA_ARGS__)
+#define MESOSPHERE_VLOG(fmt, vl) MESOSPHERE_RELEASE_VLOG((fmt), (vl))
+#else
+#define MESOSPHERE_LOG(fmt, ...) do { MESOSPHERE_UNUSED(fmt); MESOSPHERE_UNUSED(__VA_ARGS__); } while (0)
+#define MESOSPHERE_VLOG(fmt, vl) do { MESOSPHERE_UNUSED(fmt); MESOSPHERE_UNUSED(vl); } while (0)
+#endif

libraries/libmesosphere/include/mesosphere/kern_initial_process.hpp

@@ -14,8 +14,8 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 #pragma once
-#include <vapours.hpp>
-#include "kern_panic.hpp"
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_k_initial_process_reader.hpp>
 
 namespace ams::kern {
 
@@ -29,4 +29,10 @@ namespace ams::kern {
         u32 reserved;
     };
 
+    NOINLINE void CopyInitialProcessBinaryToKernelMemory();
+    NOINLINE void CreateAndRunInitialProcesses();
+
+    u64 GetInitialProcessIdMin();
+    u64 GetInitialProcessIdMax();
+
 }

libraries/libmesosphere/include/mesosphere/kern_k_address_arbiter.hpp

@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_k_condition_variable.hpp>
+
+namespace ams::kern {
+
+    class KAddressArbiter {
+        public:
+            using ThreadTree = KConditionVariable::ThreadTree;
+        private:
+            ThreadTree tree;
+        public:
+            constexpr KAddressArbiter() : tree() { /* ... */ }
+
+            Result SignalToAddress(uintptr_t addr, ams::svc::SignalType type, s32 value, s32 count) {
+                switch (type) {
+                    case ams::svc::SignalType_Signal:
+                        return this->Signal(addr, count);
+                    case ams::svc::SignalType_SignalAndIncrementIfEqual:
+                        return this->SignalAndIncrementIfEqual(addr, value, count);
+                    case ams::svc::SignalType_SignalAndModifyByWaitingCountIfEqual:
+                        return this->SignalAndModifyByWaitingCountIfEqual(addr, value, count);
+                    MESOSPHERE_UNREACHABLE_DEFAULT_CASE();
+                }
+            }
+
+            Result WaitForAddress(uintptr_t addr, ams::svc::ArbitrationType type, s32 value, s64 timeout) {
+                switch (type) {
+                    case ams::svc::ArbitrationType_WaitIfLessThan:
+                        return this->WaitIfLessThan(addr, value, false, timeout);
+                    case ams::svc::ArbitrationType_DecrementAndWaitIfLessThan:
+                        return this->WaitIfLessThan(addr, value, true, timeout);
+                    case ams::svc::ArbitrationType_WaitIfEqual:
+                        return this->WaitIfEqual(addr, value, timeout);
+                    MESOSPHERE_UNREACHABLE_DEFAULT_CASE();
+                }
+            }
+        private:
+            Result Signal(uintptr_t addr, s32 count);
+            Result SignalAndIncrementIfEqual(uintptr_t addr, s32 value, s32 count);
+            Result SignalAndModifyByWaitingCountIfEqual(uintptr_t addr, s32 value, s32 count);
+            Result WaitIfLessThan(uintptr_t addr, s32 value, bool decrement, s64 timeout);
+            Result WaitIfEqual(uintptr_t addr, s32 value, s64 timeout);
+    };
+
+}

libraries/libmesosphere/include/mesosphere/kern_k_address_space_info.hpp

@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+
+namespace ams::kern {
+
+
+
+    struct KAddressSpaceInfo {
+        public:
+            enum Type {
+                Type_32Bit      = 0,
+                Type_Small64Bit = 1,
+                Type_Large64Bit = 2,
+                Type_Heap       = 3,
+                Type_Stack      = 4,
+                Type_Alias      = 5,
+
+                Type_Count,
+            };
+        private:
+            size_t bit_width;
+            size_t address;
+            size_t size;
+            Type type;
+        public:
+            static uintptr_t GetAddressSpaceStart(size_t width, Type type);
+            static size_t GetAddressSpaceSize(size_t width, Type type);
+
+            constexpr KAddressSpaceInfo(size_t bw, size_t a, size_t s, Type t) : bit_width(bw), address(a), size(s), type(t) { /* ... */ }
+
+            constexpr size_t GetWidth() const { return this->bit_width; }
+            constexpr size_t GetAddress() const { return this->address; }
+            constexpr size_t GetSize() const { return this->size; }
+            constexpr Type GetType() const { return this->type; }
+    };
+
+}

libraries/libmesosphere/include/mesosphere/kern_k_affinity_mask.hpp

@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_select_cpu.hpp>
+
+namespace ams::kern {
+
+    class KAffinityMask {
+        private:
+            static constexpr u64 AllowedAffinityMask = (1ul << cpu::NumCores) - 1;
+        private:
+            u64 mask;
+        private:
+            static constexpr ALWAYS_INLINE u64 GetCoreBit(s32 core) {
+                MESOSPHERE_ASSERT(0 <= core && core < static_cast<s32>(cpu::NumCores));
+                return (1ul << core);
+            }
+        public:
+            constexpr ALWAYS_INLINE KAffinityMask() : mask(0) { MESOSPHERE_ASSERT_THIS(); }
+
+            constexpr ALWAYS_INLINE u64 GetAffinityMask() const { return this->mask; }
+
+            constexpr ALWAYS_INLINE void SetAffinityMask(u64 new_mask) {
+                MESOSPHERE_ASSERT((new_mask & ~AllowedAffinityMask) == 0);
+                this->mask = new_mask;
+            }
+
+            constexpr ALWAYS_INLINE bool GetAffinity(s32 core) const {
+                return this->mask & GetCoreBit(core);
+            }
+
+            constexpr ALWAYS_INLINE void SetAffinity(s32 core, bool set) {
+                MESOSPHERE_ASSERT(0 <= core && core < static_cast<s32>(cpu::NumCores));
+
+                if (set) {
+                    this->mask |= GetCoreBit(core);
+                } else {
+                    this->mask &= ~GetCoreBit(core);
+                }
+            }
+
+            constexpr ALWAYS_INLINE void SetAll() {
+                this->mask = AllowedAffinityMask;
+            }
+    };
+
+}