add entire atmosphere source code and also add ram timing fixes

Source/Atmosphere/libraries/libmesosphere/include/mesosphere/arch/arm/kern_generic_interrupt_controller.hpp

@@ -0,0 +1,282 @@
+/*
+ * Copyright (c) Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <vapours.hpp>
+#include <mesosphere/kern_select_cpu.hpp>
+#include <mesosphere/kern_k_typed_address.hpp>
+
+namespace ams::kern::arch::arm {
+
+    struct GicDistributor {
+        u32 ctlr;
+        u32 typer;
+        u32 iidr;
+        u32 reserved_0x0c;
+        u32 statusr;
+        u32 reserved_0x14[3];
+        u32 impldef_0x20[8];
+        u32 setspi_nsr;
+        u32 reserved_0x44;
+        u32 clrspi_nsr;
+        u32 reserved_0x4c;
+        u32 setspi_sr;
+        u32 reserved_0x54;
+        u32 clrspi_sr;
+        u32 reserved_0x5c[9];
+        u32 igroupr[32];
+        u32 isenabler[32];
+        u32 icenabler[32];
+        u32 ispendr[32];
+        u32 icpendr[32];
+        u32 isactiver[32];
+        u32 icactiver[32];
+        union {
+            u8  bytes[1020];
+            u32 words[255];
+        } ipriorityr;
+        u32 _0x7fc;
+        union {
+            u8  bytes[1020];
+            u32 words[255];
+        } itargetsr;
+        u32 _0xbfc;
+        u32 icfgr[64];
+        u32 igrpmodr[32];
+        u32 _0xd80[32];
+        u32 nsacr[64];
+        u32 sgir;
+        u32 _0xf04[3];
+        u32 cpendsgir[4];
+        u32 spendsgir[4];
+        u32 reserved_0xf30[52];
+
+        static constexpr size_t SgirCpuTargetListShift = 16;
+
+        enum SgirTargetListFilter : u32 {
+            SgirTargetListFilter_CpuTargetList = (0 << 24),
+            SgirTargetListFilter_Others        = (1 << 24),
+            SgirTargetListFilter_Self          = (2 << 24),
+            SgirTargetListFilter_Reserved      = (3 << 24),
+        };
+    };
+    static_assert(util::is_pod<GicDistributor>::value);
+    static_assert(sizeof(GicDistributor) == 0x1000);
+
+    struct GicCpuInterface {
+        u32 ctlr;
+        u32 pmr;
+        u32 bpr;
+        u32 iar;
+        u32 eoir;
+        u32 rpr;
+        u32 hppir;
+        u32 abpr;
+        u32 aiar;
+        u32 aeoir;
+        u32 ahppir;
+        u32 statusr;
+        u32 reserved_30[4];
+        u32 impldef_40[36];
+        u32 apr[4];
+        u32 nsapr[4];
+        u32 reserved_f0[3];
+        u32 iidr;
+        u32 reserved_100[960];
+        u32 dir;
+        u32 _0x1004[1023];
+    };
+    static_assert(util::is_pod<GicCpuInterface>::value);
+    static_assert(sizeof(GicCpuInterface) == 0x2000);
+
+    struct KInterruptController {
+        NON_COPYABLE(KInterruptController);
+        NON_MOVEABLE(KInterruptController);
+        public:
+            static constexpr s32 NumSoftwareInterrupts = 16;
+            static constexpr s32 NumLocalInterrupts = NumSoftwareInterrupts + 16;
+            static constexpr s32 NumGlobalInterrupts = 988;
+            static constexpr s32 NumInterrupts = NumLocalInterrupts + NumGlobalInterrupts;
+            static constexpr s32 NumPriorityLevels = 4;
+        public:
+            struct LocalState {
+                u32 isenabler[NumLocalInterrupts / 32];
+                u32 ipriorityr[NumLocalInterrupts / 4];
+                u32 itargetsr[NumLocalInterrupts / 4];
+                u32 icfgr[NumLocalInterrupts / 16];
+                u32 spendsgir[4];
+            };
+            static_assert(sizeof(LocalState{}.spendsgir) == sizeof(GicDistributor{}.spendsgir));
+
+            struct GlobalState {
+                u32 isenabler[NumGlobalInterrupts / 32];
+                u32 ipriorityr[NumGlobalInterrupts / 4];
+                u32 itargetsr[NumGlobalInterrupts / 4];
+                u32 icfgr[NumGlobalInterrupts / 16];
+            };
+
+            enum PriorityLevel : u8 {
+                PriorityLevel_High  = 0,
+                PriorityLevel_Low   = NumPriorityLevels - 1,
+
+                PriorityLevel_Timer     = 1,
+                PriorityLevel_Scheduler = 2,
+            };
+        private:
+            static constinit inline u32 s_mask[cpu::NumCores];
+        private:
+            volatile GicDistributor  *m_gicd;
+            volatile GicCpuInterface *m_gicc;
+        public:
+            constexpr KInterruptController() : m_gicd(nullptr), m_gicc(nullptr) { /* ... */ }
+
+            void Initialize(s32 core_id);
+            void Finalize(s32 core_id);
+
+            void SaveCoreLocal(LocalState *state) const;
+            void SaveGlobal(GlobalState *state) const;
+            void RestoreCoreLocal(const LocalState *state) const;
+            void RestoreGlobal(const GlobalState *state) const;
+        public:
+            u32 GetIrq() const {
+                return m_gicc->iar;
+            }
+
+            static constexpr s32 ConvertRawIrq(u32 irq) {
+                return (irq == 0x3FF) ? -1 : (irq & 0x3FF);
+            }
+
+            void Enable(s32 irq) const {
+                m_gicd->isenabler[irq / BITSIZEOF(u32)] = (1u << (irq % BITSIZEOF(u32)));
+            }
+
+            void Disable(s32 irq) const {
+                m_gicd->icenabler[irq / BITSIZEOF(u32)] = (1u << (irq % BITSIZEOF(u32)));
+            }
+
+            void Clear(s32 irq) const {
+                m_gicd->icpendr[irq / BITSIZEOF(u32)] = (1u << (irq % BITSIZEOF(u32)));
+            }
+
+            void SetTarget(s32 irq, s32 core_id) const {
+                m_gicd->itargetsr.bytes[irq] = m_gicd->itargetsr.bytes[irq] | GetGicMask(core_id);
+            }
+
+            void ClearTarget(s32 irq, s32 core_id) const {
+                m_gicd->itargetsr.bytes[irq] = m_gicd->itargetsr.bytes[irq] & ~GetGicMask(core_id);
+            }
+
+            void SetPriorityLevel(s32 irq, s32 level) const {
+                MESOSPHERE_ASSERT(PriorityLevel_High <= level && level <= PriorityLevel_Low);
+                m_gicd->ipriorityr.bytes[irq] = ToGicPriorityValue(level);
+            }
+
+            s32 GetPriorityLevel(s32 irq) const {
+                return FromGicPriorityValue(m_gicd->ipriorityr.bytes[irq]);
+            }
+
+            void SetPriorityLevel(s32 level) const {
+                MESOSPHERE_ASSERT(PriorityLevel_High <= level && level <= PriorityLevel_Low);
+                m_gicc->pmr = ToGicPriorityValue(level);
+            }
+
+            void SetEdge(s32 irq) const {
+                u32 cfg = m_gicd->icfgr[irq / (BITSIZEOF(u32) / 2)];
+                cfg &= ~(0x3 << (2 * (irq % (BITSIZEOF(u32) / 2))));
+                cfg |=  (0x2 << (2 * (irq % (BITSIZEOF(u32) / 2))));
+                m_gicd->icfgr[irq / (BITSIZEOF(u32) / 2)] = cfg;
+            }
+
+            void SetLevel(s32 irq) const {
+                u32 cfg = m_gicd->icfgr[irq / (BITSIZEOF(u32) / 2)];
+                cfg &= ~(0x3 << (2 * (irq % (BITSIZEOF(u32) / 2))));
+                cfg |=  (0x0 << (2 * (irq % (BITSIZEOF(u32) / 2))));
+                m_gicd->icfgr[irq / (BITSIZEOF(u32) / 2)] = cfg;
+            }
+
+            void SendInterProcessorInterrupt(s32 irq, u64 core_mask) {
+                MESOSPHERE_ASSERT(IsSoftware(irq));
+                m_gicd->sgir = GetCpuTargetListMask(irq, core_mask);
+            }
+
+            void SendInterProcessorInterrupt(s32 irq) {
+                MESOSPHERE_ASSERT(IsSoftware(irq));
+                m_gicd->sgir = GicDistributor::SgirTargetListFilter_Others | irq;
+            }
+
+            void EndOfInterrupt(u32 irq) const {
+                m_gicc->eoir = irq;
+            }
+
+            bool IsInterruptDefined(s32 irq) const {
+                const s32 num_interrupts = std::min(32 + 32 * (m_gicd->typer & 0x1F), static_cast<u32>(NumInterrupts));
+                return (0 <= irq && irq < num_interrupts);
+            }
+        public:
+            static constexpr ALWAYS_INLINE bool IsSoftware(s32 id) {
+                MESOSPHERE_ASSERT(0 <= id && id < NumInterrupts);
+                return id < NumSoftwareInterrupts;
+            }
+
+            static constexpr ALWAYS_INLINE bool IsLocal(s32 id) {
+                MESOSPHERE_ASSERT(0 <= id && id < NumInterrupts);
+                return id < NumLocalInterrupts;
+            }
+
+            static constexpr ALWAYS_INLINE bool IsGlobal(s32 id) {
+                MESOSPHERE_ASSERT(0 <= id && id < NumInterrupts);
+                return NumLocalInterrupts <= id;
+            }
+
+            static constexpr size_t GetGlobalInterruptIndex(s32 id) {
+                MESOSPHERE_ASSERT(IsGlobal(id));
+                return id - NumLocalInterrupts;
+            }
+
+            static constexpr size_t GetLocalInterruptIndex(s32 id) {
+                MESOSPHERE_ASSERT(IsLocal(id));
+                return id;
+            }
+        private:
+            static constexpr size_t PriorityShift = BITSIZEOF(u8) - util::CountTrailingZeros(NumPriorityLevels);
+            static_assert(PriorityShift < BITSIZEOF(u8));
+            static_assert(util::IsPowerOfTwo(NumPriorityLevels));
+
+            static constexpr ALWAYS_INLINE u8 ToGicPriorityValue(s32 level) {
+                return (level << PriorityShift) | ((1 << PriorityShift) - 1);
+            }
+
+            static constexpr ALWAYS_INLINE s32 FromGicPriorityValue(u8 priority) {
+                return (priority >> PriorityShift) & (NumPriorityLevels - 1);
+            }
+
+            static constexpr ALWAYS_INLINE s32 GetCpuTargetListMask(s32 irq, u64 core_mask) {
+                MESOSPHERE_ASSERT(IsSoftware(irq));
+                MESOSPHERE_ASSERT(core_mask < (1ul << cpu::NumCores));
+                return GicDistributor::SgirTargetListFilter_CpuTargetList | irq | (static_cast<u16>(core_mask) << GicDistributor::SgirCpuTargetListShift);
+            }
+
+            static ALWAYS_INLINE s32 GetGicMask(s32 core_id) {
+                return s_mask[core_id];
+            }
+
+            ALWAYS_INLINE void SetGicMask(s32 core_id) const {
+                s_mask[core_id] = m_gicd->itargetsr.bytes[0];
+            }
+
+            NOINLINE void SetupInterruptLines(s32 core_id) const;
+    };
+
+}

Source/Atmosphere/libraries/libmesosphere/include/mesosphere/arch/arm/kern_k_interrupt_controller.hpp

@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <vapours.hpp>
+#include <mesosphere/kern_select_cpu.hpp>
+#include <mesosphere/kern_k_typed_address.hpp>
+
+#if 1
+
+    #include <mesosphere/arch/arm/kern_generic_interrupt_controller.hpp>
+
+#else
+
+    #error "Unknown board for KInterruptController"
+
+#endif

Source/Atmosphere/libraries/libmesosphere/include/mesosphere/arch/arm/kern_k_memory_region_device_types.inc

@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/* All architectures must define NumArchitectureDeviceRegions. */
+constexpr inline const auto NumArchitectureDeviceRegions = 3;
+
+constexpr inline const auto KMemoryRegionType_Uart                  = KMemoryRegionType_ArchDeviceBase.DeriveSparse(0, NumArchitectureDeviceRegions, 0);
+constexpr inline const auto KMemoryRegionType_InterruptCpuInterface = KMemoryRegionType_ArchDeviceBase.DeriveSparse(0, NumArchitectureDeviceRegions, 1).SetAttribute(KMemoryRegionAttr_NoUserMap);
+constexpr inline const auto KMemoryRegionType_InterruptDistributor  = KMemoryRegionType_ArchDeviceBase.DeriveSparse(0, NumArchitectureDeviceRegions, 2).SetAttribute(KMemoryRegionAttr_NoUserMap);
+static_assert(KMemoryRegionType_Uart                 .GetValue() == (0x1D));
+static_assert(KMemoryRegionType_InterruptCpuInterface.GetValue() == (0x2D | KMemoryRegionAttr_NoUserMap));
+static_assert(KMemoryRegionType_InterruptDistributor .GetValue() == (0x4D | KMemoryRegionAttr_NoUserMap));

Source/Atmosphere/libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_arguments.hpp

@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_select_assembly_offsets.h>
+
+namespace ams::kern::init {
+
+    struct alignas(util::CeilingPowerOfTwo(INIT_ARGUMENTS_SIZE)) KInitArguments {
+        u64 cpuactlr;
+        u64 cpuectlr;
+        u64 sp;
+        u64 entrypoint;
+        u64 argument;
+    };
+    static_assert(alignof(KInitArguments) == util::CeilingPowerOfTwo(INIT_ARGUMENTS_SIZE));
+    static_assert(sizeof(KInitArguments) == std::max(INIT_ARGUMENTS_SIZE, util::CeilingPowerOfTwo(INIT_ARGUMENTS_SIZE)));
+
+    static_assert(AMS_OFFSETOF(KInitArguments, cpuactlr)        == INIT_ARGUMENTS_CPUACTLR);
+    static_assert(AMS_OFFSETOF(KInitArguments, cpuectlr)        == INIT_ARGUMENTS_CPUECTLR);
+    static_assert(AMS_OFFSETOF(KInitArguments, sp)              == INIT_ARGUMENTS_SP);
+    static_assert(AMS_OFFSETOF(KInitArguments, entrypoint)      == INIT_ARGUMENTS_ENTRYPOINT);
+    static_assert(AMS_OFFSETOF(KInitArguments, argument)        == INIT_ARGUMENTS_ARGUMENT);
+
+}

Source/Atmosphere/libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp

@@ -0,0 +1,954 @@
+/*
+ * Copyright (c) Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <vapours.hpp>
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_k_typed_address.hpp>
+#include <mesosphere/kern_select_cpu.hpp>
+#include <mesosphere/arch/arm64/kern_k_page_table_entry.hpp>
+#include <mesosphere/kern_select_system_control.hpp>
+
+namespace ams::kern::arch::arm64::init {
+
+    /* NOTE: Nintendo uses virtual functions, rather than a concept + template. */
+    template<typename T>
+    concept IsInitialPageAllocator = requires (T &t, KPhysicalAddress phys_addr, size_t size) {
+        { t.Allocate(size) }        -> std::same_as<KPhysicalAddress>;
+        { t.Free(phys_addr, size) } -> std::same_as<void>;
+    };
+
+    class KInitialPageTable {
+        private:
+            KPhysicalAddress m_l1_tables[2];
+            u32 m_num_entries[2];
+        public:
+            template<IsInitialPageAllocator PageAllocator>
+            KInitialPageTable(KVirtualAddress start_address, KVirtualAddress end_address, PageAllocator &allocator) {
+                /* Set tables. */
+                m_l1_tables[0] = AllocateNewPageTable(allocator, 0);
+                m_l1_tables[1] = AllocateNewPageTable(allocator, 0);
+
+                /* Set counts. */
+                m_num_entries[0] = MaxPageTableEntries;
+                m_num_entries[1] = ((end_address / L1BlockSize) & (MaxPageTableEntries - 1)) - ((start_address / L1BlockSize) & (MaxPageTableEntries - 1)) + 1;
+            }
+
+            KInitialPageTable() {
+                /* Set tables. */
+                m_l1_tables[0] = util::AlignDown(cpu::GetTtbr0El1(), PageSize);
+                m_l1_tables[1] = util::AlignDown(cpu::GetTtbr1El1(), PageSize);
+
+                /* Set counts. */
+                cpu::TranslationControlRegisterAccessor tcr;
+                m_num_entries[0] = tcr.GetT0Size() / L1BlockSize;
+                m_num_entries[1] = tcr.GetT1Size() / L1BlockSize;
+
+                /* Check counts. */
+                MESOSPHERE_INIT_ABORT_UNLESS(0 < m_num_entries[0] && m_num_entries[0] <= MaxPageTableEntries);
+                MESOSPHERE_INIT_ABORT_UNLESS(0 < m_num_entries[1] && m_num_entries[1] <= MaxPageTableEntries);
+            }
+
+            constexpr ALWAYS_INLINE uintptr_t GetTtbr0L1TableAddress() const {
+                return GetInteger(m_l1_tables[0]);
+            }
+
+            constexpr ALWAYS_INLINE uintptr_t GetTtbr1L1TableAddress() const {
+                return GetInteger(m_l1_tables[1]);
+            }
+        private:
+            constexpr ALWAYS_INLINE L1PageTableEntry *GetL1Entry(KVirtualAddress address, u64 phys_to_virt_offset = 0) const {
+                const size_t index = (GetInteger(address) >> (BITSIZEOF(address) - 1)) & 1;
+                L1PageTableEntry *l1_table = reinterpret_cast<L1PageTableEntry *>(GetInteger(m_l1_tables[index]) + phys_to_virt_offset);
+                return l1_table + ((GetInteger(address) / L1BlockSize) & (m_num_entries[index] - 1));
+            }
+
+            static constexpr ALWAYS_INLINE L2PageTableEntry *GetL2Entry(const L1PageTableEntry *entry, KVirtualAddress address, u64 phys_to_virt_offset = 0) {
+                L2PageTableEntry *l2_table = reinterpret_cast<L2PageTableEntry *>(GetInteger(entry->GetTable()) + phys_to_virt_offset);
+                return l2_table + ((GetInteger(address) / L2BlockSize) & (MaxPageTableEntries - 1));
+            }
+
+            static constexpr ALWAYS_INLINE L3PageTableEntry *GetL3Entry(const L2PageTableEntry *entry, KVirtualAddress address, u64 phys_to_virt_offset = 0) {
+                L3PageTableEntry *l3_table = reinterpret_cast<L3PageTableEntry *>(GetInteger(entry->GetTable()) + phys_to_virt_offset);
+                return l3_table + ((GetInteger(address) / L3BlockSize) & (MaxPageTableEntries - 1));
+            }
+
+            template<IsInitialPageAllocator PageAllocator>
+            static ALWAYS_INLINE KPhysicalAddress AllocateNewPageTable(PageAllocator &allocator, u64 phys_to_virt_offset) {
+                MESOSPHERE_UNUSED(phys_to_virt_offset);
+                return allocator.Allocate(PageSize);
+            }
+
+            static ALWAYS_INLINE void ClearNewPageTable(KPhysicalAddress address, u64 phys_to_virt_offset) {
+                /* Convert to a deferenceable address, and clear. */
+                volatile u64 *ptr = reinterpret_cast<volatile u64 *>(GetInteger(address) + phys_to_virt_offset);
+                for (size_t i = 0; i < PageSize / sizeof(u64); ++i) {
+                    ptr[i] = 0;
+                }
+            }
+        public:
+            static consteval size_t GetMaximumOverheadSize(size_t size) {
+                return (util::DivideUp(size, L1BlockSize) + util::DivideUp(size, L2BlockSize)) * PageSize;
+            }
+        private:
+            size_t NOINLINE GetBlockCount(KVirtualAddress virt_addr, size_t size, size_t block_size) {
+                const KVirtualAddress end_virt_addr = virt_addr + size;
+                size_t count = 0;
+                while (virt_addr < end_virt_addr) {
+                    L1PageTableEntry *l1_entry = this->GetL1Entry(virt_addr);
+
+                    /* If an L1 block is mapped or we're empty, advance by L1BlockSize. */
+                    if (l1_entry->IsMappedBlock() || l1_entry->IsMappedEmpty()) {
+                        MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), L1BlockSize));
+                        MESOSPHERE_INIT_ABORT_UNLESS(static_cast<size_t>(end_virt_addr - virt_addr) >= L1BlockSize);
+                        virt_addr += L1BlockSize;
+                        if (l1_entry->IsMappedBlock() && block_size == L1BlockSize) {
+                            count++;
+                        }
+                        continue;
+                    }
+
+                    /* Non empty and non-block must be table. */
+                    MESOSPHERE_INIT_ABORT_UNLESS(l1_entry->IsMappedTable());
+
+                    /* Table, so check if we're mapped in L2. */
+                    L2PageTableEntry *l2_entry = GetL2Entry(l1_entry, virt_addr);
+
+                    if (l2_entry->IsMappedBlock() || l2_entry->IsMappedEmpty()) {
+                        const size_t advance_size = (l2_entry->IsMappedBlock() && l2_entry->IsContiguous()) ? L2ContiguousBlockSize : L2BlockSize;
+                        MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), advance_size));
+                        MESOSPHERE_INIT_ABORT_UNLESS(static_cast<size_t>(end_virt_addr - virt_addr) >= advance_size);
+                        virt_addr += advance_size;
+                        if (l2_entry->IsMappedBlock() && block_size == advance_size) {
+                            count++;
+                        }
+                        continue;
+                    }
+
+                    /* Non empty and non-block must be table. */
+                    MESOSPHERE_INIT_ABORT_UNLESS(l2_entry->IsMappedTable());
+
+                    /* Table, so check if we're mapped in L3. */
+                    L3PageTableEntry *l3_entry = GetL3Entry(l2_entry, virt_addr);
+
+                    /* L3 must be block or empty. */
+                    MESOSPHERE_INIT_ABORT_UNLESS(l3_entry->IsMappedBlock() || l3_entry->IsMappedEmpty());
+
+                    const size_t advance_size = (l3_entry->IsMappedBlock() && l3_entry->IsContiguous()) ? L3ContiguousBlockSize : L3BlockSize;
+                    MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), advance_size));
+                    MESOSPHERE_INIT_ABORT_UNLESS(static_cast<size_t>(end_virt_addr - virt_addr) >= advance_size);
+                    virt_addr += advance_size;
+                    if (l3_entry->IsMappedBlock() && block_size == advance_size) {
+                        count++;
+                    }
+                }
+                return count;
+            }
+
+            KVirtualAddress NOINLINE GetBlockByIndex(KVirtualAddress virt_addr, size_t size, size_t block_size, size_t index) {
+                const KVirtualAddress end_virt_addr = virt_addr + size;
+                size_t count = 0;
+                while (virt_addr < end_virt_addr) {
+                    L1PageTableEntry *l1_entry = this->GetL1Entry(virt_addr);
+
+                    /* If an L1 block is mapped or we're empty, advance by L1BlockSize. */
+                    if (l1_entry->IsMappedBlock() || l1_entry->IsMappedEmpty()) {
+                        MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), L1BlockSize));
+                        MESOSPHERE_INIT_ABORT_UNLESS(static_cast<size_t>(end_virt_addr - virt_addr) >= L1BlockSize);
+                        if (l1_entry->IsMappedBlock() && block_size == L1BlockSize) {
+                            if ((count++) == index) {
+                                return virt_addr;
+                            }
+                        }
+                        virt_addr += L1BlockSize;
+                        continue;
+                    }
+
+                    /* Non empty and non-block must be table. */
+                    MESOSPHERE_INIT_ABORT_UNLESS(l1_entry->IsMappedTable());
+
+                    /* Table, so check if we're mapped in L2. */
+                    L2PageTableEntry *l2_entry = GetL2Entry(l1_entry, virt_addr);
+
+                    if (l2_entry->IsMappedBlock() || l2_entry->IsMappedEmpty()) {
+                        const size_t advance_size = (l2_entry->IsMappedBlock() && l2_entry->IsContiguous()) ? L2ContiguousBlockSize : L2BlockSize;
+                        MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), advance_size));
+                        MESOSPHERE_INIT_ABORT_UNLESS(static_cast<size_t>(end_virt_addr - virt_addr) >= advance_size);
+                        if (l2_entry->IsMappedBlock() && block_size == advance_size) {
+                            if ((count++) == index) {
+                                return virt_addr;
+                            }
+                        }
+                        virt_addr += advance_size;
+                        continue;
+                    }
+
+                    /* Non empty and non-block must be table. */
+                    MESOSPHERE_INIT_ABORT_UNLESS(l2_entry->IsMappedTable());
+
+                    /* Table, so check if we're mapped in L3. */
+                    L3PageTableEntry *l3_entry = GetL3Entry(l2_entry, virt_addr);
+
+                    /* L3 must be block or empty. */
+                    MESOSPHERE_INIT_ABORT_UNLESS(l3_entry->IsMappedBlock() || l3_entry->IsMappedEmpty());
+
+                    const size_t advance_size = (l3_entry->IsMappedBlock() && l3_entry->IsContiguous()) ? L3ContiguousBlockSize : L3BlockSize;
+                    MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), advance_size));
+                    MESOSPHERE_INIT_ABORT_UNLESS(static_cast<size_t>(end_virt_addr - virt_addr) >= advance_size);
+                    if (l3_entry->IsMappedBlock() && block_size == advance_size) {
+                        if ((count++) == index) {
+                            return virt_addr;
+                        }
+                    }
+                    virt_addr += advance_size;
+                }
+                return Null<KVirtualAddress>;
+            }
+
+            PageTableEntry *GetMappingEntry(KVirtualAddress virt_addr, size_t block_size) {
+                L1PageTableEntry *l1_entry = this->GetL1Entry(virt_addr);
+
+                if (l1_entry->IsMappedBlock()) {
+                    MESOSPHERE_INIT_ABORT_UNLESS(block_size == L1BlockSize);
+                    return l1_entry;
+                }
+
+                MESOSPHERE_INIT_ABORT_UNLESS(l1_entry->IsMappedTable());
+
+                /* Table, so check if we're mapped in L2. */
+                L2PageTableEntry *l2_entry = GetL2Entry(l1_entry, virt_addr);
+
+                if (l2_entry->IsMappedBlock()) {
+                    const size_t real_size = (l2_entry->IsContiguous()) ? L2ContiguousBlockSize : L2BlockSize;
+                    MESOSPHERE_INIT_ABORT_UNLESS(real_size == block_size);
+                    return l2_entry;
+                }
+
+                MESOSPHERE_INIT_ABORT_UNLESS(l2_entry->IsMappedTable());
+
+                /* Table, so check if we're mapped in L3. */
+                L3PageTableEntry *l3_entry = GetL3Entry(l2_entry, virt_addr);
+
+                /* L3 must be block. */
+                MESOSPHERE_INIT_ABORT_UNLESS(l3_entry->IsMappedBlock());
+
+                const size_t real_size = (l3_entry->IsContiguous()) ? L3ContiguousBlockSize : L3BlockSize;
+                MESOSPHERE_INIT_ABORT_UNLESS(real_size == block_size);
+                return l3_entry;
+            }
+
+            void NOINLINE SwapBlocks(KVirtualAddress src_virt_addr, KVirtualAddress dst_virt_addr, size_t block_size, bool do_copy) {
+                static_assert(L2ContiguousBlockSize / L2BlockSize == L3ContiguousBlockSize / L3BlockSize);
+                const bool contig = (block_size == L2ContiguousBlockSize || block_size == L3ContiguousBlockSize);
+                const size_t num_mappings = contig ? L2ContiguousBlockSize / L2BlockSize : 1;
+
+                /* Unmap the source. */
+                PageTableEntry *src_entry = this->GetMappingEntry(src_virt_addr, block_size);
+                const auto src_saved = *src_entry;
+                for (size_t i = 0; i < num_mappings; i++) {
+                    src_entry[i] = InvalidPageTableEntry;
+                }
+
+                /* Unmap the target. */
+                PageTableEntry *dst_entry = this->GetMappingEntry(dst_virt_addr, block_size);
+                const auto dst_saved = *dst_entry;
+                for (size_t i = 0; i < num_mappings; i++) {
+                    dst_entry[i] = InvalidPageTableEntry;
+                }
+
+                /* Invalidate the entire tlb. */
+                cpu::DataSynchronizationBarrierInnerShareable();
+                cpu::InvalidateEntireTlb();
+
+                /* Copy data, if we should. */
+                const u64 negative_block_size_for_mask = static_cast<u64>(-static_cast<s64>(block_size));
+                const u64 offset_mask                  = negative_block_size_for_mask & ((1ul << 48) - 1);
+                const KVirtualAddress copy_src_addr = KVirtualAddress(src_saved.GetRawAttributesUnsafeForSwap() & offset_mask);
+                const KVirtualAddress copy_dst_addr = KVirtualAddress(dst_saved.GetRawAttributesUnsafeForSwap() & offset_mask);
+                if (do_copy) {
+                    u8 tmp[0x100];
+                    for (size_t ofs = 0; ofs < block_size; ofs += sizeof(tmp)) {
+                        std::memcpy(tmp, GetVoidPointer(copy_src_addr + ofs), sizeof(tmp));
+                        std::memcpy(GetVoidPointer(copy_src_addr + ofs), GetVoidPointer(copy_dst_addr + ofs), sizeof(tmp));
+                        std::memcpy(GetVoidPointer(copy_dst_addr + ofs), tmp, sizeof(tmp));
+                    }
+                    cpu::DataSynchronizationBarrierInnerShareable();
+                }
+
+                /* Swap the mappings. */
+                const u64 attr_preserve_mask  = (block_size - 1) | 0xFFFF000000000000ul;
+                const size_t shift_for_contig = contig ? 4 : 0;
+                size_t advanced_size = 0;
+                const u64 src_attr_val = src_saved.GetRawAttributesUnsafeForSwap() & attr_preserve_mask;
+                const u64 dst_attr_val = dst_saved.GetRawAttributesUnsafeForSwap() & attr_preserve_mask;
+                for (size_t i = 0; i < num_mappings; i++) {
+                    reinterpret_cast<u64 *>(src_entry)[i] = GetInteger(copy_dst_addr + (advanced_size >> shift_for_contig)) | src_attr_val;
+                    reinterpret_cast<u64 *>(dst_entry)[i] = GetInteger(copy_src_addr + (advanced_size >> shift_for_contig)) | dst_attr_val;
+                    advanced_size += block_size;
+                }
+
+                cpu::DataSynchronizationBarrierInnerShareable();
+            }
+
+            void NOINLINE PhysicallyRandomize(KVirtualAddress virt_addr, size_t size, size_t block_size, bool do_copy) {
+                const size_t block_count = this->GetBlockCount(virt_addr, size, block_size);
+                if (block_count > 1) {
+                    for (size_t cur_block = 0; cur_block < block_count; cur_block++) {
+                        const size_t target_block = KSystemControl::Init::GenerateRandomRange(cur_block, block_count - 1);
+                        if (cur_block != target_block) {
+                            const KVirtualAddress cur_virt_addr    = this->GetBlockByIndex(virt_addr, size, block_size, cur_block);
+                            const KVirtualAddress target_virt_addr = this->GetBlockByIndex(virt_addr, size, block_size, target_block);
+                            MESOSPHERE_INIT_ABORT_UNLESS(cur_virt_addr    != Null<KVirtualAddress>);
+                            MESOSPHERE_INIT_ABORT_UNLESS(target_virt_addr != Null<KVirtualAddress>);
+                            this->SwapBlocks(cur_virt_addr, target_virt_addr, block_size, do_copy);
+                        }
+                    }
+                }
+            }
+        public:
+            template<IsInitialPageAllocator PageAllocator>
+            void NOINLINE Map(KVirtualAddress virt_addr, size_t size, KPhysicalAddress phys_addr, const PageTableEntry &attr, PageAllocator &allocator, u64 phys_to_virt_offset) {
+                /* Ensure that addresses and sizes are page aligned. */
+                MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr),    PageSize));
+                MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(phys_addr),    PageSize));
+                MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(size,                     PageSize));
+
+                /* Iteratively map pages until the requested region is mapped. */
+                while (size > 0) {
+                    L1PageTableEntry *l1_entry = this->GetL1Entry(virt_addr, phys_to_virt_offset);
+
+                    /* Can we make an L1 block? */
+                    if (util::IsAligned(GetInteger(virt_addr), L1BlockSize) && util::IsAligned(GetInteger(phys_addr), L1BlockSize) && size >= L1BlockSize) {
+                        *l1_entry = L1PageTableEntry(PageTableEntry::BlockTag{}, phys_addr, attr, PageTableEntry::SoftwareReservedBit_None, false);
+
+                        virt_addr += L1BlockSize;
+                        phys_addr += L1BlockSize;
+                        size      -= L1BlockSize;
+                        continue;
+                    }
+
+                    /* If we don't already have an L2 table, we need to make a new one. */
+                    if (!l1_entry->IsMappedTable()) {
+                        KPhysicalAddress new_table = AllocateNewPageTable(allocator, phys_to_virt_offset);
+                        cpu::DataSynchronizationBarrierInnerShareable();
+                        *l1_entry = L1PageTableEntry(PageTableEntry::TableTag{}, new_table, attr.IsPrivilegedExecuteNever());
+                    }
+
+                    L2PageTableEntry *l2_entry = GetL2Entry(l1_entry, virt_addr, phys_to_virt_offset);
+
+                    /* Can we make a contiguous L2 block? */
+                    if (util::IsAligned(GetInteger(virt_addr), L2ContiguousBlockSize) && util::IsAligned(GetInteger(phys_addr), L2ContiguousBlockSize) && size >= L2ContiguousBlockSize) {
+                        for (size_t i = 0; i < L2ContiguousBlockSize / L2BlockSize; i++) {
+                            l2_entry[i] = L2PageTableEntry(PageTableEntry::BlockTag{}, phys_addr, attr, PageTableEntry::SoftwareReservedBit_None, true);
+
+                            virt_addr += L2BlockSize;
+                            phys_addr += L2BlockSize;
+                            size      -= L2BlockSize;
+                        }
+                        continue;
+                    }
+
+                    /* Can we make an L2 block? */
+                    if (util::IsAligned(GetInteger(virt_addr), L2BlockSize) && util::IsAligned(GetInteger(phys_addr), L2BlockSize) && size >= L2BlockSize) {
+                        *l2_entry = L2PageTableEntry(PageTableEntry::BlockTag{}, phys_addr, attr, PageTableEntry::SoftwareReservedBit_None, false);
+
+                        virt_addr += L2BlockSize;
+                        phys_addr += L2BlockSize;
+                        size      -= L2BlockSize;
+                        continue;
+                    }
+
+                    /* If we don't already have an L3 table, we need to make a new one. */
+                    if (!l2_entry->IsMappedTable()) {
+                        KPhysicalAddress new_table = AllocateNewPageTable(allocator, phys_to_virt_offset);
+                        cpu::DataSynchronizationBarrierInnerShareable();
+                        *l2_entry = L2PageTableEntry(PageTableEntry::TableTag{}, new_table, attr.IsPrivilegedExecuteNever());
+                    }
+
+                    L3PageTableEntry *l3_entry = GetL3Entry(l2_entry, virt_addr, phys_to_virt_offset);
+
+                    /* Can we make a contiguous L3 block? */
+                    if (util::IsAligned(GetInteger(virt_addr), L3ContiguousBlockSize) && util::IsAligned(GetInteger(phys_addr), L3ContiguousBlockSize) && size >= L3ContiguousBlockSize) {
+                        for (size_t i = 0; i < L3ContiguousBlockSize / L3BlockSize; i++) {
+                            l3_entry[i] = L3PageTableEntry(PageTableEntry::BlockTag{}, phys_addr, attr, PageTableEntry::SoftwareReservedBit_None, true);
+
+                            virt_addr += L3BlockSize;
+                            phys_addr += L3BlockSize;
+                            size      -= L3BlockSize;
+                        }
+                        continue;
+                    }
+
+                    /* Make an L3 block. */
+                    *l3_entry = L3PageTableEntry(PageTableEntry::BlockTag{}, phys_addr, attr, PageTableEntry::SoftwareReservedBit_None, false);
+                    virt_addr += L3BlockSize;
+                    phys_addr += L3BlockSize;
+                    size      -= L3BlockSize;
+                }
+
+                /* Ensure data consistency after our mapping is added. */
+                cpu::DataSynchronizationBarrierInnerShareable();
+            }
+
+            void UnmapTtbr0Entries(u64 phys_to_virt_offset) {
+                /* Ensure data consistency before we unmap. */
+                cpu::DataSynchronizationBarrierInnerShareable();
+
+                /* Define helper, as we only want to clear non-nGnRE pages. */
+                constexpr auto ShouldUnmap = [](const PageTableEntry *entry) ALWAYS_INLINE_LAMBDA -> bool {
+                    return entry->GetPageAttribute() != PageTableEntry::PageAttribute_Device_nGnRE;
+                };
+
+                /* Iterate all L1 entries. */
+                L1PageTableEntry * const l1_table = reinterpret_cast<L1PageTableEntry *>(GetInteger(m_l1_tables[0]) + phys_to_virt_offset);
+                for (size_t l1_index = 0; l1_index < m_num_entries[0]; l1_index++) {
+                    /* Get L1 entry. */
+                    L1PageTableEntry * const l1_entry = l1_table + l1_index;
+                    if (l1_entry->IsMappedBlock()) {
+                        /* Unmap the L1 entry, if we should. */
+                        if (ShouldUnmap(l1_entry)) {
+                            *static_cast<PageTableEntry *>(l1_entry) = InvalidPageTableEntry;
+                        }
+                    } else if (l1_entry->IsMappedTable()) {
+                        /* Get the L2 table. */
+                        L2PageTableEntry * const l2_table = reinterpret_cast<L2PageTableEntry *>(GetInteger(l1_entry->GetTable()) + phys_to_virt_offset);
+
+                        /* Unmap all L2 entries, as relevant. */
+                        size_t remaining_l2_entries = 0;
+                        for (size_t l2_index = 0; l2_index < MaxPageTableEntries; ++l2_index) {
+                            /* Get L2 entry. */
+                            L2PageTableEntry * const l2_entry = l2_table + l2_index;
+                            if (l2_entry->IsMappedBlock()) {
+                                const size_t num_to_clear = (l2_entry->IsContiguous() ? L2ContiguousBlockSize : L2BlockSize) / L2BlockSize;
+
+                                if (ShouldUnmap(l2_entry)) {
+                                    for (size_t i = 0; i < num_to_clear; ++i) {
+                                        static_cast<PageTableEntry *>(l2_entry)[i] = InvalidPageTableEntry;
+                                    }
+                                } else {
+                                    remaining_l2_entries += num_to_clear;
+                                }
+
+                                l2_index = l2_index + num_to_clear - 1;
+                            } else if (l2_entry->IsMappedTable()) {
+                                /* Get the L3 table. */
+                                L3PageTableEntry * const l3_table = reinterpret_cast<L3PageTableEntry *>(GetInteger(l2_entry->GetTable()) + phys_to_virt_offset);
+
+                                /* Unmap all L3 entries, as relevant. */
+                                size_t remaining_l3_entries = 0;
+                                for (size_t l3_index = 0; l3_index < MaxPageTableEntries; ++l3_index) {
+                                    /* Get L3 entry. */
+                                    if (L3PageTableEntry * const l3_entry = l3_table + l3_index; l3_entry->IsMappedBlock()) {
+                                        const size_t num_to_clear = (l3_entry->IsContiguous() ? L3ContiguousBlockSize : L3BlockSize) / L3BlockSize;
+
+                                        if (ShouldUnmap(l3_entry)) {
+                                            for (size_t i = 0; i < num_to_clear; ++i) {
+                                                static_cast<PageTableEntry *>(l3_entry)[i] = InvalidPageTableEntry;
+                                            }
+                                        } else {
+                                            remaining_l3_entries += num_to_clear;
+                                        }
+
+                                        l3_index = l3_index + num_to_clear - 1;
+                                    }
+                                }
+
+                                /* If we unmapped all L3 entries, clear the L2 entry. */
+                                if (remaining_l3_entries == 0) {
+                                    *static_cast<PageTableEntry *>(l2_entry) = InvalidPageTableEntry;
+
+                                    /* Invalidate the entire tlb. */
+                                    cpu::DataSynchronizationBarrierInnerShareable();
+                                    cpu::InvalidateEntireTlb();
+                                } else {
+                                    remaining_l2_entries++;
+                                }
+                            }
+                        }
+
+                        /* If we unmapped all L2 entries, clear the L1 entry. */
+                        if (remaining_l2_entries == 0) {
+                            *static_cast<PageTableEntry *>(l1_entry) = InvalidPageTableEntry;
+
+                            /* Invalidate the entire tlb. */
+                            cpu::DataSynchronizationBarrierInnerShareable();
+                            cpu::InvalidateEntireTlb();
+                        }
+                    }
+                }
+
+                /* Invalidate the entire tlb. */
+                cpu::DataSynchronizationBarrierInnerShareable();
+                cpu::InvalidateEntireTlb();
+            }
+
+            KPhysicalAddress GetPhysicalAddress(KVirtualAddress virt_addr) const {
+                /* Get the L1 entry. */
+                const L1PageTableEntry *l1_entry = this->GetL1Entry(virt_addr);
+
+                if (l1_entry->IsMappedBlock()) {
+                    return l1_entry->GetBlock() + (GetInteger(virt_addr) & (L1BlockSize - 1));
+                }
+
+                MESOSPHERE_INIT_ABORT_UNLESS(l1_entry->IsMappedTable());
+
+                /* Get the L2 entry. */
+                const L2PageTableEntry *l2_entry = GetL2Entry(l1_entry, virt_addr);
+
+                if (l2_entry->IsMappedBlock()) {
+                    return l2_entry->GetBlock() + (GetInteger(virt_addr) & (L2BlockSize - 1));
+                }
+
+                MESOSPHERE_INIT_ABORT_UNLESS(l2_entry->IsMappedTable());
+
+                /* Get the L3 entry. */
+                const L3PageTableEntry *l3_entry = GetL3Entry(l2_entry, virt_addr);
+
+                MESOSPHERE_INIT_ABORT_UNLESS(l3_entry->IsMappedBlock());
+
+                return l3_entry->GetBlock() + (GetInteger(virt_addr) & (L3BlockSize - 1));
+            }
+
+            KPhysicalAddress GetPhysicalAddressOfRandomizedRange(KVirtualAddress virt_addr, size_t size) const {
+                /* Define tracking variables for ourselves to use. */
+                KPhysicalAddress min_phys_addr = Null<KPhysicalAddress>;
+                KPhysicalAddress max_phys_addr = Null<KPhysicalAddress>;
+
+                /* Ensure the range we're querying is valid. */
+                const KVirtualAddress end_virt_addr = virt_addr + size;
+                if (virt_addr > end_virt_addr) {
+                    MESOSPHERE_INIT_ABORT_UNLESS(size == 0);
+                    return min_phys_addr;
+                }
+
+                auto UpdateExtents = [&](const KPhysicalAddress block, size_t block_size) ALWAYS_INLINE_LAMBDA {
+                    /* Ensure that we are allowed to have the block here. */
+                    MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), block_size));
+                    MESOSPHERE_INIT_ABORT_UNLESS(block_size <= GetInteger(end_virt_addr) - GetInteger(virt_addr));
+                    MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(block),     block_size));
+                    MESOSPHERE_INIT_ABORT_UNLESS(size >= block_size);
+
+                    const KPhysicalAddress block_end = block + block_size;
+
+                    /* We want to update min phys addr when it's 0 or > block. */
+                    /* This is equivalent in two's complement to (n - 1) >= block. */
+                    if ((GetInteger(min_phys_addr) - 1) >= GetInteger(block)) {
+                        min_phys_addr = block;
+                    }
+
+                    /* Update max phys addr when it's 0 or < block_end. */
+                    if (GetInteger(max_phys_addr) < GetInteger(block_end) || GetInteger(max_phys_addr) == 0) {
+                        max_phys_addr = block_end;
+                    }
+
+                    /* Traverse onwards. */
+                    virt_addr += block_size;
+                };
+
+                while (virt_addr < end_virt_addr) {
+                    L1PageTableEntry *l1_entry = this->GetL1Entry(virt_addr);
+
+                    /* If an L1 block is mapped, update. */
+                    if (l1_entry->IsMappedBlock()) {
+                        UpdateExtents(l1_entry->GetBlock(), L1BlockSize);
+                        continue;
+                    }
+
+                    /* Not a block, so we must have a table. */
+                    MESOSPHERE_INIT_ABORT_UNLESS(l1_entry->IsMappedTable());
+
+                    L2PageTableEntry *l2_entry = GetL2Entry(l1_entry, virt_addr);
+                    if (l2_entry->IsMappedBlock()) {
+                        UpdateExtents(l2_entry->GetBlock(), l2_entry->IsContiguous() ? L2ContiguousBlockSize : L2BlockSize);
+                        continue;
+                    }
+
+                    /* Not a block, so we must have a table. */
+                    MESOSPHERE_INIT_ABORT_UNLESS(l2_entry->IsMappedTable());
+
+                    /* We must have a mapped l3 entry to inspect. */
+                    L3PageTableEntry *l3_entry = GetL3Entry(l2_entry, virt_addr);
+                    MESOSPHERE_INIT_ABORT_UNLESS(l3_entry->IsMappedBlock());
+
+                    UpdateExtents(l3_entry->GetBlock(), l3_entry->IsContiguous() ? L3ContiguousBlockSize : L3BlockSize);
+                }
+
+                /* Ensure we got the right range. */
+                MESOSPHERE_INIT_ABORT_UNLESS(GetInteger(max_phys_addr) - GetInteger(min_phys_addr) == size);
+
+                /* Write the address that we found. */
+                return min_phys_addr;
+            }
+
+            bool IsFree(KVirtualAddress virt_addr, size_t size) {
+                /* Ensure that addresses and sizes are page aligned. */
+                MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr),    PageSize));
+                MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(size,                     PageSize));
+
+                const KVirtualAddress end_virt_addr = virt_addr + size;
+                while (virt_addr < end_virt_addr) {
+                    L1PageTableEntry *l1_entry = this->GetL1Entry(virt_addr);
+
+                    /* If an L1 block is mapped, the address isn't free. */
+                    if (l1_entry->IsMappedBlock()) {
+                        return false;
+                    }
+
+                    if (!l1_entry->IsMappedTable()) {
+                        /* Not a table, so just move to check the next region. */
+                        virt_addr = util::AlignDown(GetInteger(virt_addr) + L1BlockSize, L1BlockSize);
+                        continue;
+                    }
+
+                    /* Table, so check if we're mapped in L2. */
+                    L2PageTableEntry *l2_entry = GetL2Entry(l1_entry, virt_addr);
+
+                    if (l2_entry->IsMappedBlock()) {
+                        return false;
+                    }
+
+                    if (!l2_entry->IsMappedTable()) {
+                        /* Not a table, so just move to check the next region. */
+                        virt_addr = util::AlignDown(GetInteger(virt_addr) + L2BlockSize, L2BlockSize);
+                        continue;
+                    }
+
+                    /* Table, so check if we're mapped in L3. */
+                    L3PageTableEntry *l3_entry = GetL3Entry(l2_entry, virt_addr);
+
+                    if (l3_entry->IsMappedBlock()) {
+                        return false;
+                    }
+
+                    /* Not a block, so move on to check the next page. */
+                    virt_addr = util::AlignDown(GetInteger(virt_addr) + L3BlockSize, L3BlockSize);
+                }
+                return true;
+            }
+
+            void Reprotect(KVirtualAddress virt_addr, size_t size, const PageTableEntry &attr_before, const PageTableEntry &attr_after) {
+                /* Ensure that addresses and sizes are page aligned. */
+                MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr),    PageSize));
+                MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(size,                     PageSize));
+
+                /* Iteratively reprotect pages until the requested region is reprotected. */
+                while (size > 0) {
+                    L1PageTableEntry *l1_entry = this->GetL1Entry(virt_addr);
+
+                    /* Check if an L1 block is present. */
+                    if (l1_entry->IsMappedBlock()) {
+                        /* Ensure that we are allowed to have an L1 block here. */
+                        const KPhysicalAddress block = l1_entry->GetBlock();
+                        MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), L1BlockSize));
+                        MESOSPHERE_INIT_ABORT_UNLESS(size >= L1BlockSize);
+                        MESOSPHERE_INIT_ABORT_UNLESS(l1_entry->IsCompatibleWithAttribute(attr_before, PageTableEntry::SoftwareReservedBit_None, false));
+
+                        /* Invalidate the existing L1 block. */
+                        *static_cast<PageTableEntry *>(l1_entry) = InvalidPageTableEntry;
+                        cpu::DataSynchronizationBarrierInnerShareable();
+                        cpu::InvalidateEntireTlb();
+
+                        /* Create new L1 block. */
+                        *l1_entry = L1PageTableEntry(PageTableEntry::BlockTag{}, block, attr_after, PageTableEntry::SoftwareReservedBit_None, false);
+
+                        virt_addr += L1BlockSize;
+                        size      -= L1BlockSize;
+                        continue;
+                    }
+
+                    /* Not a block, so we must be a table. */
+                    MESOSPHERE_INIT_ABORT_UNLESS(l1_entry->IsMappedTable());
+
+                    L2PageTableEntry *l2_entry = GetL2Entry(l1_entry, virt_addr);
+                    if (l2_entry->IsMappedBlock()) {
+                        const KPhysicalAddress block = l2_entry->GetBlock();
+
+                        if (l2_entry->IsContiguous()) {
+                            /* Ensure that we are allowed to have a contiguous L2 block here. */
+                            MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), L2ContiguousBlockSize));
+                            MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(block),     L2ContiguousBlockSize));
+                            MESOSPHERE_INIT_ABORT_UNLESS(size >= L2ContiguousBlockSize);
+
+                            /* Invalidate the existing contiguous L2 block. */
+                            for (size_t i = 0; i < L2ContiguousBlockSize / L2BlockSize; i++) {
+                                /* Ensure that the entry is valid. */
+                                MESOSPHERE_INIT_ABORT_UNLESS(l2_entry[i].IsCompatibleWithAttribute(attr_before, PageTableEntry::SoftwareReservedBit_None, true));
+                                static_cast<PageTableEntry *>(l2_entry)[i] = InvalidPageTableEntry;
+                            }
+                            cpu::DataSynchronizationBarrierInnerShareable();
+                            cpu::InvalidateEntireTlb();
+
+                            /* Create a new contiguous L2 block. */
+                            for (size_t i = 0; i < L2ContiguousBlockSize / L2BlockSize; i++) {
+                                l2_entry[i] = L2PageTableEntry(PageTableEntry::BlockTag{}, block + L2BlockSize * i, attr_after, PageTableEntry::SoftwareReservedBit_None, true);
+                            }
+
+                            virt_addr += L2ContiguousBlockSize;
+                            size      -= L2ContiguousBlockSize;
+                        } else {
+                            /* Ensure that we are allowed to have an L2 block here. */
+                            MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), L2BlockSize));
+                            MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(block),     L2BlockSize));
+                            MESOSPHERE_INIT_ABORT_UNLESS(size >= L2BlockSize);
+                            MESOSPHERE_INIT_ABORT_UNLESS(l2_entry->IsCompatibleWithAttribute(attr_before, PageTableEntry::SoftwareReservedBit_None, false));
+
+                            /* Invalidate the existing L2 block. */
+                            *static_cast<PageTableEntry *>(l2_entry) = InvalidPageTableEntry;
+                            cpu::DataSynchronizationBarrierInnerShareable();
+                            cpu::InvalidateEntireTlb();
+
+                            /* Create new L2 block. */
+                            *l2_entry = L2PageTableEntry(PageTableEntry::BlockTag{}, block, attr_after, PageTableEntry::SoftwareReservedBit_None, false);
+
+                            virt_addr += L2BlockSize;
+                            size      -= L2BlockSize;
+                        }
+
+                        continue;
+                    }
+
+                    /* Not a block, so we must be a table. */
+                    MESOSPHERE_INIT_ABORT_UNLESS(l2_entry->IsMappedTable());
+
+                    /* We must have a mapped l3 entry to reprotect. */
+                    L3PageTableEntry *l3_entry = GetL3Entry(l2_entry, virt_addr);
+                    MESOSPHERE_INIT_ABORT_UNLESS(l3_entry->IsMappedBlock());
+                    const KPhysicalAddress block = l3_entry->GetBlock();
+
+                    if (l3_entry->IsContiguous()) {
+                        /* Ensure that we are allowed to have a contiguous L3 block here. */
+                        MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), L3ContiguousBlockSize));
+                        MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(block),     L3ContiguousBlockSize));
+                        MESOSPHERE_INIT_ABORT_UNLESS(size >= L3ContiguousBlockSize);
+
+                        /* Invalidate the existing contiguous L3 block. */
+                        for (size_t i = 0; i < L3ContiguousBlockSize / L3BlockSize; i++) {
+                            /* Ensure that the entry is valid. */
+                            MESOSPHERE_INIT_ABORT_UNLESS(l3_entry[i].IsCompatibleWithAttribute(attr_before, PageTableEntry::SoftwareReservedBit_None, true));
+                            static_cast<PageTableEntry *>(l3_entry)[i] = InvalidPageTableEntry;
+                        }
+                        cpu::DataSynchronizationBarrierInnerShareable();
+                        cpu::InvalidateEntireTlb();
+
+                        /* Create a new contiguous L3 block. */
+                        for (size_t i = 0; i < L3ContiguousBlockSize / L3BlockSize; i++) {
+                            l3_entry[i] = L3PageTableEntry(PageTableEntry::BlockTag{}, block + L3BlockSize * i, attr_after, PageTableEntry::SoftwareReservedBit_None, true);
+                        }
+
+                        virt_addr += L3ContiguousBlockSize;
+                        size      -= L3ContiguousBlockSize;
+                    } else {
+                        /* Ensure that we are allowed to have an L3 block here. */
+                        MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), L3BlockSize));
+                        MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(block),     L3BlockSize));
+                        MESOSPHERE_INIT_ABORT_UNLESS(size >= L3BlockSize);
+                        MESOSPHERE_INIT_ABORT_UNLESS(l3_entry->IsCompatibleWithAttribute(attr_before, PageTableEntry::SoftwareReservedBit_None, false));
+
+                        /* Invalidate the existing L3 block. */
+                        *static_cast<PageTableEntry *>(l3_entry) = InvalidPageTableEntry;
+                        cpu::DataSynchronizationBarrierInnerShareable();
+                        cpu::InvalidateEntireTlb();
+
+                        /* Create new L3 block. */
+                        *l3_entry = L3PageTableEntry(PageTableEntry::BlockTag{}, block, attr_after, PageTableEntry::SoftwareReservedBit_None, false);
+
+                        virt_addr += L3BlockSize;
+                        size      -= L3BlockSize;
+                    }
+                }
+
+                /* Ensure data consistency after we complete reprotection. */
+                cpu::DataSynchronizationBarrierInnerShareable();
+            }
+
+            void PhysicallyRandomize(KVirtualAddress virt_addr, size_t size, bool do_copy) {
+                this->PhysicallyRandomize(virt_addr, size, L1BlockSize, do_copy);
+                this->PhysicallyRandomize(virt_addr, size, L2ContiguousBlockSize, do_copy);
+                this->PhysicallyRandomize(virt_addr, size, L2BlockSize, do_copy);
+                this->PhysicallyRandomize(virt_addr, size, L3ContiguousBlockSize, do_copy);
+                this->PhysicallyRandomize(virt_addr, size, L3BlockSize, do_copy);
+                cpu::StoreCacheForInit(GetVoidPointer(virt_addr), size);
+            }
+    };
+
+    class KInitialPageAllocator final {
+        private:
+            static constexpr inline size_t FreeUnitSize = BITSIZEOF(u64) * PageSize;
+            struct FreeListEntry {
+                FreeListEntry *next;
+                size_t size;
+            };
+        public:
+            struct State {
+                uintptr_t start_address;
+                uintptr_t end_address;
+                FreeListEntry *free_head;
+            };
+        private:
+            State m_state;
+        public:
+            constexpr ALWAYS_INLINE KInitialPageAllocator() : m_state{} { /* ... */ }
+
+            ALWAYS_INLINE void Initialize(uintptr_t address) {
+                m_state.start_address = address;
+                m_state.end_address  = address;
+            }
+
+            ALWAYS_INLINE void InitializeFromState(const State *state) {
+                m_state = *state;
+            }
+
+            ALWAYS_INLINE void GetFinalState(State *out) {
+                *out = m_state;
+                m_state = {};
+            }
+        private:
+            bool CanAllocate(size_t align, size_t size) const {
+                for (auto *cur = m_state.free_head; cur != nullptr; cur = cur->next) {
+                    const uintptr_t cur_last   = reinterpret_cast<uintptr_t>(cur) + cur->size - 1;
+                    const uintptr_t alloc_last = util::AlignUp(reinterpret_cast<uintptr_t>(cur), align) + size - 1;
+                    if (alloc_last <= cur_last) {
+                        return true;
+                    }
+                }
+                return false;
+            }
+
+            bool TryAllocate(uintptr_t address, size_t size) {
+                /* Try to allocate the region. */
+                auto **prev_next = std::addressof(m_state.free_head);
+                for (auto *cur = m_state.free_head; cur != nullptr; prev_next = std::addressof(cur->next), cur = cur->next) {
+                    const uintptr_t cur_start = reinterpret_cast<uintptr_t>(cur);
+                    const uintptr_t cur_last  = cur_start + cur->size - 1;
+                    if (cur_start <= address && address + size - 1 <= cur_last) {
+                        auto *alloc = reinterpret_cast<FreeListEntry *>(address);
+
+                        /* Perform fragmentation at front. */
+                        if (cur != alloc) {
+                            prev_next = std::addressof(cur->next);
+                            *alloc = {
+                                .next = cur->next,
+                                .size = cur_start + cur->size - address,
+                            };
+                            *cur = {
+                                .next = alloc,
+                                .size = address - cur_start,
+                            };
+                        }
+
+                        /* Perform fragmentation at tail. */
+                        if (alloc->size != size) {
+                            auto *next = reinterpret_cast<FreeListEntry *>(address + size);
+                            *next = {
+                                .next = alloc->next,
+                                .size = alloc->size - size,
+                            };
+                            *alloc = {
+                                .next = next,
+                                .size = size,
+                            };
+                        }
+
+                        *prev_next = alloc->next;
+                        return true;
+                    }
+                }
+
+                return false;
+            }
+        public:
+            KPhysicalAddress Allocate(size_t align, size_t size) {
+                /* Ensure that the free list is non-empty. */
+                while (!this->CanAllocate(align, size)) {
+                    this->Free(m_state.end_address, FreeUnitSize);
+                    m_state.end_address += FreeUnitSize;
+                }
+
+                /* Allocate a random address. */
+                const uintptr_t aligned_start = util::AlignUp(m_state.start_address, align);
+                const uintptr_t aligned_end   = util::AlignDown(m_state.end_address, align);
+                const size_t ind_max = ((aligned_end - aligned_start) / align) - 1;
+                while (true) {
+                    if (const uintptr_t random_address = aligned_start + (KSystemControl::Init::GenerateRandomRange(0, ind_max) * align); this->TryAllocate(random_address, size)) {
+                        /* Clear the allocated pages. */
+                        volatile u64 *ptr = reinterpret_cast<volatile u64 *>(random_address);
+                        for (size_t i = 0; i < size / sizeof(u64); ++i) {
+                            ptr[i] = 0;
+                        }
+
+                        return random_address;
+                    }
+                }
+            }
+
+            KPhysicalAddress Allocate(size_t size) {
+                return this->Allocate(size, size);
+            }
+
+            void Free(KPhysicalAddress phys_addr, size_t size) {
+                auto **prev_next = std::addressof(m_state.free_head);
+                auto *new_chunk  = reinterpret_cast<FreeListEntry *>(GetInteger(phys_addr));
+                if (auto *cur = m_state.free_head; cur != nullptr) {
+                    const uintptr_t new_start = reinterpret_cast<uintptr_t>(new_chunk);
+                    const uintptr_t new_end   = GetInteger(phys_addr) + size;
+                    while (true) {
+                        /* Attempt coalescing. */
+                        const uintptr_t cur_start = reinterpret_cast<uintptr_t>(cur);
+                        const uintptr_t cur_end   = cur_start + cur->size;
+                        if (new_start < new_end) {
+                            if (new_end < cur_start) {
+                                *new_chunk = {
+                                    .next = cur,
+                                    .size = size,
+                                };
+                                break;
+                            } else if (new_end == cur_start) {
+                                *new_chunk = {
+                                    .next = cur->next,
+                                    .size = cur->size + size,
+                                };
+                                break;
+                            }
+                        } else if (cur_end == new_start) {
+                            cur->size += size;
+                            return;
+                        }
+
+                        prev_next = std::addressof(cur->next);
+                        if (cur->next != nullptr) {
+                            cur = cur->next;
+                        } else {
+                            *new_chunk = {
+                                .next = nullptr,
+                                .size = size,
+                            };
+                            cur->next = new_chunk;
+                            return;
+                        }
+                    }
+
+                } else {
+                    *new_chunk = {
+                        .next = nullptr,
+                        .size = size,
+                    };
+                }
+
+                *prev_next = new_chunk;
+            }
+    };
+    static_assert(IsInitialPageAllocator<KInitialPageAllocator>);
+
+}

Source/Atmosphere/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_assembly_macros.h

@@ -0,0 +1,101 @@
+/*
+ * Copyright (c) Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/arch/arm64/kern_assembly_offsets.h>
+
+#define ENABLE_FPU(tmp)          \
+    mrs     tmp, cpacr_el1;      \
+    orr     tmp, tmp, #0x300000; \
+    msr     cpacr_el1, tmp;      \
+    isb;
+
+#define GET_THREAD_CONTEXT_AND_RESTORE_FPCR_FPSR(ctx, xtmp1, xtmp2, wtmp1, wtmp2)        \
+    add     ctx, sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_THREAD_CONTEXT); \
+    ldp     wtmp1, wtmp2, [ctx, #(THREAD_CONTEXT_FPCR_FPSR)];                            \
+    msr     fpcr, xtmp1;                                                                 \
+    msr     fpsr, xtmp2;
+
+#define RESTORE_FPU64_CALLEE_SAVE_REGISTERS(ctx)              \
+    ldp     q8,  q9,  [ctx, #(THREAD_CONTEXT_FPU64_Q8_Q9)];   \
+    ldp     q10, q11, [ctx, #(THREAD_CONTEXT_FPU64_Q10_Q11)]; \
+    ldp     q12, q13, [ctx, #(THREAD_CONTEXT_FPU64_Q12_Q13)]; \
+    ldp     q14, q15, [ctx, #(THREAD_CONTEXT_FPU64_Q14_Q15)];
+
+#define RESTORE_FPU64_CALLER_SAVE_REGISTERS(tmp)                                                      \
+    ldr     tmp, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_CALLER_SAVE_FPU_REGISTERS)]; \
+    ldp     q0,  q1,  [tmp, #(THREAD_FPU64_CONTEXT_Q0_Q1)];                                           \
+    ldp     q2,  q3,  [tmp, #(THREAD_FPU64_CONTEXT_Q2_Q3)];                                           \
+    ldp     q4,  q5,  [tmp, #(THREAD_FPU64_CONTEXT_Q4_Q5)];                                           \
+    ldp     q6,  q7,  [tmp, #(THREAD_FPU64_CONTEXT_Q6_Q7)];                                           \
+    ldp     q16, q17, [tmp, #(THREAD_FPU64_CONTEXT_Q16_Q17)];                                         \
+    ldp     q18, q19, [tmp, #(THREAD_FPU64_CONTEXT_Q18_Q19)];                                         \
+    ldp     q20, q21, [tmp, #(THREAD_FPU64_CONTEXT_Q20_Q21)];                                         \
+    ldp     q22, q23, [tmp, #(THREAD_FPU64_CONTEXT_Q22_Q23)];                                         \
+    ldp     q24, q25, [tmp, #(THREAD_FPU64_CONTEXT_Q24_Q25)];                                         \
+    ldp     q26, q27, [tmp, #(THREAD_FPU64_CONTEXT_Q26_Q27)];                                         \
+    ldp     q28, q29, [tmp, #(THREAD_FPU64_CONTEXT_Q28_Q29)];                                         \
+    ldp     q30, q31, [tmp, #(THREAD_FPU64_CONTEXT_Q30_Q31)];
+
+#define RESTORE_FPU64_ALL_REGISTERS(ctx, tmp) \
+    RESTORE_FPU64_CALLEE_SAVE_REGISTERS(ctx) \
+    RESTORE_FPU64_CALLER_SAVE_REGISTERS(tmp)
+
+#define RESTORE_FPU32_CALLEE_SAVE_REGISTERS(ctx)            \
+    ldp     q4,  q5,  [ctx, #(THREAD_CONTEXT_FPU32_Q4_Q5)]; \
+    ldp     q6,  q7,  [ctx, #(THREAD_CONTEXT_FPU32_Q6_Q7)];
+
+#define RESTORE_FPU32_CALLER_SAVE_REGISTERS(tmp)                                                      \
+    ldr     tmp, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_CALLER_SAVE_FPU_REGISTERS)]; \
+    ldp     q0,  q1,  [tmp, #(THREAD_FPU32_CONTEXT_Q0_Q1)];                                           \
+    ldp     q2,  q3,  [tmp, #(THREAD_FPU32_CONTEXT_Q2_Q3)];                                           \
+    ldp     q8,  q9,  [tmp, #(THREAD_FPU32_CONTEXT_Q8_Q9)];                                           \
+    ldp     q10, q11, [tmp, #(THREAD_FPU32_CONTEXT_Q10_Q11)];                                         \
+    ldp     q12, q13, [tmp, #(THREAD_FPU32_CONTEXT_Q12_Q13)];                                         \
+    ldp     q14, q15, [tmp, #(THREAD_FPU32_CONTEXT_Q14_Q15)];
+
+#define RESTORE_FPU32_ALL_REGISTERS(ctx, tmp) \
+    RESTORE_FPU32_CALLEE_SAVE_REGISTERS(ctx)  \
+    RESTORE_FPU32_CALLER_SAVE_REGISTERS(tmp)
+
+#define ENABLE_AND_RESTORE_FPU(ctx, xtmp1, xtmp2, wtmp1, wtmp2, label_32, label_done)         \
+    ENABLE_FPU(xtmp1)                                                                         \
+    GET_THREAD_CONTEXT_AND_RESTORE_FPCR_FPSR(ctx, xtmp1, xtmp2, wtmp1, wtmp2)                 \
+                                                                                              \
+    ldrb    wtmp1, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]; \
+    tbz     wtmp1, #(THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_FPU_64_BIT), label_32##f;             \
+                                                                                              \
+    RESTORE_FPU64_ALL_REGISTERS(ctx, xtmp1)                                                   \
+                                                                                              \
+    b       label_done##f;                                                                    \
+                                                                                              \
+label_32:                                                                                     \
+    RESTORE_FPU32_ALL_REGISTERS(ctx, xtmp1)                                                   \
+label_done:
+
+#define ENABLE_AND_RESTORE_FPU64(ctx, xtmp1, xtmp2, wtmp1, wtmp2)             \
+    ENABLE_FPU(xtmp1)                                                           \
+    GET_THREAD_CONTEXT_AND_RESTORE_FPCR_FPSR(ctx, xtmp1, xtmp2, wtmp1, wtmp2) \
+    RESTORE_FPU64_ALL_REGISTERS(ctx, xtmp1)
+
+#define ENABLE_AND_RESTORE_FPU32(ctx, xtmp1, xtmp2, wtmp1, wtmp2)             \
+    ENABLE_FPU(xtmp1)                                                           \
+    GET_THREAD_CONTEXT_AND_RESTORE_FPCR_FPSR(ctx, xtmp1, xtmp2, wtmp1, wtmp2) \
+    RESTORE_FPU32_ALL_REGISTERS(ctx, xtmp1)
+
+#define ERET_WITH_SPECULATION_BARRIER \
+    eret;                             \
+    dsb nsh;                          \
+    isb

Source/Atmosphere/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_assembly_offsets.h

@@ -0,0 +1,263 @@
+/*
+ * Copyright (c) Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_build_config.hpp>
+
+/* TODO: Different header for this? */
+#define AMS_KERN_NUM_SUPERVISOR_CALLS 0xC0
+
+/* ams::kern::KThread, https://github.com/Atmosphere-NX/Atmosphere/blob/master/libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp */
+#define THREAD_KERNEL_STACK_TOP 0x280
+
+/* ams::kern::KThread::StackParameters, https://github.com/Atmosphere-NX/Atmosphere/blob/master/libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp */
+#define THREAD_STACK_PARAMETERS_SIZE                      0x140
+#define THREAD_STACK_PARAMETERS_SVC_PERMISSION            0x00
+#define THREAD_STACK_PARAMETERS_CALLER_SAVE_FPU_REGISTERS 0x18
+#define THREAD_STACK_PARAMETERS_CUR_THREAD                0x20
+#define THREAD_STACK_PARAMETERS_DISABLE_COUNT             0x28
+#define THREAD_STACK_PARAMETERS_DPC_FLAGS                 0x2A
+#define THREAD_STACK_PARAMETERS_CURRENT_SVC_ID            0x2B
+#define THREAD_STACK_PARAMETERS_RESERVED_2C               0x2C
+#define THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS           0x2D
+#define THREAD_STACK_PARAMETERS_IS_PINNED                 0x2E
+#define THREAD_STACK_PARAMETERS_RESERVED_2F               0x2F
+#define THREAD_STACK_PARAMETERS_RESERVED_30               0x30
+#define THREAD_STACK_PARAMETERS_THREAD_CONTEXT            0x40
+
+#define THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_CALLING_SVC                    (0)
+#define THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_IN_EXCEPTION_HANDLER           (1)
+#define THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_FPU_CONTEXT_RESTORE_NEEDED     (2)
+#define THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_FPU_64_BIT                     (3)
+#define THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_IN_USERMODE_EXCEPTION_HANDLER  (4)
+#define THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_IN_CACHE_MAINTENANCE_OPERATION (5)
+#define THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_IN_TLB_MAINTENANCE_OPERATION   (6)
+
+#if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP)
+#define THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_HARDWARE_SINGLE_STEP           (7)
+#endif
+
+#define THREAD_EXCEPTION_FLAG_IS_CALLING_SVC                    (1 << THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_CALLING_SVC)
+#define THREAD_EXCEPTION_FLAG_IS_IN_EXCEPTION_HANDLER           (1 << THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_IN_EXCEPTION_HANDLER)
+#define THREAD_EXCEPTION_FLAG_IS_FPU_CONTEXT_RESTORE_NEEDED     (1 << THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_FPU_CONTEXT_RESTORE_NEEDED)
+#define THREAD_EXCEPTION_FLAG_IS_FPU_64_BIT                     (1 << THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_FPU_64_BIT)
+#define THREAD_EXCEPTION_FLAG_IS_IN_USERMODE_EXCEPTION_HANDLER  (1 << THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_IN_USERMODE_EXCEPTION_HANDLER)
+#define THREAD_EXCEPTION_FLAG_IS_IN_CACHE_MAINTENANCE_OPERATION (1 << THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_IN_CACHE_MAINTENANCE_OPERATION)
+#define THREAD_EXCEPTION_FLAG_IS_IN_TLB_MAINTENANCE_OPERATION   (1 << THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_IN_TLB_MAINTENANCE_OPERATION)
+
+#if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP)
+#define THREAD_EXCEPTION_FLAG_IS_HARDWARE_SINGLE_STEP           (1 << THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_HARDWARE_SINGLE_STEP)
+#endif
+
+/* ams::kern::arch::arm64::KThreadContext, https://github.com/Atmosphere-NX/Atmosphere/blob/master/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_thread_context.hpp */
+#define THREAD_CONTEXT_SIZE               0x100
+#define THREAD_CONTEXT_CPU_REGISTERS      0x000
+#define THREAD_CONTEXT_X19                0x000
+#define THREAD_CONTEXT_X20                0x008
+#define THREAD_CONTEXT_X21                0x010
+#define THREAD_CONTEXT_X22                0x018
+#define THREAD_CONTEXT_X23                0x020
+#define THREAD_CONTEXT_X24                0x028
+#define THREAD_CONTEXT_X25                0x030
+#define THREAD_CONTEXT_X26                0x038
+#define THREAD_CONTEXT_X27                0x040
+#define THREAD_CONTEXT_X28                0x048
+#define THREAD_CONTEXT_X29                0x050
+#define THREAD_CONTEXT_LR                 0x058
+#define THREAD_CONTEXT_SP                 0x060
+#define THREAD_CONTEXT_FPCR               0x068
+#define THREAD_CONTEXT_FPSR               0x06C
+#define THREAD_CONTEXT_FPU_REGISTERS      0x070
+#define THREAD_CONTEXT_LOCKED             0x0F0
+
+#define THREAD_CONTEXT_X19_X20      THREAD_CONTEXT_X19
+#define THREAD_CONTEXT_X21_X22      THREAD_CONTEXT_X21
+#define THREAD_CONTEXT_X23_X24      THREAD_CONTEXT_X23
+#define THREAD_CONTEXT_X25_X26      THREAD_CONTEXT_X25
+#define THREAD_CONTEXT_X27_X28      THREAD_CONTEXT_X27
+#define THREAD_CONTEXT_X29_X30      THREAD_CONTEXT_X29
+#define THREAD_CONTEXT_LR_SP        THREAD_CONTEXT_LR
+#define THREAD_CONTEXT_SP_FPCR_FPSR THREAD_CONTEXT_SP
+
+#define THREAD_CONTEXT_FPCR_FPSR    THREAD_CONTEXT_FPCR
+
+#define THREAD_CONTEXT_FPU64_Q8  (THREAD_CONTEXT_FPU_REGISTERS + 0x00)
+#define THREAD_CONTEXT_FPU64_Q9  (THREAD_CONTEXT_FPU_REGISTERS + 0x10)
+#define THREAD_CONTEXT_FPU64_Q10 (THREAD_CONTEXT_FPU_REGISTERS + 0x20)
+#define THREAD_CONTEXT_FPU64_Q11 (THREAD_CONTEXT_FPU_REGISTERS + 0x30)
+#define THREAD_CONTEXT_FPU64_Q12 (THREAD_CONTEXT_FPU_REGISTERS + 0x40)
+#define THREAD_CONTEXT_FPU64_Q13 (THREAD_CONTEXT_FPU_REGISTERS + 0x50)
+#define THREAD_CONTEXT_FPU64_Q14 (THREAD_CONTEXT_FPU_REGISTERS + 0x60)
+#define THREAD_CONTEXT_FPU64_Q15 (THREAD_CONTEXT_FPU_REGISTERS + 0x70)
+
+#define THREAD_CONTEXT_FPU64_Q8_Q9   THREAD_CONTEXT_FPU64_Q8
+#define THREAD_CONTEXT_FPU64_Q10_Q11 THREAD_CONTEXT_FPU64_Q10
+#define THREAD_CONTEXT_FPU64_Q12_Q13 THREAD_CONTEXT_FPU64_Q12
+#define THREAD_CONTEXT_FPU64_Q14_Q15 THREAD_CONTEXT_FPU64_Q14
+
+#define THREAD_CONTEXT_FPU32_Q4 (THREAD_CONTEXT_FPU_REGISTERS + 0x00)
+#define THREAD_CONTEXT_FPU32_Q5 (THREAD_CONTEXT_FPU_REGISTERS + 0x10)
+#define THREAD_CONTEXT_FPU32_Q6 (THREAD_CONTEXT_FPU_REGISTERS + 0x20)
+#define THREAD_CONTEXT_FPU32_Q7 (THREAD_CONTEXT_FPU_REGISTERS + 0x30)
+
+#define THREAD_CONTEXT_FPU32_Q4_Q5 THREAD_CONTEXT_FPU32_Q4
+#define THREAD_CONTEXT_FPU32_Q6_Q7 THREAD_CONTEXT_FPU32_Q6
+
+#define THREAD_FPU64_CONTEXT_Q0  0x000
+#define THREAD_FPU64_CONTEXT_Q1  0x010
+#define THREAD_FPU64_CONTEXT_Q2  0x020
+#define THREAD_FPU64_CONTEXT_Q3  0x030
+#define THREAD_FPU64_CONTEXT_Q4  0x040
+#define THREAD_FPU64_CONTEXT_Q5  0x050
+#define THREAD_FPU64_CONTEXT_Q6  0x060
+#define THREAD_FPU64_CONTEXT_Q7  0x070
+#define THREAD_FPU64_CONTEXT_Q16 0x080
+#define THREAD_FPU64_CONTEXT_Q17 0x090
+#define THREAD_FPU64_CONTEXT_Q18 0x0A0
+#define THREAD_FPU64_CONTEXT_Q19 0x0B0
+#define THREAD_FPU64_CONTEXT_Q20 0x0C0
+#define THREAD_FPU64_CONTEXT_Q21 0x0D0
+#define THREAD_FPU64_CONTEXT_Q22 0x0E0
+#define THREAD_FPU64_CONTEXT_Q23 0x0F0
+#define THREAD_FPU64_CONTEXT_Q24 0x100
+#define THREAD_FPU64_CONTEXT_Q25 0x110
+#define THREAD_FPU64_CONTEXT_Q26 0x120
+#define THREAD_FPU64_CONTEXT_Q27 0x130
+#define THREAD_FPU64_CONTEXT_Q28 0x140
+#define THREAD_FPU64_CONTEXT_Q29 0x150
+#define THREAD_FPU64_CONTEXT_Q30 0x160
+#define THREAD_FPU64_CONTEXT_Q31 0x170
+
+#define THREAD_FPU64_CONTEXT_Q0_Q1   THREAD_FPU64_CONTEXT_Q0
+#define THREAD_FPU64_CONTEXT_Q2_Q3   THREAD_FPU64_CONTEXT_Q2
+#define THREAD_FPU64_CONTEXT_Q4_Q5   THREAD_FPU64_CONTEXT_Q4
+#define THREAD_FPU64_CONTEXT_Q6_Q7   THREAD_FPU64_CONTEXT_Q6
+#define THREAD_FPU64_CONTEXT_Q16_Q17 THREAD_FPU64_CONTEXT_Q16
+#define THREAD_FPU64_CONTEXT_Q18_Q19 THREAD_FPU64_CONTEXT_Q18
+#define THREAD_FPU64_CONTEXT_Q20_Q21 THREAD_FPU64_CONTEXT_Q20
+#define THREAD_FPU64_CONTEXT_Q22_Q23 THREAD_FPU64_CONTEXT_Q22
+#define THREAD_FPU64_CONTEXT_Q24_Q25 THREAD_FPU64_CONTEXT_Q24
+#define THREAD_FPU64_CONTEXT_Q26_Q27 THREAD_FPU64_CONTEXT_Q26
+#define THREAD_FPU64_CONTEXT_Q28_Q29 THREAD_FPU64_CONTEXT_Q28
+#define THREAD_FPU64_CONTEXT_Q30_Q31 THREAD_FPU64_CONTEXT_Q30
+
+#define THREAD_FPU32_CONTEXT_Q0  0x000
+#define THREAD_FPU32_CONTEXT_Q1  0x010
+#define THREAD_FPU32_CONTEXT_Q2  0x020
+#define THREAD_FPU32_CONTEXT_Q3  0x030
+#define THREAD_FPU32_CONTEXT_Q8  0x040
+#define THREAD_FPU32_CONTEXT_Q9  0x050
+#define THREAD_FPU32_CONTEXT_Q10 0x060
+#define THREAD_FPU32_CONTEXT_Q11 0x070
+#define THREAD_FPU32_CONTEXT_Q12 0x080
+#define THREAD_FPU32_CONTEXT_Q13 0x090
+#define THREAD_FPU32_CONTEXT_Q14 0x0A0
+#define THREAD_FPU32_CONTEXT_Q15 0x0B0
+
+#define THREAD_FPU32_CONTEXT_Q0_Q1   THREAD_FPU32_CONTEXT_Q0
+#define THREAD_FPU32_CONTEXT_Q2_Q3   THREAD_FPU32_CONTEXT_Q2
+#define THREAD_FPU32_CONTEXT_Q8_Q9   THREAD_FPU32_CONTEXT_Q8
+#define THREAD_FPU32_CONTEXT_Q10_Q11 THREAD_FPU32_CONTEXT_Q10
+#define THREAD_FPU32_CONTEXT_Q12_Q13 THREAD_FPU32_CONTEXT_Q12
+#define THREAD_FPU32_CONTEXT_Q14_Q15 THREAD_FPU32_CONTEXT_Q14
+
+/* ams::kern::arch::arm64::KExceptionContext, https://github.com/Atmosphere-NX/Atmosphere/blob/master/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_exception_context.hpp */
+#define EXCEPTION_CONTEXT_SIZE  0x120
+#define EXCEPTION_CONTEXT_X0    0x000
+#define EXCEPTION_CONTEXT_X1    0x008
+#define EXCEPTION_CONTEXT_X2    0x010
+#define EXCEPTION_CONTEXT_X3    0x018
+#define EXCEPTION_CONTEXT_X4    0x020
+#define EXCEPTION_CONTEXT_X5    0x028
+#define EXCEPTION_CONTEXT_X6    0x030
+#define EXCEPTION_CONTEXT_X7    0x038
+#define EXCEPTION_CONTEXT_X8    0x040
+#define EXCEPTION_CONTEXT_X9    0x048
+#define EXCEPTION_CONTEXT_X10   0x050
+#define EXCEPTION_CONTEXT_X11   0x058
+#define EXCEPTION_CONTEXT_X12   0x060
+#define EXCEPTION_CONTEXT_X13   0x068
+#define EXCEPTION_CONTEXT_X14   0x070
+#define EXCEPTION_CONTEXT_X15   0x078
+#define EXCEPTION_CONTEXT_X16   0x080
+#define EXCEPTION_CONTEXT_X17   0x088
+#define EXCEPTION_CONTEXT_X18   0x090
+#define EXCEPTION_CONTEXT_X19   0x098
+#define EXCEPTION_CONTEXT_X20   0x0A0
+#define EXCEPTION_CONTEXT_X21   0x0A8
+#define EXCEPTION_CONTEXT_X22   0x0B0
+#define EXCEPTION_CONTEXT_X23   0x0B8
+#define EXCEPTION_CONTEXT_X24   0x0C0
+#define EXCEPTION_CONTEXT_X25   0x0C8
+#define EXCEPTION_CONTEXT_X26   0x0D0
+#define EXCEPTION_CONTEXT_X27   0x0D8
+#define EXCEPTION_CONTEXT_X28   0x0E0
+#define EXCEPTION_CONTEXT_X29   0x0E8
+#define EXCEPTION_CONTEXT_X30   0x0F0
+#define EXCEPTION_CONTEXT_SP    0x0F8
+#define EXCEPTION_CONTEXT_PC    0x100
+#define EXCEPTION_CONTEXT_PSR   0x108
+#define EXCEPTION_CONTEXT_TPIDR 0x110
+
+#define EXCEPTION_CONTEXT_X0_X1   EXCEPTION_CONTEXT_X0
+#define EXCEPTION_CONTEXT_X2_X3   EXCEPTION_CONTEXT_X2
+#define EXCEPTION_CONTEXT_X4_X5   EXCEPTION_CONTEXT_X4
+#define EXCEPTION_CONTEXT_X6_X7   EXCEPTION_CONTEXT_X6
+#define EXCEPTION_CONTEXT_X8_X9   EXCEPTION_CONTEXT_X8
+#define EXCEPTION_CONTEXT_X10_X11 EXCEPTION_CONTEXT_X10
+#define EXCEPTION_CONTEXT_X12_X13 EXCEPTION_CONTEXT_X12
+#define EXCEPTION_CONTEXT_X14_X15 EXCEPTION_CONTEXT_X14
+#define EXCEPTION_CONTEXT_X16_X17 EXCEPTION_CONTEXT_X16
+#define EXCEPTION_CONTEXT_X18_X19 EXCEPTION_CONTEXT_X18
+#define EXCEPTION_CONTEXT_X20_X21 EXCEPTION_CONTEXT_X20
+#define EXCEPTION_CONTEXT_X22_X23 EXCEPTION_CONTEXT_X22
+#define EXCEPTION_CONTEXT_X24_X25 EXCEPTION_CONTEXT_X24
+#define EXCEPTION_CONTEXT_X26_X27 EXCEPTION_CONTEXT_X26
+#define EXCEPTION_CONTEXT_X28_X29 EXCEPTION_CONTEXT_X28
+#define EXCEPTION_CONTEXT_X30_SP  EXCEPTION_CONTEXT_X30
+#define EXCEPTION_CONTEXT_PC_PSR  EXCEPTION_CONTEXT_PC
+
+#define EXCEPTION_CONTEXT_X9_X10    EXCEPTION_CONTEXT_X9
+#define EXCEPTION_CONTEXT_X19_X20   EXCEPTION_CONTEXT_X19
+#define EXCEPTION_CONTEXT_X21_X22   EXCEPTION_CONTEXT_X21
+#define EXCEPTION_CONTEXT_X23_X24   EXCEPTION_CONTEXT_X23
+#define EXCEPTION_CONTEXT_X25_X26   EXCEPTION_CONTEXT_X25
+#define EXCEPTION_CONTEXT_X27_X28   EXCEPTION_CONTEXT_X27
+#define EXCEPTION_CONTEXT_X29_X30   EXCEPTION_CONTEXT_X29
+#define EXCEPTION_CONTEXT_SP_PC     EXCEPTION_CONTEXT_SP
+#define EXCEPTION_CONTEXT_PSR_TPIDR EXCEPTION_CONTEXT_PSR
+
+/* ams::svc::arch::arm64::ThreadLocalRegion, https://github.com/Atmosphere-NX/Atmosphere/blob/master/libraries/libvapours/include/vapours/svc/arch/arm64/svc_thread_local_region.hpp */
+#define THREAD_LOCAL_REGION_MESSAGE_BUFFER 0x000
+#define THREAD_LOCAL_REGION_DISABLE_COUNT  0x100
+#define THREAD_LOCAL_REGION_INTERRUPT_FLAG 0x102
+#define THREAD_LOCAL_REGION_SIZE           0x200
+
+/* ams::kern::init::KInitArguments, https://github.com/Atmosphere-NX/Atmosphere/blob/master/libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_arguments.hpp */
+#define INIT_ARGUMENTS_SIZE            0x28
+#define INIT_ARGUMENTS_CPUACTLR        0x00
+#define INIT_ARGUMENTS_CPUECTLR        0x08
+#define INIT_ARGUMENTS_SP              0x10
+#define INIT_ARGUMENTS_ENTRYPOINT      0x18
+#define INIT_ARGUMENTS_ARGUMENT        0x20
+
+/* ams::kern::KScheduler (::SchedulingState), https://github.com/Atmosphere-NX/Atmosphere/blob/master/libraries/libmesosphere/include/mesosphere/kern_k_scheduler.hpp */
+/* NOTE: Due to constraints on ldarb relative offsets, KSCHEDULER_NEEDS_SCHEDULING cannot trivially be changed, and will require assembly edits. */
+#define KSCHEDULER_NEEDS_SCHEDULING        0x00
+#define KSCHEDULER_INTERRUPT_TASK_RUNNABLE 0x01
+#define KSCHEDULER_HIGHEST_PRIORITY_THREAD 0x18
+#define KSCHEDULER_IDLE_THREAD_STACK       0x20
+#define KSCHEDULER_PREVIOUS_THREAD         0x28
+#define KSCHEDULER_INTERRUPT_TASK_MANAGER  0x30

Source/Atmosphere/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu.hpp

@@ -0,0 +1,271 @@
+/*
+ * Copyright (c) Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <vapours.hpp>
+#include <mesosphere/arch/arm64/kern_cpu_system_registers.hpp>
+#include <mesosphere/kern_select_userspace_memory_access.hpp>
+
+namespace ams::kern::arch::arm64::cpu {
+
+#if defined(ATMOSPHERE_CPU_ARM_CORTEX_A57) || defined(ATMOSPHERE_CPU_ARM_CORTEX_A53)
+    constexpr inline size_t InstructionCacheLineSize = 0x40;
+    constexpr inline size_t DataCacheLineSize        = 0x40;
+    constexpr inline size_t NumPerformanceCounters   = 6;
+#else
+    #error "Unknown CPU for cache line sizes"
+#endif
+
+#if defined(ATMOSPHERE_BOARD_NINTENDO_NX)
+    constexpr inline size_t NumCores = 4;
+#elif defined(ATMOSPHERE_BOARD_QEMU_VIRT)
+    constexpr inline size_t NumCores = 4;
+#else
+    #error "Unknown Board for cpu::NumCores"
+#endif
+
+    constexpr inline u32 El0Aarch64PsrMask = 0xF0000000;
+    constexpr inline u32 El0Aarch32PsrMask = 0xFE0FFE20;
+
+    /* Initialization. */
+    NOINLINE void InitializeInterruptThreads(s32 core_id);
+
+    /* Helpers for managing memory state. */
+    ALWAYS_INLINE void DataSynchronizationBarrier() {
+        __asm__ __volatile__("dsb sy" ::: "memory");
+    }
+
+    ALWAYS_INLINE void DataSynchronizationBarrierInnerShareable() {
+        __asm__ __volatile__("dsb ish" ::: "memory");
+    }
+
+    ALWAYS_INLINE void DataSynchronizationBarrierInnerShareableStore() {
+        __asm__ __volatile__("dsb ishst" ::: "memory");
+    }
+
+    ALWAYS_INLINE void DataMemoryBarrier() {
+        __asm__ __volatile__("dmb sy" ::: "memory");
+    }
+
+    ALWAYS_INLINE void DataMemoryBarrierInnerShareable() {
+        __asm__ __volatile__("dmb ish" ::: "memory");
+    }
+
+    ALWAYS_INLINE void DataMemoryBarrierInnerShareableStore() {
+        __asm__ __volatile__("dmb ishst" ::: "memory");
+    }
+
+    ALWAYS_INLINE void InstructionMemoryBarrier() {
+        __asm__ __volatile__("isb" ::: "memory");
+    }
+
+    ALWAYS_INLINE void EnsureInstructionConsistency() {
+        DataSynchronizationBarrierInnerShareable();
+        InstructionMemoryBarrier();
+    }
+
+    ALWAYS_INLINE void EnsureInstructionConsistencyFullSystem() {
+        DataSynchronizationBarrier();
+        InstructionMemoryBarrier();
+    }
+
+    ALWAYS_INLINE void Yield() {
+        __asm__ __volatile__("yield" ::: "memory");
+    }
+
+    ALWAYS_INLINE void SwitchProcess(u64 ttbr, u32 proc_id) {
+        SetTtbr0El1(ttbr);
+        ContextIdRegisterAccessor(0).SetProcId(proc_id).Store();
+        InstructionMemoryBarrier();
+    }
+
+    /* Performance counter helpers. */
+    ALWAYS_INLINE u64 GetCycleCounter() {
+        return cpu::GetPmcCntrEl0();
+    }
+
+    ALWAYS_INLINE u32 GetPerformanceCounter(s32 n) {
+        u64 counter = 0;
+        if (n < static_cast<s32>(NumPerformanceCounters)) {
+            switch (n) {
+                case 0:
+                    counter = cpu::GetPmevCntr0El0();
+                    break;
+                case 1:
+                    counter = cpu::GetPmevCntr1El0();
+                    break;
+                case 2:
+                    counter = cpu::GetPmevCntr2El0();
+                    break;
+                case 3:
+                    counter = cpu::GetPmevCntr3El0();
+                    break;
+                case 4:
+                    counter = cpu::GetPmevCntr4El0();
+                    break;
+                case 5:
+                    counter = cpu::GetPmevCntr5El0();
+                    break;
+                default:
+                    break;
+            }
+        }
+        return static_cast<u32>(counter);
+    }
+
+    /* Helper for address access. */
+    ALWAYS_INLINE bool GetPhysicalAddressWritable(KPhysicalAddress *out, KVirtualAddress addr, bool privileged = false) {
+        const uintptr_t va = GetInteger(addr);
+
+        if (privileged) {
+            __asm__ __volatile__("at s1e1w, %[va]" :: [va]"r"(va) : "memory");
+        } else {
+            __asm__ __volatile__("at s1e0w, %[va]" :: [va]"r"(va) : "memory");
+        }
+        InstructionMemoryBarrier();
+
+        u64 par = GetParEl1();
+
+        if (par & 0x1) {
+            return false;
+        }
+
+        if (out) {
+            *out = KPhysicalAddress((par & 0xFFFFFFFFF000ull) | (va & 0xFFFull));
+        }
+        return true;
+    }
+
+    ALWAYS_INLINE bool GetPhysicalAddressReadable(KPhysicalAddress *out, KVirtualAddress addr, bool privileged = false) {
+        const uintptr_t va = GetInteger(addr);
+
+        if (privileged) {
+            __asm__ __volatile__("at s1e1r, %[va]" :: [va]"r"(va) : "memory");
+        } else {
+            __asm__ __volatile__("at s1e0r, %[va]" :: [va]"r"(va) : "memory");
+        }
+        InstructionMemoryBarrier();
+
+        u64 par = GetParEl1();
+
+        if (par & 0x1) {
+            return false;
+        }
+
+        if (out) {
+            *out = KPhysicalAddress((par & 0xFFFFFFFFF000ull) | (va & 0xFFFull));
+        }
+        return true;
+    }
+
+    ALWAYS_INLINE bool CanAccessAtomic(KProcessAddress addr, bool privileged = false) {
+        const uintptr_t va = GetInteger(addr);
+
+        if (privileged) {
+            __asm__ __volatile__("at s1e1w, %[va]" :: [va]"r"(va) : "memory");
+        } else {
+            __asm__ __volatile__("at s1e0w, %[va]" :: [va]"r"(va) : "memory");
+        }
+        InstructionMemoryBarrier();
+
+        u64 par = GetParEl1();
+
+        if (par & 0x1) {
+            return false;
+        }
+
+        return (par >> (BITSIZEOF(par) - BITSIZEOF(u8))) == 0xFF;
+    }
+
+    ALWAYS_INLINE void StoreDataCacheForInitArguments(const void *addr, size_t size) {
+        const uintptr_t start = util::AlignDown(reinterpret_cast<uintptr_t>(addr), DataCacheLineSize);
+        for (size_t stored = 0; stored < size; stored += cpu::DataCacheLineSize) {
+            __asm__ __volatile__("dc cvac, %[cur]" :: [cur]"r"(start + stored) : "memory");
+        }
+        DataSynchronizationBarrier();
+    }
+
+    /* Synchronization helpers. */
+    NOINLINE void SynchronizeAllCores();
+    void SynchronizeCores(u64 core_mask);
+
+    /* Cache management helpers. */
+    void StoreCacheForInit(void *addr, size_t size);
+
+    void FlushEntireDataCache();
+
+    Result InvalidateDataCache(void *addr, size_t size);
+    Result StoreDataCache(const void *addr, size_t size);
+    Result FlushDataCache(const void *addr, size_t size);
+
+    void InvalidateEntireInstructionCache();
+
+    void ClearPageToZeroImpl(void *);
+
+    ALWAYS_INLINE void ClearPageToZero(void * const page) {
+        MESOSPHERE_ASSERT(util::IsAligned(reinterpret_cast<uintptr_t>(page), PageSize));
+        MESOSPHERE_ASSERT(page != nullptr);
+
+        ClearPageToZeroImpl(page);
+    }
+
+    ALWAYS_INLINE void InvalidateTlbByAsid(u32 asid) {
+        const u64 value = (static_cast<u64>(asid) << 48);
+        __asm__ __volatile__("tlbi aside1is, %[value]" :: [value]"r"(value) : "memory");
+        EnsureInstructionConsistency();
+    }
+
+    ALWAYS_INLINE void InvalidateTlbByAsidAndVa(u32 asid, KProcessAddress virt_addr) {
+        const u64 value = (static_cast<u64>(asid) << 48) | ((GetInteger(virt_addr) >> 12) & 0xFFFFFFFFFFFul);
+        __asm__ __volatile__("tlbi aside1is, %[value]" :: [value]"r"(value) : "memory");
+        EnsureInstructionConsistency();
+    }
+
+    ALWAYS_INLINE void InvalidateEntireTlb() {
+        __asm__ __volatile__("tlbi vmalle1is" ::: "memory");
+        EnsureInstructionConsistency();
+    }
+
+    ALWAYS_INLINE void InvalidateEntireTlbDataOnly() {
+        __asm__ __volatile__("tlbi vmalle1is" ::: "memory");
+        DataSynchronizationBarrierInnerShareable();
+    }
+
+    ALWAYS_INLINE void InvalidateTlbByVaDataOnly(KProcessAddress virt_addr) {
+        const u64 value = ((GetInteger(virt_addr) >> 12) & 0xFFFFFFFFFFFul);
+        __asm__ __volatile__("tlbi vaae1is, %[value]" :: [value]"r"(value) : "memory");
+        DataSynchronizationBarrierInnerShareable();
+    }
+
+    ALWAYS_INLINE uintptr_t GetCurrentThreadPointerValue() {
+        register uintptr_t x18 asm("x18");
+        __asm__ __volatile__("" : [x18]"=r"(x18));
+        return x18;
+    }
+
+    ALWAYS_INLINE void SetCurrentThreadPointerValue(uintptr_t value) {
+        register uintptr_t x18 asm("x18") = value;
+        __asm__ __volatile__("":: [x18]"r"(x18));
+    }
+
+    ALWAYS_INLINE void SetExceptionThreadStackTop(uintptr_t top) {
+        cpu::SetCntvCvalEl0(top);
+    }
+
+    ALWAYS_INLINE void SwitchThreadLocalRegion(uintptr_t tlr) {
+        cpu::SetTpidrRoEl0(tlr);
+    }
+
+}

Source/Atmosphere/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu_system_registers.hpp

@@ -0,0 +1,493 @@
+/*
+ * Copyright (c) Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <vapours.hpp>
+
+namespace ams::kern::arch::arm64::cpu {
+
+    #define MESOSPHERE_CPU_GET_SYSREG(name)                                            \
+        ({                                                                             \
+            u64 temp_value;                                                            \
+            __asm__ __volatile__("mrs %0, " #name "" : "=&r"(temp_value) :: "memory"); \
+            temp_value;                                                                \
+        })
+
+    #define MESOSPHERE_CPU_SET_SYSREG(name, value)                                    \
+        ({                                                                            \
+            __asm__ __volatile__("msr " #name ", %0" :: "r"(value) : "memory", "cc"); \
+        })
+
+    #define MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(name, reg_name)                          \
+    ALWAYS_INLINE void Set##name(u64 value) { MESOSPHERE_CPU_SET_SYSREG(reg_name, value); } \
+    ALWAYS_INLINE u64  Get##name()          { return MESOSPHERE_CPU_GET_SYSREG(reg_name); }
+
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(Ttbr0El1, ttbr0_el1)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(Ttbr1El1, ttbr1_el1)
+
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(TcrEl1, tcr_el1)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(MairEl1, mair_el1)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(TpidrEl1, tpidr_el1)
+
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(VbarEl1, vbar_el1)
+
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(FarEl1, far_el1)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(ParEl1, par_el1)
+
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(SctlrEl1, sctlr_el1)
+
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(CpuActlrEl1, s3_1_c15_c2_0)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(CpuEctlrEl1, s3_1_c15_c2_1)
+
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(CsselrEl1, csselr_el1)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(CcsidrEl1, ccsidr_el1)
+
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(OslarEl1, oslar_el1)
+
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(TpidrEl0,   tpidr_el0)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(TpidrRoEl0, tpidrro_el0)
+
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(ElrEl1,   elr_el1)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(EsrEl1,   esr_el1)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(SpsrEl1,  spsr_el1)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(Afsr0El1, afsr0_el1)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(Afsr1El1, afsr1_el1)
+
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(MdscrEl1, mdscr_el1)
+
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(CpacrEl1, cpacr_el1)
+
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(ContextidrEl1, contextidr_el1)
+
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(CntkCtlEl1,  cntkctl_el1)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(CntpCtlEl0,  cntp_ctl_el0)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(CntpCvalEl0, cntp_cval_el0)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(CntvCvalEl0, cntv_cval_el0)
+
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(Daif, daif)
+
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(SpEl0, sp_el0)
+
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(IdAa64Dfr0El1, id_aa64dfr0_el1)
+
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(PmcrEl0,       pmcr_el0)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(PmUserEnrEl0,  pmuserenr_el0)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(PmcCntrEl0,    pmccntr_el0)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(PmSelrEl0,     pmselr_el0)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(PmcCfiltrEl0,  pmccfiltr_el0)
+
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(PmIntEnSetEl1, pmintenset_el1)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(PmCntEnSetEl0, pmcntenset_el0)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(PmOvsSetEl0,   pmovsset_el0)
+
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(PmIntEnClrEl1, pmintenclr_el1)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(PmCntEnClrEl0, pmcntenclr_el0)
+    MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(PmOvsClrEl0,   pmovsclr_el0)
+
+    #define FOR_I_IN_0_TO_30(HANDLER, ...)                                                                              \
+        HANDLER(0,  ## __VA_ARGS__) HANDLER(1,  ## __VA_ARGS__) HANDLER(2,  ## __VA_ARGS__) HANDLER(3,  ## __VA_ARGS__) \
+        HANDLER(4,  ## __VA_ARGS__) HANDLER(5,  ## __VA_ARGS__) HANDLER(6,  ## __VA_ARGS__) HANDLER(7,  ## __VA_ARGS__) \
+        HANDLER(8,  ## __VA_ARGS__) HANDLER(9,  ## __VA_ARGS__) HANDLER(10, ## __VA_ARGS__) HANDLER(11, ## __VA_ARGS__) \
+        HANDLER(12, ## __VA_ARGS__) HANDLER(13, ## __VA_ARGS__) HANDLER(14, ## __VA_ARGS__) HANDLER(15, ## __VA_ARGS__) \
+        HANDLER(16, ## __VA_ARGS__) HANDLER(17, ## __VA_ARGS__) HANDLER(18, ## __VA_ARGS__) HANDLER(19, ## __VA_ARGS__) \
+        HANDLER(20, ## __VA_ARGS__) HANDLER(21, ## __VA_ARGS__) HANDLER(22, ## __VA_ARGS__) HANDLER(23, ## __VA_ARGS__) \
+        HANDLER(24, ## __VA_ARGS__) HANDLER(25, ## __VA_ARGS__) HANDLER(26, ## __VA_ARGS__) HANDLER(27, ## __VA_ARGS__) \
+        HANDLER(28, ## __VA_ARGS__) HANDLER(29, ## __VA_ARGS__) HANDLER(30, ## __VA_ARGS__)
+
+    #define MESOSPHERE_CPU_DEFINE_PMEV_ACCESSORS(ID, ...)                               \
+        MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(PmevCntr##ID##El0,  pmevcntr##ID##_el0)  \
+        MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(PmevTyper##ID##El0, pmevtyper##ID##_el0)
+
+    FOR_I_IN_0_TO_30(MESOSPHERE_CPU_DEFINE_PMEV_ACCESSORS)
+
+    #undef MESOSPHERE_CPU_DEFINE_PMEV_ACCESSORS
+    #undef FOR_I_IN_0_TO_30
+
+    #define FOR_I_IN_0_TO_15(HANDLER, ...)                                                                              \
+        HANDLER(0,  ## __VA_ARGS__) HANDLER(1,  ## __VA_ARGS__) HANDLER(2,  ## __VA_ARGS__) HANDLER(3,  ## __VA_ARGS__) \
+        HANDLER(4,  ## __VA_ARGS__) HANDLER(5,  ## __VA_ARGS__) HANDLER(6,  ## __VA_ARGS__) HANDLER(7,  ## __VA_ARGS__) \
+        HANDLER(8,  ## __VA_ARGS__) HANDLER(9,  ## __VA_ARGS__) HANDLER(10, ## __VA_ARGS__) HANDLER(11, ## __VA_ARGS__) \
+        HANDLER(12, ## __VA_ARGS__) HANDLER(13, ## __VA_ARGS__) HANDLER(14, ## __VA_ARGS__) HANDLER(15, ## __VA_ARGS__)
+
+    #define MESOSPHERE_CPU_DEFINE_DBG_SYSREG_ACCESSORS(ID, ...)                     \
+        MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(DbgWcr##ID##El1, dbgwcr##ID##_el1)   \
+        MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(DbgWvr##ID##El1, dbgwvr##ID##_el1)   \
+        MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(DbgBcr##ID##El1, dbgbcr##ID##_el1)   \
+        MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(DbgBvr##ID##El1, dbgbvr##ID##_el1)
+
+    FOR_I_IN_0_TO_15(MESOSPHERE_CPU_DEFINE_DBG_SYSREG_ACCESSORS)
+
+    #undef MESOSPHERE_CPU_DEFINE_DBG_SYSREG_ACCESSORS
+
+    /* Base class for register accessors. */
+    class GenericRegisterAccessorBase {
+        NON_COPYABLE(GenericRegisterAccessorBase);
+        NON_MOVEABLE(GenericRegisterAccessorBase);
+        private:
+            u64 m_value;
+        public:
+            constexpr ALWAYS_INLINE GenericRegisterAccessorBase(u64 v) : m_value(v) { /* ... */ }
+        protected:
+            constexpr ALWAYS_INLINE u64 GetValue() const {
+                return m_value;
+            }
+
+            constexpr ALWAYS_INLINE u64 GetBits(size_t offset, size_t count) const {
+                return (m_value >> offset) & ((1ul << count) - 1);
+            }
+
+            constexpr ALWAYS_INLINE void SetBits(size_t offset, size_t count, u64 value) {
+                const u64 mask = ((1ul << count) - 1) << offset;
+                m_value &= ~mask;
+                m_value |= (value & (mask >> offset)) << offset;
+            }
+
+            constexpr ALWAYS_INLINE void SetBitsDirect(size_t offset, size_t count, u64 value) {
+                const u64 mask = ((1ul << count) - 1) << offset;
+                m_value &= ~mask;
+                m_value |= (value & mask);
+            }
+
+            constexpr ALWAYS_INLINE void SetBit(size_t offset, bool enabled) {
+                const u64 mask = 1ul << offset;
+                if (enabled) {
+                    m_value |= mask;
+                } else {
+                    m_value &= ~mask;
+                }
+            }
+    };
+
+    template<typename Derived>
+    class GenericRegisterAccessor : public GenericRegisterAccessorBase {
+        public:
+            constexpr ALWAYS_INLINE GenericRegisterAccessor(u64 v) : GenericRegisterAccessorBase(v) { /* ... */ }
+        protected:
+            ALWAYS_INLINE void Store() const {
+                static_cast<const Derived *>(this)->Store();
+            }
+    };
+
+    #define MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(name) class name##RegisterAccessor : public GenericRegisterAccessor<name##RegisterAccessor>
+
+    #define MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(accessor, reg_name)                                                  \
+        ALWAYS_INLINE accessor##RegisterAccessor() : GenericRegisterAccessor(MESOSPHERE_CPU_GET_SYSREG(reg_name)) { /* ... */ } \
+        constexpr ALWAYS_INLINE accessor##RegisterAccessor(u64 v) : GenericRegisterAccessor(v) { /* ... */ }                    \
+                                                                                                                                \
+        ALWAYS_INLINE void Store() { const u64 v = this->GetValue(); MESOSPHERE_CPU_SET_SYSREG(reg_name, v); }
+
+    /* Accessors. */
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(MemoryAccessIndirection) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(MemoryAccessIndirection, mair_el1)
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(TranslationControl) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(TranslationControl, tcr_el1)
+
+            constexpr ALWAYS_INLINE size_t GetT0Size() const {
+                const size_t shift_value = this->GetBits(0, 6);
+                return size_t(1) << (size_t(64) - shift_value);
+            }
+
+            constexpr ALWAYS_INLINE size_t GetT1Size() const {
+                const size_t shift_value = this->GetBits(16, 6);
+                return size_t(1) << (size_t(64) - shift_value);
+            }
+
+            constexpr ALWAYS_INLINE bool GetEpd0() const {
+                return this->GetBits(7, 1) != 0;
+            }
+
+            constexpr ALWAYS_INLINE decltype(auto) SetEpd0(bool set) {
+                this->SetBit(7, set);
+                return *this;
+            }
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(ArchitecturalFeatureAccessControl) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(ArchitecturalFeatureAccessControl, cpacr_el1)
+
+            constexpr ALWAYS_INLINE decltype(auto) SetFpEnabled(bool en) {
+                if (en) {
+                    this->SetBits(20, 2, 0x3);
+                } else {
+                    this->SetBits(20, 2, 0x0);
+                }
+                return *this;
+            }
+
+            constexpr ALWAYS_INLINE bool IsFpEnabled() {
+                return this->GetBits(20, 2) != 0;
+            }
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(DebugFeature) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(DebugFeature, id_aa64dfr0_el1)
+
+            constexpr ALWAYS_INLINE size_t GetNumWatchpoints() const {
+                return this->GetBits(20, 4);
+            }
+
+            constexpr ALWAYS_INLINE size_t GetNumBreakpoints() const {
+                return this->GetBits(12, 4);
+            }
+
+            constexpr ALWAYS_INLINE size_t GetNumContextAwareBreakpoints() const {
+                return this->GetBits(28, 4);
+            }
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(MonitorDebugSystemControl) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(MonitorDebugSystemControl, mdscr_el1)
+
+            constexpr ALWAYS_INLINE bool GetMde() const {
+                return this->GetBits(15, 1) != 0;
+            }
+
+            constexpr ALWAYS_INLINE size_t GetTdcc() const {
+                return this->GetBits(12, 1) != 0;
+            }
+
+            constexpr ALWAYS_INLINE bool GetSoftwareStep() const {
+                return this->GetBits(0, 1) != 0;
+            }
+
+            constexpr ALWAYS_INLINE decltype(auto) SetMde(bool set) {
+                this->SetBit(15, set);
+                return *this;
+            }
+
+            constexpr ALWAYS_INLINE decltype(auto) SetTdcc(bool set) {
+                this->SetBit(12, set);
+                return *this;
+            }
+
+            constexpr ALWAYS_INLINE decltype(auto) SetSoftwareStep(bool set) {
+                this->SetBit(0, set);
+                return *this;
+            }
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(MultiprocessorAffinity) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(MultiprocessorAffinity, mpidr_el1)
+
+            constexpr ALWAYS_INLINE u64 GetAff0() const {
+                return this->GetBits(0, 8);
+            }
+
+            constexpr ALWAYS_INLINE u64 GetAff1() const {
+                return this->GetBits(8, 8);
+            }
+
+            constexpr ALWAYS_INLINE u64 GetAff2() const {
+                return this->GetBits(16, 8);
+            }
+
+            constexpr ALWAYS_INLINE u64 GetAff3() const {
+                return this->GetBits(32, 8);
+            }
+
+            constexpr ALWAYS_INLINE u64 GetCpuOnArgument() const {
+                constexpr u64 Mask = 0x000000FF00FFFF00ul;
+                return this->GetValue() & Mask;
+            }
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(ThreadId) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(ThreadId, tpidr_el1)
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(OsLockAccess) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(OsLockAccess, oslar_el1)
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(ContextId) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(ContextId, contextidr_el1)
+
+            constexpr ALWAYS_INLINE decltype(auto) SetProcId(u32 proc_id) {
+                this->SetBits(0, BITSIZEOF(proc_id), proc_id);
+                return *this;
+            }
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(MainId) {
+        public:
+            enum class Implementer {
+                ArmLimited = 0x41,
+            };
+            enum class PrimaryPartNumber {
+                CortexA53  = 0xD03,
+                CortexA57  = 0xD07,
+            };
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(MainId, midr_el1)
+        public:
+            constexpr ALWAYS_INLINE Implementer GetImplementer() const {
+                return static_cast<Implementer>(this->GetBits(24, 8));
+            }
+
+            constexpr ALWAYS_INLINE u64 GetVariant() const {
+                return this->GetBits(20, 4);
+            }
+
+            constexpr ALWAYS_INLINE u64 GetArchitecture() const {
+                return this->GetBits(16, 4);
+            }
+
+            constexpr ALWAYS_INLINE PrimaryPartNumber GetPrimaryPartNumber() const {
+                return static_cast<PrimaryPartNumber>(this->GetBits(4, 12));
+            }
+
+            constexpr ALWAYS_INLINE u64 GetRevision() const {
+                return this->GetBits(0, 4);
+            }
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(SystemControl) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(SystemControl, sctlr_el1)
+
+            constexpr ALWAYS_INLINE decltype(auto) SetWxn(bool en) {
+                this->SetBit(19, en);
+                return *this;
+            }
+
+            constexpr ALWAYS_INLINE bool GetWxn() const {
+                return this->GetBits(19, 1) != 0;
+            }
+    };
+
+    /* Accessors for timer registers. */
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(CounterTimerKernelControl) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(CounterTimerKernelControl, cntkctl_el1)
+
+            constexpr ALWAYS_INLINE decltype(auto) SetEl0PctEn(bool en) {
+                this->SetBit(0, en);
+                return *this;
+            }
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(CounterTimerPhysicalTimerControl) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(CounterTimerPhysicalTimerControl, cntp_ctl_el0)
+
+            constexpr ALWAYS_INLINE decltype(auto) SetEnable(bool en) {
+                this->SetBit(0, en);
+                return *this;
+            }
+
+            constexpr ALWAYS_INLINE decltype(auto) SetIMask(bool en) {
+                this->SetBit(1, en);
+                return *this;
+            }
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(CounterTimerPhysicalTimerCompareValue) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(CounterTimerPhysicalTimerCompareValue, cntp_cval_el0)
+
+            constexpr ALWAYS_INLINE u64 GetCompareValue() {
+                return this->GetValue();
+            }
+
+            constexpr ALWAYS_INLINE decltype(auto) SetCompareValue(u64 value) {
+                this->SetBits(0, BITSIZEOF(value), value);
+                return *this;
+            }
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(CounterTimerPhysicalCountValue) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(CounterTimerPhysicalCountValue, cntpct_el0)
+
+            constexpr ALWAYS_INLINE u64 GetCount() {
+                return this->GetValue();
+            }
+    };
+
+    /* Accessors for cache registers. */
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(CacheLineId) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(CacheLineId, clidr_el1)
+        public:
+            constexpr ALWAYS_INLINE int GetLevelsOfCoherency() const {
+                return static_cast<int>(this->GetBits(24, 3));
+            }
+
+            constexpr ALWAYS_INLINE int GetLevelsOfUnification() const {
+                return static_cast<int>(this->GetBits(21, 3));
+            }
+
+            /* TODO: Other bitfield accessors? */
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(CacheSizeId) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(CacheSizeId, ccsidr_el1)
+        public:
+            constexpr ALWAYS_INLINE int GetNumberOfSets() const {
+                return static_cast<int>(this->GetBits(13, 15));
+            }
+
+            constexpr ALWAYS_INLINE int GetAssociativity() const {
+                return static_cast<int>(this->GetBits(3, 10));
+            }
+
+            constexpr ALWAYS_INLINE int GetLineSize() const {
+                return static_cast<int>(this->GetBits(0, 3));
+            }
+
+            /* TODO: Other bitfield accessors? */
+    };
+
+    MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(PerformanceMonitorsControl) {
+        public:
+            MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(PerformanceMonitorsControl, pmcr_el0)
+        public:
+            constexpr ALWAYS_INLINE u64 GetN() const {
+                return this->GetBits(11, 5);
+            }
+
+            constexpr ALWAYS_INLINE decltype(auto) SetEventCounterReset(bool en) {
+                this->SetBit(1, en);
+                return *this;
+            }
+
+            constexpr ALWAYS_INLINE decltype(auto) SetCycleCounterReset(bool en) {
+                this->SetBit(2, en);
+                return *this;
+            }
+
+            /* TODO: Other bitfield accessors? */
+    };
+
+    #undef  FOR_I_IN_0_TO_15
+    #undef  MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS
+    #undef  MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS
+    #undef  MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS
+    #undef  MESOSPHERE_CPU_GET_SYSREG
+    #undef  MESOSPHERE_CPU_SET_SYSREG
+
+}

Source/Atmosphere/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_debug.hpp

@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_select_cpu.hpp>
+#include <mesosphere/kern_k_debug_base.hpp>
+
+namespace ams::kern {
+
+    class KThread;
+    class KProcess;
+
+}
+
+namespace ams::kern::arch::arm64 {
+
+    class KDebug final : public KAutoObjectWithSlabHeapAndContainer<KDebug, KDebugBase> {
+        MESOSPHERE_AUTOOBJECT_TRAITS(KDebug, KSynchronizationObject);
+        public:
+            explicit KDebug() { /* ... */ }
+
+            static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
+        public:
+            /* NOTE: These are virtual in Nintendo's kernel. */
+            Result GetThreadContextImpl(ams::svc::ThreadContext *out, KThread *thread, u32 context_flags);
+            Result SetThreadContextImpl(const ams::svc::ThreadContext &ctx, KThread *thread, u32 context_flags);
+        private:
+            Result GetFpuContext(ams::svc::ThreadContext *out, KThread *thread, u32 context_flags);
+            Result SetFpuContext(const ams::svc::ThreadContext &ctx, KThread *thread, u32 context_flags);
+        public:
+            static uintptr_t GetProgramCounter(const KThread &thread);
+            static void SetPreviousProgramCounter();
+
+            static void PrintRegister(KThread *thread = nullptr);
+            static void PrintBacktrace(KThread *thread = nullptr);
+
+            static Result BreakIfAttached(ams::svc::BreakReason break_reason, uintptr_t address, size_t size);
+            static Result SetHardwareBreakPoint(ams::svc::HardwareBreakPointRegisterName name, u64 flags, u64 value);
+
+            static constexpr bool IsBreakInstruction(u32 insn, u32 psr) {
+                constexpr u32 BreakInstructionAarch64 = 0xE7FFFFFF;
+                constexpr u32 BreakInstructionAarch32 = 0xE7FFDEFE;
+                constexpr u32 BreakInstructionThumb32 = 0xB68E;
+                if ((psr & 0x10) == 0) {
+                    return insn == BreakInstructionAarch64;
+                } else {
+                    if ((psr & 0x20) == 0) {
+                        return insn == BreakInstructionAarch32;
+                    } else {
+                        return insn == BreakInstructionThumb32;
+                    }
+                }
+            }
+    };
+
+}

Source/Atmosphere/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_exception_context.hpp

@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+
+namespace ams::kern::arch::arm64 {
+
+    struct KExceptionContext {
+        u64 x[(30 - 0) + 1];
+        u64 sp;
+        u64 pc;
+        u32 psr;
+        u32 write;
+        u64 tpidr;
+        u64 reserved;
+
+        constexpr void GetSvcThreadContext(ams::svc::LastThreadContext *out) const {
+            if ((this->psr & 0x10) == 0) {
+                /* aarch64 thread. */
+                out->fp = this->x[29];
+                out->sp = this->sp;
+                out->lr = this->x[30];
+                out->pc = this->pc;
+            } else {
+                /* aarch32 thread. */
+                out->fp = this->x[11];
+                out->sp = this->x[13];
+                out->lr = this->x[14];
+                out->pc = this->pc;
+            }
+        }
+    };
+    static_assert(sizeof(KExceptionContext) == EXCEPTION_CONTEXT_SIZE);
+
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[ 0]) == EXCEPTION_CONTEXT_X0);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[ 1]) == EXCEPTION_CONTEXT_X1);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[ 2]) == EXCEPTION_CONTEXT_X2);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[ 3]) == EXCEPTION_CONTEXT_X3);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[ 4]) == EXCEPTION_CONTEXT_X4);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[ 5]) == EXCEPTION_CONTEXT_X5);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[ 6]) == EXCEPTION_CONTEXT_X6);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[ 7]) == EXCEPTION_CONTEXT_X7);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[ 8]) == EXCEPTION_CONTEXT_X8);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[ 9]) == EXCEPTION_CONTEXT_X9);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[10]) == EXCEPTION_CONTEXT_X10);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[11]) == EXCEPTION_CONTEXT_X11);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[12]) == EXCEPTION_CONTEXT_X12);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[13]) == EXCEPTION_CONTEXT_X13);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[14]) == EXCEPTION_CONTEXT_X14);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[15]) == EXCEPTION_CONTEXT_X15);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[16]) == EXCEPTION_CONTEXT_X16);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[17]) == EXCEPTION_CONTEXT_X17);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[18]) == EXCEPTION_CONTEXT_X18);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[19]) == EXCEPTION_CONTEXT_X19);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[20]) == EXCEPTION_CONTEXT_X20);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[21]) == EXCEPTION_CONTEXT_X21);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[22]) == EXCEPTION_CONTEXT_X22);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[23]) == EXCEPTION_CONTEXT_X23);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[24]) == EXCEPTION_CONTEXT_X24);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[25]) == EXCEPTION_CONTEXT_X25);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[26]) == EXCEPTION_CONTEXT_X26);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[27]) == EXCEPTION_CONTEXT_X27);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[28]) == EXCEPTION_CONTEXT_X28);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[29]) == EXCEPTION_CONTEXT_X29);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[30]) == EXCEPTION_CONTEXT_X30);
+    static_assert(AMS_OFFSETOF(KExceptionContext, sp)    == EXCEPTION_CONTEXT_SP);
+    static_assert(AMS_OFFSETOF(KExceptionContext, pc)    == EXCEPTION_CONTEXT_PC);
+    static_assert(AMS_OFFSETOF(KExceptionContext, psr)   == EXCEPTION_CONTEXT_PSR);
+    static_assert(AMS_OFFSETOF(KExceptionContext, tpidr) == EXCEPTION_CONTEXT_TPIDR);
+
+}

Source/Atmosphere/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_hardware_timer.hpp

@@ -0,0 +1,95 @@
+/*
+ * Copyright (c) Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_select_cpu.hpp>
+#include <mesosphere/kern_k_hardware_timer_base.hpp>
+
+namespace ams::kern::arch::arm64 {
+
+    class KHardwareTimer : public KInterruptTask, public KHardwareTimerBase {
+        private:
+            s64 m_maximum_time;
+        public:
+            constexpr KHardwareTimer() : KInterruptTask(), KHardwareTimerBase(), m_maximum_time(std::numeric_limits<s64>::max()) { /* ... */ }
+        public:
+            /* Public API. */
+            NOINLINE void Initialize();
+            NOINLINE void Finalize();
+
+            static s64 GetTick() {
+                return GetCount();
+            }
+
+            void RegisterAbsoluteTask(KTimerTask *task, s64 task_time) {
+                KScopedDisableDispatch dd;
+                KScopedSpinLock lk(this->GetLock());
+
+                if (this->RegisterAbsoluteTaskImpl(task, task_time)) {
+                    if (task_time <= m_maximum_time) {
+                        SetCompareValue(task_time);
+                        EnableInterrupt();
+                    }
+                }
+            }
+        private:
+            /* Hardware register accessors. */
+            static ALWAYS_INLINE void InitializeGlobalTimer() {
+                /* Set kernel control. */
+                cpu::CounterTimerKernelControlRegisterAccessor(0).SetEl0PctEn(true).Store();
+
+                /* Disable the physical timer. */
+                cpu::CounterTimerPhysicalTimerControlRegisterAccessor(0).SetEnable(false).SetIMask(false).Store();
+
+                /* Set the compare value to the maximum. */
+                cpu::CounterTimerPhysicalTimerCompareValueRegisterAccessor(0).SetCompareValue(std::numeric_limits<u64>::max()).Store();
+
+                /* Enable the physical timer, with interrupt masked. */
+                cpu::CounterTimerPhysicalTimerControlRegisterAccessor(0).SetEnable(true).SetIMask(true).Store();
+            }
+
+            static ALWAYS_INLINE void EnableInterrupt() {
+                cpu::CounterTimerPhysicalTimerControlRegisterAccessor(0).SetEnable(true).SetIMask(false).Store();
+            }
+
+            static ALWAYS_INLINE void DisableInterrupt() {
+                cpu::CounterTimerPhysicalTimerControlRegisterAccessor(0).SetEnable(true).SetIMask(true).Store();
+            }
+
+            static ALWAYS_INLINE void StopTimer() {
+                /* Set the compare value to the maximum. */
+                cpu::CounterTimerPhysicalTimerCompareValueRegisterAccessor(0).SetCompareValue(std::numeric_limits<u64>::max()).Store();
+
+                /* Disable the physical timer. */
+                cpu::CounterTimerPhysicalTimerControlRegisterAccessor(0).SetEnable(false).SetIMask(false).Store();
+            }
+
+            static ALWAYS_INLINE s64 GetCount() {
+                return cpu::CounterTimerPhysicalCountValueRegisterAccessor().GetCount();
+            }
+
+            static ALWAYS_INLINE void SetCompareValue(s64 value) {
+                cpu::CounterTimerPhysicalTimerCompareValueRegisterAccessor(0).SetCompareValue(static_cast<u64>(value)).Store();
+            }
+        public:
+            virtual KInterruptTask *OnInterrupt(s32 interrupt_id) override {
+                MESOSPHERE_UNUSED(interrupt_id);
+                return this;
+            }
+
+            virtual void DoTask() override;
+    };
+
+}

Source/Atmosphere/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_interrupt_controller.hpp

@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <vapours.hpp>
+#include <mesosphere/kern_select_cpu.hpp>
+#include <mesosphere/kern_k_typed_address.hpp>
+
+#if 1
+
+    #include <mesosphere/arch/arm/kern_generic_interrupt_controller.hpp>
+    namespace ams::kern::arch::arm64 {
+
+        using ams::kern::arch::arm::GicDistributor;
+        using ams::kern::arch::arm::GicCpuInterface;
+        using ams::kern::arch::arm::KInterruptController;
+
+    }
+
+#else
+
+    #error "Unknown board for KInterruptController"
+
+#endif

Source/Atmosphere/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_interrupt_manager.hpp

@@ -0,0 +1,151 @@
+/*
+ * Copyright (c) Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <vapours.hpp>
+#include <mesosphere/kern_select_cpu.hpp>
+#include <mesosphere/kern_k_spin_lock.hpp>
+#include <mesosphere/kern_k_interrupt_task.hpp>
+#include <mesosphere/kern_select_interrupt_controller.hpp>
+
+namespace ams::kern::arch::arm64 {
+
+    class KInterruptManager {
+        NON_COPYABLE(KInterruptManager);
+        NON_MOVEABLE(KInterruptManager);
+        private:
+            struct KCoreLocalInterruptEntry {
+                KInterruptHandler *handler;
+                bool manually_cleared;
+                bool needs_clear;
+                u8 priority;
+
+                constexpr KCoreLocalInterruptEntry()
+                    : handler(nullptr), manually_cleared(false), needs_clear(false), priority(KInterruptController::PriorityLevel_Low)
+                {
+                    /* ... */
+                }
+            };
+
+            struct KGlobalInterruptEntry {
+                KInterruptHandler *handler;
+                bool manually_cleared;
+                bool needs_clear;
+
+                constexpr KGlobalInterruptEntry() : handler(nullptr), manually_cleared(false), needs_clear(false) { /* ... */ }
+            };
+        private:
+            KCoreLocalInterruptEntry m_core_local_interrupts[cpu::NumCores][KInterruptController::NumLocalInterrupts]{};
+            KInterruptController m_interrupt_controller{};
+            KInterruptController::LocalState m_local_states[cpu::NumCores]{};
+            bool m_local_state_saved[cpu::NumCores]{};
+            mutable KSpinLock m_global_interrupt_lock{};
+            KGlobalInterruptEntry m_global_interrupts[KInterruptController::NumGlobalInterrupts]{};
+            KInterruptController::GlobalState m_global_state{};
+            bool m_global_state_saved{};
+        private:
+            ALWAYS_INLINE KSpinLock &GetGlobalInterruptLock() const { return m_global_interrupt_lock; }
+            ALWAYS_INLINE KGlobalInterruptEntry &GetGlobalInterruptEntry(s32 irq) { return m_global_interrupts[KInterruptController::GetGlobalInterruptIndex(irq)]; }
+            ALWAYS_INLINE KCoreLocalInterruptEntry &GetLocalInterruptEntry(s32 irq) { return m_core_local_interrupts[GetCurrentCoreId()][KInterruptController::GetLocalInterruptIndex(irq)]; }
+
+            bool OnHandleInterrupt();
+        public:
+            constexpr KInterruptManager() = default;
+
+            NOINLINE void Initialize(s32 core_id);
+            NOINLINE void Finalize(s32 core_id);
+
+            NOINLINE void Save(s32 core_id);
+            NOINLINE void Restore(s32 core_id);
+
+            bool IsInterruptDefined(s32 irq) const {
+                return m_interrupt_controller.IsInterruptDefined(irq);
+            }
+
+            bool IsGlobal(s32 irq) const {
+                return m_interrupt_controller.IsGlobal(irq);
+            }
+
+            bool IsLocal(s32 irq) const {
+                return m_interrupt_controller.IsLocal(irq);
+            }
+
+            NOINLINE Result BindHandler(KInterruptHandler *handler, s32 irq, s32 core_id, s32 priority, bool manual_clear, bool level);
+            NOINLINE Result UnbindHandler(s32 irq, s32 core);
+
+            NOINLINE Result ClearInterrupt(s32 irq, s32 core_id);
+
+            ALWAYS_INLINE void SendInterProcessorInterrupt(s32 irq, u64 core_mask) {
+                m_interrupt_controller.SendInterProcessorInterrupt(irq, core_mask);
+            }
+
+            ALWAYS_INLINE void SendInterProcessorInterrupt(s32 irq) {
+                m_interrupt_controller.SendInterProcessorInterrupt(irq);
+            }
+
+            static void HandleInterrupt(bool user_mode);
+        private:
+            Result BindGlobal(KInterruptHandler *handler, s32 irq, s32 core_id, s32 priority, bool manual_clear, bool level);
+            Result BindLocal(KInterruptHandler *handler, s32 irq, s32 priority, bool manual_clear);
+            Result UnbindGlobal(s32 irq);
+            Result UnbindLocal(s32 irq);
+            Result ClearGlobal(s32 irq);
+            Result ClearLocal(s32 irq);
+        private:
+            [[nodiscard]] static ALWAYS_INLINE u32 GetInterruptsEnabledState() {
+                u64 intr_state;
+                __asm__ __volatile__("mrs   %[intr_state], daif\n"
+                                     "ubfx  %[intr_state], %[intr_state], #7, #1"
+                                     : [intr_state]"=r"(intr_state)
+                                     :: "memory");
+                return intr_state;
+            }
+        public:
+            static ALWAYS_INLINE void EnableInterrupts() {
+                __asm__ __volatile__("msr   daifclr, #2" ::: "memory");
+            }
+
+            static ALWAYS_INLINE void DisableInterrupts() {
+                __asm__ __volatile__("msr   daifset, #2" ::: "memory");
+            }
+
+            [[nodiscard]] static ALWAYS_INLINE u32 GetInterruptsEnabledStateAndDisableInterrupts() {
+                const auto intr_state = GetInterruptsEnabledState();
+                DisableInterrupts();
+                return intr_state;
+            }
+
+            [[nodiscard]] static ALWAYS_INLINE u32 GetInterruptsEnabledStateAndEnableInterrupts() {
+                const auto intr_state = GetInterruptsEnabledState();
+                EnableInterrupts();
+                return intr_state;
+            }
+
+            static ALWAYS_INLINE void RestoreInterrupts(u32 intr_state) {
+                u64 tmp;
+                __asm__ __volatile__("mrs   %[tmp], daif\n"
+                                     "bfi   %[tmp], %[intr_state], #7, #1\n"
+                                     "msr daif, %[tmp]"
+                                     : [tmp]"=&r"(tmp)
+                                     : [intr_state]"r"(intr_state)
+                                     : "memory");
+            }
+
+            static ALWAYS_INLINE bool AreInterruptsEnabled() {
+                return GetInterruptsEnabledState() == 0;
+            }
+    };
+
+}

Source/Atmosphere/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_interrupt_name.hpp

@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+
+namespace ams::kern::arch::arm64 {
+
+    namespace interrupt_name {
+
+        enum KInterruptName : s32 {
+            /* SGIs */
+            KInterruptName_ThreadTerminate        = 0,
+            KInterruptName_CacheOperation         = 1,
+            KInterruptName_Scheduler              = 2,
+            KInterruptName_CoreBarrier            = 3,
+
+            KInterruptName_PerformanceCounter     = 4,
+
+            /* PPIs */
+    #if defined(ATMOSPHERE_BOARD_NINTENDO_NX)
+            KInterruptName_VirtualMaintenance     = 25,
+            KInterruptName_HypervisorTimer        = 26,
+            KInterruptName_VirtualTimer           = 27,
+            KInterruptName_LegacyNFiq             = 28,
+            KInterruptName_SecurePhysicalTimer    = 29,
+            KInterruptName_NonSecurePhysicalTimer = 30,
+            KInterruptName_LegacyNIrq             = 31,
+    #elif defined(ATMOSPHERE_BOARD_QEMU_VIRT)
+            KInterruptName_VirtualTimer           = 27,
+            KInterruptName_SecurePhysicalTimer    = 29,
+            KInterruptName_NonSecurePhysicalTimer = 30,
+    #endif
+
+    #if defined(ATMOSPHERE_BOARD_NINTENDO_NX)
+            KInterruptName_MemoryController       = 109,
+    #endif
+        };
+
+    };
+
+}

Source/Atmosphere/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_memory_region_device_types.inc

@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/* All architectures must define NumArchitectureDeviceRegions. */
+constexpr inline const auto NumArchitectureDeviceRegions = 3;
+
+constexpr inline const auto KMemoryRegionType_Uart                  = KMemoryRegionType_ArchDeviceBase.DeriveSparse(0, NumArchitectureDeviceRegions, 0);
+constexpr inline const auto KMemoryRegionType_InterruptCpuInterface = KMemoryRegionType_ArchDeviceBase.DeriveSparse(0, NumArchitectureDeviceRegions, 1).SetAttribute(KMemoryRegionAttr_NoUserMap);
+constexpr inline const auto KMemoryRegionType_InterruptDistributor  = KMemoryRegionType_ArchDeviceBase.DeriveSparse(0, NumArchitectureDeviceRegions, 2).SetAttribute(KMemoryRegionAttr_NoUserMap);
+static_assert(KMemoryRegionType_Uart                 .GetValue() == (0x1D));
+static_assert(KMemoryRegionType_InterruptCpuInterface.GetValue() == (0x2D | KMemoryRegionAttr_NoUserMap));
+static_assert(KMemoryRegionType_InterruptDistributor .GetValue() == (0x4D | KMemoryRegionAttr_NoUserMap));

Source/Atmosphere/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table.hpp

@@ -0,0 +1,270 @@
+/*
+ * Copyright (c) Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_select_cpu.hpp>
+#include <mesosphere/kern_k_page_table_base.hpp>
+#include <mesosphere/kern_k_page_group.hpp>
+#include <mesosphere/kern_k_page_table_manager.hpp>
+
+namespace ams::kern::arch::arm64 {
+
+    class KPageTable final : public KPageTableBase {
+        NON_COPYABLE(KPageTable);
+        NON_MOVEABLE(KPageTable);
+        private:
+            friend class KPageTableBase;
+        public:
+            using TraversalEntry   = KPageTableImpl::TraversalEntry;
+            using TraversalContext = KPageTableImpl::TraversalContext;
+
+            enum BlockType {
+                BlockType_L3Block,
+                BlockType_L3ContiguousBlock,
+                BlockType_L2Block,
+
+#ifdef ATMOSPHERE_BOARD_NINTENDO_NX
+                BlockType_L2TegraSmmuBlock,
+#endif
+
+                BlockType_L2ContiguousBlock,
+                BlockType_L1Block,
+
+                BlockType_Count,
+            };
+            static_assert(L3BlockSize == PageSize);
+            static constexpr size_t ContiguousPageSize = L3ContiguousBlockSize;
+
+#ifdef ATMOSPHERE_BOARD_NINTENDO_NX
+            static constexpr size_t L2TegraSmmuBlockSize = 2 * L2BlockSize;
+#endif
+            static constexpr size_t BlockSizes[BlockType_Count] = {
+                [BlockType_L3Block]           = L3BlockSize,
+                [BlockType_L3ContiguousBlock] = L3ContiguousBlockSize,
+                [BlockType_L2Block]           = L2BlockSize,
+#ifdef ATMOSPHERE_BOARD_NINTENDO_NX
+                [BlockType_L2TegraSmmuBlock]  = L2TegraSmmuBlockSize,
+#endif
+                [BlockType_L2ContiguousBlock] = L2ContiguousBlockSize,
+                [BlockType_L1Block]           = L1BlockSize,
+            };
+
+            static constexpr BlockType GetMaxBlockType() {
+                return BlockType_L1Block;
+            }
+
+            static constexpr size_t GetBlockSize(BlockType type) {
+                return BlockSizes[type];
+            }
+
+            static constexpr BlockType GetBlockType(size_t size) {
+                switch (size) {
+                    case L3BlockSize:           return BlockType_L3Block;
+                    case L3ContiguousBlockSize: return BlockType_L3ContiguousBlock;
+                    case L2BlockSize:           return BlockType_L2Block;
+#ifdef ATMOSPHERE_BOARD_NINTENDO_NX
+                    case L2TegraSmmuBlockSize:  return BlockType_L2TegraSmmuBlock;
+#endif
+                    case L2ContiguousBlockSize: return BlockType_L2ContiguousBlock;
+                    case L1BlockSize:           return BlockType_L1Block;
+                    MESOSPHERE_UNREACHABLE_DEFAULT_CASE();
+                }
+            }
+
+            static constexpr size_t GetSmallerAlignment(size_t alignment) {
+                MESOSPHERE_ASSERT(alignment > L3BlockSize);
+                return KPageTable::GetBlockSize(static_cast<KPageTable::BlockType>(KPageTable::GetBlockType(alignment) - 1));
+            }
+
+            static constexpr size_t GetLargerAlignment(size_t alignment) {
+                MESOSPHERE_ASSERT(alignment < L1BlockSize);
+                return KPageTable::GetBlockSize(static_cast<KPageTable::BlockType>(KPageTable::GetBlockType(alignment) + 1));
+            }
+        public:
+            /* TODO: How should this size be determined. Does the KProcess slab count need to go in a header as a define? */
+            static constexpr size_t NumTtbr0Entries = 81;
+        private:
+            static constinit inline const volatile u64 s_ttbr0_entries[NumTtbr0Entries] = {};
+        private:
+            KPageTableManager *m_manager;
+            u8 m_asid;
+        protected:
+            Result OperateImpl(PageLinkedList *page_list, KProcessAddress virt_addr, size_t num_pages, KPhysicalAddress phys_addr, bool is_pa_valid, const KPageProperties properties, OperationType operation, bool reuse_ll);
+            Result OperateImpl(PageLinkedList *page_list, KProcessAddress virt_addr, size_t num_pages, const KPageGroup &page_group, const KPageProperties properties, OperationType operation, bool reuse_ll);
+            void FinalizeUpdateImpl(PageLinkedList *page_list);
+
+            KPageTableManager &GetPageTableManager() const { return *m_manager; }
+        private:
+            constexpr PageTableEntry GetEntryTemplate(const KPageProperties properties) const {
+                /* Check that the property is not kernel execute. */
+                MESOSPHERE_ABORT_UNLESS((properties.perm & KMemoryPermission_KernelExecute) == 0);
+
+                /* Set basic attributes. */
+                PageTableEntry entry{PageTableEntry::ExtensionFlag_Valid};
+                entry.SetPrivilegedExecuteNever(true);
+                entry.SetAccessFlag(PageTableEntry::AccessFlag_Accessed);
+                entry.SetShareable(PageTableEntry::Shareable_InnerShareable);
+
+                if (!this->IsKernel()) {
+                    entry.SetGlobal(false);
+                }
+
+                /* Set page attribute. */
+                if (properties.io) {
+                    MESOSPHERE_ABORT_UNLESS(!properties.uncached);
+                    MESOSPHERE_ABORT_UNLESS((properties.perm & KMemoryPermission_UserExecute) == 0);
+
+                    entry.SetPageAttribute(PageTableEntry::PageAttribute_Device_nGnRnE)
+                         .SetUserExecuteNever(true);
+                } else if (properties.uncached) {
+                    MESOSPHERE_ABORT_UNLESS((properties.perm & KMemoryPermission_UserExecute) == 0);
+
+                    entry.SetPageAttribute(PageTableEntry::PageAttribute_NormalMemoryNotCacheable)
+                         .SetUserExecuteNever(true);
+                } else {
+                    entry.SetPageAttribute(PageTableEntry::PageAttribute_NormalMemory);
+
+                    if ((properties.perm & KMemoryPermission_UserExecute) != 0) {
+                        /* Check that the permission is either r--/--x or r--/r-x. */
+                        MESOSPHERE_ABORT_UNLESS((properties.perm & ~ams::svc::MemoryPermission_Read) == (KMemoryPermission_KernelRead | KMemoryPermission_UserExecute));
+                    } else {
+                        entry.SetUserExecuteNever(true);
+                    }
+                }
+
+                /* Set AP[1] based on perm. */
+                switch (properties.perm & KMemoryPermission_UserReadWrite) {
+                    case KMemoryPermission_UserReadWrite:
+                    case KMemoryPermission_UserRead:
+                        entry.SetUserAccessible(true);
+                        break;
+                    case KMemoryPermission_KernelReadWrite:
+                    case KMemoryPermission_KernelRead:
+                        entry.SetUserAccessible(false);
+                        break;
+                    MESOSPHERE_UNREACHABLE_DEFAULT_CASE();
+                }
+
+                /* Set AP[2] based on perm. */
+                switch (properties.perm & KMemoryPermission_UserReadWrite) {
+                    case KMemoryPermission_UserReadWrite:
+                    case KMemoryPermission_KernelReadWrite:
+                        entry.SetReadOnly(false);
+                        break;
+                    case KMemoryPermission_KernelRead:
+                    case KMemoryPermission_UserRead:
+                        entry.SetReadOnly(true);
+                        break;
+                    MESOSPHERE_UNREACHABLE_DEFAULT_CASE();
+                }
+
+                /* Set the fault bit based on whether the page is mapped. */
+                entry.SetMapped((properties.perm & KMemoryPermission_NotMapped) == 0);
+
+                return entry;
+            }
+        public:
+            constexpr explicit KPageTable(util::ConstantInitializeTag) : KPageTableBase(util::ConstantInitialize), m_manager(), m_asid() { /* ... */ }
+            explicit KPageTable() { /* ... */ }
+
+            static NOINLINE void Initialize(s32 core_id);
+
+            static const volatile u64 &GetTtbr0Entry(size_t index) { return s_ttbr0_entries[index]; }
+
+            static ALWAYS_INLINE u64 GetKernelTtbr0() {
+                return s_ttbr0_entries[0];
+            }
+
+            static ALWAYS_INLINE void ActivateKernel() {
+                /* Activate, using asid 0 and process id = 0xFFFFFFFF */
+                cpu::SwitchProcess(GetKernelTtbr0(), 0xFFFFFFFF);
+            }
+
+            static ALWAYS_INLINE void ActivateProcess(size_t proc_idx, u32 proc_id) {
+                cpu::SwitchProcess(s_ttbr0_entries[proc_idx + 1], proc_id);
+            }
+
+            NOINLINE Result InitializeForKernel(void *table, KVirtualAddress start, KVirtualAddress end);
+            NOINLINE Result InitializeForProcess(ams::svc::CreateProcessFlag flags, bool from_back, KMemoryManager::Pool pool, KProcessAddress code_address, size_t code_size, KSystemResource *system_resource, KResourceLimit *resource_limit, size_t process_index);
+            Result Finalize();
+
+            static void NoteUpdatedCallback(const void *pt) {
+                /* Note the update. */
+                static_cast<const KPageTable *>(pt)->NoteUpdated();
+            }
+        private:
+            Result Unmap(KProcessAddress virt_addr, size_t num_pages, PageLinkedList *page_list, bool force, bool reuse_ll);
+
+            Result Map(KProcessAddress virt_addr, KPhysicalAddress phys_addr, size_t num_pages, PageTableEntry entry_template, bool disable_head_merge, size_t page_size, PageLinkedList *page_list, bool reuse_ll);
+
+            Result MapContiguous(KProcessAddress virt_addr, KPhysicalAddress phys_addr, size_t num_pages, PageTableEntry entry_template, bool disable_head_merge, PageLinkedList *page_list, bool reuse_ll);
+            Result MapGroup(KProcessAddress virt_addr, const KPageGroup &pg, size_t num_pages, PageTableEntry entry_template, bool disable_head_merge, bool not_first, PageLinkedList *page_list, bool reuse_ll);
+
+            bool MergePages(TraversalContext *context, PageLinkedList *page_list);
+            void MergePages(KProcessAddress virt_addr, size_t num_pages, PageLinkedList *page_list);
+
+            Result SeparatePagesImpl(TraversalEntry *entry, TraversalContext *context, KProcessAddress virt_addr, size_t block_size, PageLinkedList *page_list, bool reuse_ll);
+            Result SeparatePages(KProcessAddress virt_addr, size_t num_pages, PageLinkedList *page_list, bool reuse_ll);
+
+            Result ChangePermissions(KProcessAddress virt_addr, size_t num_pages, PageTableEntry entry_template, DisableMergeAttribute disable_merge_attr, bool refresh_mapping, bool flush_mapping, PageLinkedList *page_list, bool reuse_ll);
+
+            static ALWAYS_INLINE void PteDataMemoryBarrier() {
+                cpu::DataMemoryBarrierInnerShareableStore();
+            }
+
+            static ALWAYS_INLINE void ClearPageTable(KVirtualAddress table) {
+                cpu::ClearPageToZero(GetVoidPointer(table));
+            }
+
+            ALWAYS_INLINE void OnTableUpdated() const {
+                cpu::InvalidateTlbByAsid(m_asid);
+            }
+
+            ALWAYS_INLINE void OnKernelTableUpdated() const {
+                cpu::InvalidateEntireTlbDataOnly();
+            }
+
+            ALWAYS_INLINE void OnKernelTableSinglePageUpdated(KProcessAddress virt_addr) const {
+                cpu::InvalidateTlbByVaDataOnly(virt_addr);
+            }
+
+            void NoteUpdated() const;
+            void NoteSingleKernelPageUpdated(KProcessAddress virt_addr) const;
+
+            KVirtualAddress AllocatePageTable(PageLinkedList *page_list, bool reuse_ll) const {
+                KVirtualAddress table = this->GetPageTableManager().Allocate();
+
+                if (table == Null<KVirtualAddress>) {
+                    if (reuse_ll && page_list->Peek()) {
+                        table = KVirtualAddress(reinterpret_cast<uintptr_t>(page_list->Pop()));
+                    } else {
+                        return Null<KVirtualAddress>;
+                    }
+                }
+
+                MESOSPHERE_ASSERT(this->GetPageTableManager().GetRefCount(table) == 0);
+
+                return table;
+            }
+
+            void FreePageTable(PageLinkedList *page_list, KVirtualAddress table) const {
+                MESOSPHERE_ASSERT(this->GetPageTableManager().IsInPageTableHeap(table));
+                MESOSPHERE_ASSERT(this->GetPageTableManager().GetRefCount(table) == 0);
+                page_list->Push(table);
+            }
+    };
+
+}

Source/Atmosphere/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table_entry.hpp

@@ -0,0 +1,472 @@
+/*
+ * Copyright (c) Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_select_cpu.hpp>
+#include <mesosphere/kern_k_typed_address.hpp>
+
+namespace ams::kern::arch::arm64 {
+
+    constexpr size_t BlocksPerContiguousBlock = 0x10;
+    constexpr size_t BlocksPerTable           = PageSize / sizeof(u64);
+
+    constexpr size_t L1BlockSize           = 1_GB;
+    constexpr size_t L1ContiguousBlockSize = BlocksPerContiguousBlock * L1BlockSize;
+    constexpr size_t L2BlockSize           = 2_MB;
+    constexpr size_t L2ContiguousBlockSize = BlocksPerContiguousBlock * L2BlockSize;
+    constexpr size_t L3BlockSize           = PageSize;
+    constexpr size_t L3ContiguousBlockSize = BlocksPerContiguousBlock * L3BlockSize;
+
+    class PageTableEntry {
+        public:
+            struct InvalidTag{};
+            struct TableTag{};
+            struct BlockTag{};
+            struct SeparateContiguousTag{};
+
+            enum Permission : u64 {
+                Permission_KernelRWX = ((0ul << 53) | (1ul << 54) | (0ul << 6)),
+                Permission_KernelRX  = ((0ul << 53) | (1ul << 54) | (2ul << 6)),
+                Permission_KernelR   = ((1ul << 53) | (1ul << 54) | (2ul << 6)),
+                Permission_KernelRW  = ((1ul << 53) | (1ul << 54) | (0ul << 6)),
+
+                Permission_UserRX    = ((1ul << 53) | (0ul << 54) | (3ul << 6)),
+                Permission_UserR     = ((1ul << 53) | (1ul << 54) | (3ul << 6)),
+                Permission_UserRW    = ((1ul << 53) | (1ul << 54) | (1ul << 6)),
+            };
+
+            enum Shareable : u64 {
+                Shareable_NonShareable   = (0 << 8),
+                Shareable_OuterShareable = (2 << 8),
+                Shareable_InnerShareable = (3 << 8),
+            };
+
+            /* Official attributes are: */
+            /*    0x00, 0x04, 0xFF, 0x44. 4-7 are unused. */
+            enum PageAttribute : u64 {
+                PageAttribute_Device_nGnRnE            = (0 << 2),
+                PageAttribute_Device_nGnRE             = (1 << 2),
+                PageAttribute_NormalMemory             = (2 << 2),
+                PageAttribute_NormalMemoryNotCacheable = (3 << 2),
+            };
+
+            enum AccessFlag : u64 {
+                AccessFlag_NotAccessed = (0 << 10),
+                AccessFlag_Accessed    = (1 << 10),
+            };
+
+            enum MappingFlag : u64 {
+                MappingFlag_NotMapped = (0 << 0),
+                MappingFlag_Mapped    = (1 << 0),
+            };
+
+            enum SoftwareReservedBit : u8 {
+                SoftwareReservedBit_None                    = 0,
+                SoftwareReservedBit_DisableMergeHead        = (1u << 0),
+                SoftwareReservedBit_DisableMergeHeadAndBody = (1u << 1),
+                SoftwareReservedBit_DisableMergeHeadTail    = (1u << 2),
+                SoftwareReservedBit_Valid                   = (1u << 3),
+            };
+
+            static constexpr ALWAYS_INLINE std::underlying_type<SoftwareReservedBit>::type EncodeSoftwareReservedBits(bool head, bool head_body, bool tail) {
+                return (head ? SoftwareReservedBit_DisableMergeHead : SoftwareReservedBit_None) | (head_body ? SoftwareReservedBit_DisableMergeHeadAndBody : SoftwareReservedBit_None) | (tail ? SoftwareReservedBit_DisableMergeHeadTail : SoftwareReservedBit_None);
+            }
+
+            enum ExtensionFlag : u64 {
+                ExtensionFlag_DisableMergeHead        = (static_cast<u64>(SoftwareReservedBit_DisableMergeHead)        << 55),
+                ExtensionFlag_DisableMergeHeadAndBody = (static_cast<u64>(SoftwareReservedBit_DisableMergeHeadAndBody) << 55),
+                ExtensionFlag_DisableMergeTail        = (static_cast<u64>(SoftwareReservedBit_DisableMergeHeadTail)    << 55),
+                ExtensionFlag_Valid                   = (static_cast<u64>(SoftwareReservedBit_Valid)                   << 55),
+
+                ExtensionFlag_ValidAndMapped = (ExtensionFlag_Valid | MappingFlag_Mapped),
+                ExtensionFlag_TestTableMask  = (ExtensionFlag_Valid | (1ul << 1)),
+            };
+
+            enum Type : u64 {
+                Type_None    = 0x0,
+                Type_L1Block = ExtensionFlag_Valid,
+                Type_L1Table = 0x2,
+                Type_L2Block = ExtensionFlag_Valid,
+                Type_L2Table = 0x2,
+                Type_L3Block = ExtensionFlag_TestTableMask,
+            };
+
+            enum ContigType : u64 {
+                ContigType_NotContiguous = (0x0ul << 52),
+                ContigType_Contiguous    = (0x1ul << 52),
+            };
+        protected:
+            u64 m_attributes;
+        public:
+            /* Take in a raw attribute. */
+            constexpr explicit ALWAYS_INLINE PageTableEntry() : m_attributes() { /* ... */ }
+            constexpr explicit ALWAYS_INLINE PageTableEntry(u64 attr) : m_attributes(attr) { /* ... */ }
+
+            constexpr explicit ALWAYS_INLINE PageTableEntry(InvalidTag) : m_attributes(0) { /* ... */ }
+
+            /* Extend a previous attribute. */
+            constexpr explicit ALWAYS_INLINE PageTableEntry(const PageTableEntry &rhs, u64 new_attr) : m_attributes(rhs.m_attributes | new_attr) { /* ... */ }
+
+            /* Construct a new attribute. */
+            constexpr explicit ALWAYS_INLINE PageTableEntry(Permission perm, PageAttribute p_a, Shareable share, MappingFlag m)
+                : m_attributes(static_cast<u64>(perm) | static_cast<u64>(AccessFlag_Accessed) | static_cast<u64>(p_a) | static_cast<u64>(share) | static_cast<u64>(m))
+            {
+                /* ... */
+            }
+
+            /* Construct a table. */
+            constexpr explicit ALWAYS_INLINE PageTableEntry(TableTag, KPhysicalAddress phys_addr, bool is_kernel, bool pxn, size_t ref_count)
+                : PageTableEntry(((is_kernel ? 0x3ul : 0) << 60) | (static_cast<u64>(pxn) << 59) | GetInteger(phys_addr) | (ref_count << 2) | 0x3)
+            {
+                /* ... */
+            }
+
+            /* Construct a block. */
+            constexpr explicit ALWAYS_INLINE PageTableEntry(BlockTag, KPhysicalAddress phys_addr, const PageTableEntry &attr, u8 sw_reserved_bits, bool contig, bool page)
+                : PageTableEntry(attr, (static_cast<u64>(sw_reserved_bits) << 55) | (static_cast<u64>(contig) << 52) | GetInteger(phys_addr) | (page ? ExtensionFlag_TestTableMask : ExtensionFlag_Valid))
+            {
+                /* ... */
+            }
+            constexpr explicit ALWAYS_INLINE PageTableEntry(BlockTag, KPhysicalAddress phys_addr, const PageTableEntry &attr, SeparateContiguousTag)
+                : PageTableEntry(attr, GetInteger(phys_addr))
+            {
+                /* ... */
+            }
+        protected:
+            constexpr ALWAYS_INLINE u64 GetBits(size_t offset, size_t count) const {
+                return (m_attributes >> offset) & ((1ul << count) - 1);
+            }
+
+            constexpr ALWAYS_INLINE u64 SelectBits(size_t offset, size_t count) const {
+                return m_attributes & (((1ul << count) - 1) << offset);
+            }
+
+            constexpr ALWAYS_INLINE void SetBits(size_t offset, size_t count, u64 value) {
+                const u64 mask = ((1ul << count) - 1) << offset;
+                m_attributes &= ~mask;
+                m_attributes |= (value & (mask >> offset)) << offset;
+            }
+
+            constexpr ALWAYS_INLINE void SetBitsDirect(size_t offset, size_t count, u64 value) {
+                const u64 mask = ((1ul << count) - 1) << offset;
+                m_attributes &= ~mask;
+                m_attributes |= (value & mask);
+            }
+
+            constexpr ALWAYS_INLINE void SetBit(size_t offset, bool enabled) {
+                const u64 mask = 1ul << offset;
+                if (enabled) {
+                    m_attributes |= mask;
+                } else {
+                    m_attributes &= ~mask;
+                }
+            }
+        public:
+            constexpr ALWAYS_INLINE u8 GetSoftwareReservedBits()            const { return this->GetBits(55, 3); }
+            constexpr ALWAYS_INLINE bool IsHeadMergeDisabled()              const { return (this->GetSoftwareReservedBits() & SoftwareReservedBit_DisableMergeHead) != 0; }
+            constexpr ALWAYS_INLINE bool IsHeadAndBodyMergeDisabled()       const { return (this->GetSoftwareReservedBits() & SoftwareReservedBit_DisableMergeHeadAndBody) != 0; }
+            constexpr ALWAYS_INLINE bool IsTailMergeDisabled()              const { return (this->GetSoftwareReservedBits() & SoftwareReservedBit_DisableMergeHeadTail) != 0; }
+            constexpr ALWAYS_INLINE bool IsHeadOrHeadAndBodyMergeDisabled() const { return (this->GetSoftwareReservedBits() & (SoftwareReservedBit_DisableMergeHead | SoftwareReservedBit_DisableMergeHeadAndBody)) != 0; }
+            constexpr ALWAYS_INLINE bool IsUserExecuteNever()               const { return this->GetBits(54, 1) != 0; }
+            constexpr ALWAYS_INLINE bool IsPrivilegedExecuteNever()         const { return this->GetBits(53, 1) != 0; }
+            constexpr ALWAYS_INLINE bool IsContiguous()                     const { return this->GetBits(52, 1) != 0; }
+            constexpr ALWAYS_INLINE bool IsGlobal()                         const { return this->GetBits(11, 1) == 0; }
+            constexpr ALWAYS_INLINE AccessFlag GetAccessFlag()              const { return static_cast<AccessFlag>(this->SelectBits(10, 1)); }
+            constexpr ALWAYS_INLINE Shareable GetShareable()                const { return static_cast<Shareable>(this->SelectBits(8, 2)); }
+            constexpr ALWAYS_INLINE PageAttribute GetPageAttribute()        const { return static_cast<PageAttribute>(this->SelectBits(2, 3)); }
+            constexpr ALWAYS_INLINE int GetAccessFlagInteger()              const { return static_cast<int>(this->GetBits(10, 1)); }
+            constexpr ALWAYS_INLINE int GetShareableInteger()               const { return static_cast<int>(this->GetBits(8, 2)); }
+            constexpr ALWAYS_INLINE int GetPageAttributeInteger()           const { return static_cast<int>(this->GetBits(2, 3)); }
+            constexpr ALWAYS_INLINE bool IsReadOnly()                       const { return this->GetBits(7, 1) != 0; }
+            constexpr ALWAYS_INLINE bool IsUserAccessible()                 const { return this->GetBits(6, 1) != 0; }
+            constexpr ALWAYS_INLINE bool IsNonSecure()                      const { return this->GetBits(5, 1) != 0; }
+
+            constexpr ALWAYS_INLINE u64 GetTestTableMask()                  const { return (m_attributes & ExtensionFlag_TestTableMask); }
+
+            constexpr ALWAYS_INLINE bool IsBlock()                          const { return (m_attributes & ExtensionFlag_TestTableMask) == ExtensionFlag_Valid; }
+            constexpr ALWAYS_INLINE bool IsPage()                           const { return (m_attributes & ExtensionFlag_TestTableMask) == ExtensionFlag_TestTableMask; }
+            constexpr ALWAYS_INLINE bool IsTable()                          const { return (m_attributes & ExtensionFlag_TestTableMask) == 2; }
+            constexpr ALWAYS_INLINE bool IsEmpty()                          const { return (m_attributes & ExtensionFlag_TestTableMask) == 0; }
+
+            constexpr ALWAYS_INLINE KPhysicalAddress GetTable()             const { return this->SelectBits(12, 36); }
+
+            constexpr ALWAYS_INLINE bool IsMappedBlock()                    const { return this->GetBits(0, 2) == 1; }
+            constexpr ALWAYS_INLINE bool IsMappedTable()                    const { return this->GetBits(0, 2) == 3; }
+            constexpr ALWAYS_INLINE bool IsMappedEmpty()                    const { return this->GetBits(0, 2) == 0; }
+            constexpr ALWAYS_INLINE bool IsMapped()                         const { return this->GetBits(0, 1) != 0; }
+
+            constexpr ALWAYS_INLINE decltype(auto) SetUserExecuteNever(bool en)       { this->SetBit(54, en); return *this; }
+            constexpr ALWAYS_INLINE decltype(auto) SetPrivilegedExecuteNever(bool en) { this->SetBit(53, en); return *this; }
+            constexpr ALWAYS_INLINE decltype(auto) SetContiguous(bool en)             { this->SetBit(52, en); return *this; }
+            constexpr ALWAYS_INLINE decltype(auto) SetGlobal(bool en)                 { this->SetBit(11, !en); return *this; }
+            constexpr ALWAYS_INLINE decltype(auto) SetAccessFlag(AccessFlag f)        { this->SetBitsDirect(10, 1, f); return *this; }
+            constexpr ALWAYS_INLINE decltype(auto) SetShareable(Shareable s)          { this->SetBitsDirect(8, 2, s); return *this; }
+            constexpr ALWAYS_INLINE decltype(auto) SetReadOnly(bool en)               { this->SetBit(7, en); return *this; }
+            constexpr ALWAYS_INLINE decltype(auto) SetUserAccessible(bool en)         { this->SetBit(6, en); return *this; }
+            constexpr ALWAYS_INLINE decltype(auto) SetPageAttribute(PageAttribute a)  { this->SetBitsDirect(2, 3, a); return *this; }
+            constexpr ALWAYS_INLINE decltype(auto) SetMapped(bool m)                  { static_assert(static_cast<u64>(MappingFlag_Mapped == (1 << 0))); this->SetBit(0, m); return *this; }
+
+            constexpr ALWAYS_INLINE size_t GetTableReferenceCount() const { return this->GetBits(2, 10); }
+            constexpr ALWAYS_INLINE decltype(auto) SetTableReferenceCount(size_t num) { this->SetBits(2, 10, num); return *this; }
+
+            constexpr ALWAYS_INLINE decltype(auto) OpenTableReferences(size_t num) { MESOSPHERE_ASSERT(this->GetTableReferenceCount() + num <= BlocksPerTable + 1); return this->SetTableReferenceCount(this->GetTableReferenceCount() + num); }
+            constexpr ALWAYS_INLINE decltype(auto) CloseTableReferences(size_t num) { MESOSPHERE_ASSERT(this->GetTableReferenceCount() >= num); return this->SetTableReferenceCount(this->GetTableReferenceCount() - num); }
+
+            constexpr ALWAYS_INLINE decltype(auto) SetValid() { MESOSPHERE_ASSERT((m_attributes & ExtensionFlag_Valid) == 0); m_attributes |= ExtensionFlag_Valid; return *this; }
+
+            constexpr ALWAYS_INLINE u64 GetEntryTemplateForMerge() const {
+                constexpr u64 BaseMask = (0xFFFF000000000FFFul & ~static_cast<u64>((0x1ul << 52) | ExtensionFlag_TestTableMask | ExtensionFlag_DisableMergeHead | ExtensionFlag_DisableMergeHeadAndBody | ExtensionFlag_DisableMergeTail));
+                return m_attributes & BaseMask;
+            }
+
+            constexpr ALWAYS_INLINE bool IsForMerge(u64 attr) const {
+                constexpr u64 BaseMaskForMerge = ~static_cast<u64>(ExtensionFlag_DisableMergeHead | ExtensionFlag_DisableMergeHeadAndBody | ExtensionFlag_DisableMergeTail);
+                return (m_attributes & BaseMaskForMerge) == attr;
+            }
+
+            static constexpr ALWAYS_INLINE u64 GetEntryTemplateForSeparateContiguousMask(size_t idx) {
+                constexpr u64 BaseMask = (0xFFFF000000000FFFul & ~static_cast<u64>((0x1ul << 52) | ExtensionFlag_DisableMergeHead | ExtensionFlag_DisableMergeHeadAndBody | ExtensionFlag_DisableMergeTail));
+                if (idx == 0) {
+                    return BaseMask | ExtensionFlag_DisableMergeHead | ExtensionFlag_DisableMergeHeadAndBody;
+                } else if (idx < BlocksPerContiguousBlock - 1) {
+                    return BaseMask;
+                } else {
+                    return BaseMask | ExtensionFlag_DisableMergeTail;
+                }
+            }
+
+            constexpr ALWAYS_INLINE u64 GetEntryTemplateForSeparateContiguous(size_t idx) const {
+                return m_attributes & GetEntryTemplateForSeparateContiguousMask(idx);
+            }
+
+            static constexpr ALWAYS_INLINE u64 GetEntryTemplateForSeparateMask(size_t idx) {
+                constexpr u64 BaseMask = (0xFFFF000000000FFFul & ~static_cast<u64>((0x1ul << 52) | ExtensionFlag_TestTableMask | ExtensionFlag_DisableMergeHead | ExtensionFlag_DisableMergeHeadAndBody | ExtensionFlag_DisableMergeTail));
+                if (idx == 0) {
+                    return BaseMask | ExtensionFlag_DisableMergeHead | ExtensionFlag_DisableMergeHeadAndBody;
+                } else if (idx < BlocksPerContiguousBlock) {
+                    return BaseMask | ExtensionFlag_DisableMergeHeadAndBody;
+                } else if (idx < BlocksPerTable - 1) {
+                    return BaseMask;
+                } else {
+                    return BaseMask | ExtensionFlag_DisableMergeTail;
+                }
+            }
+
+            constexpr ALWAYS_INLINE u64 GetEntryTemplateForSeparate(size_t idx) const {
+                return m_attributes & GetEntryTemplateForSeparateMask(idx);
+            }
+
+            constexpr ALWAYS_INLINE u64 GetRawAttributesUnsafe() const {
+                return m_attributes;
+            }
+
+            constexpr ALWAYS_INLINE u64 GetRawAttributesUnsafeForSwap() const {
+                return m_attributes;
+            }
+        protected:
+            constexpr ALWAYS_INLINE u64 GetRawAttributes() const {
+                return m_attributes;
+            }
+    };
+
+    static_assert(sizeof(PageTableEntry) == sizeof(u64));
+
+    constexpr inline PageTableEntry InvalidPageTableEntry = PageTableEntry(PageTableEntry::InvalidTag{});
+
+    constexpr inline size_t MaxPageTableEntries = PageSize / sizeof(PageTableEntry);
+
+    class L1PageTableEntry : public PageTableEntry {
+        public:
+            constexpr explicit ALWAYS_INLINE L1PageTableEntry(InvalidTag) : PageTableEntry(InvalidTag{}) { /* ... */ }
+
+            constexpr explicit ALWAYS_INLINE L1PageTableEntry(TableTag, KPhysicalAddress phys_addr, bool pxn)
+                : PageTableEntry((0x3ul << 60) | (static_cast<u64>(pxn) << 59) | GetInteger(phys_addr) | 0x3)
+            {
+                /* ... */
+            }
+
+            constexpr explicit ALWAYS_INLINE L1PageTableEntry(TableTag, KPhysicalAddress phys_addr, bool is_kernel, bool pxn)
+                : PageTableEntry(((is_kernel ? 0x3ul : 0) << 60) | (static_cast<u64>(pxn) << 59) | GetInteger(phys_addr) | 0x3)
+            {
+                /* ... */
+            }
+
+            constexpr explicit ALWAYS_INLINE L1PageTableEntry(BlockTag, KPhysicalAddress phys_addr, const PageTableEntry &attr, u8 sw_reserved_bits, bool contig)
+                : PageTableEntry(attr, (static_cast<u64>(sw_reserved_bits) << 55) | (static_cast<u64>(contig) << 52) | GetInteger(phys_addr) | 0x1)
+            {
+                /* ... */
+            }
+
+            constexpr ALWAYS_INLINE KPhysicalAddress GetBlock() const {
+                return this->SelectBits(30, 18);
+            }
+
+            constexpr ALWAYS_INLINE KPhysicalAddress GetTable() const {
+                return this->SelectBits(12, 36);
+            }
+
+            constexpr ALWAYS_INLINE bool GetTable(KPhysicalAddress &out) const {
+                if (this->IsTable()) {
+                    out = this->GetTable();
+                    return true;
+                } else {
+                    return false;
+                }
+            }
+
+            static constexpr ALWAYS_INLINE u64 GetEntryTemplateForL2BlockMask(size_t idx) {
+                constexpr u64 BaseMask = (0xFFF0000000000FFFul & ~static_cast<u64>((0x1ul << 52) | ExtensionFlag_TestTableMask | ExtensionFlag_DisableMergeHead | ExtensionFlag_DisableMergeHeadAndBody | ExtensionFlag_DisableMergeTail));
+                if (idx == 0) {
+                    return BaseMask | ExtensionFlag_DisableMergeHead | ExtensionFlag_DisableMergeHeadAndBody;
+                } else if (idx < L2ContiguousBlockSize / L2BlockSize) {
+                    return BaseMask | ExtensionFlag_DisableMergeHeadAndBody;
+                } else if (idx < (L1BlockSize - L2ContiguousBlockSize) / L2BlockSize) {
+                    return BaseMask;
+                } else {
+                    return BaseMask | ExtensionFlag_DisableMergeTail;
+                }
+            }
+
+            constexpr ALWAYS_INLINE u64 GetEntryTemplateForL2Block(size_t idx) const {
+                return m_attributes & GetEntryTemplateForL2BlockMask(idx);
+            }
+
+            constexpr ALWAYS_INLINE bool IsCompatibleWithAttribute(const PageTableEntry &rhs, u8 sw_reserved_bits, bool contig) const {
+                /* Check whether this has the same permission/etc as the desired attributes. */
+                return L1PageTableEntry(BlockTag{}, this->GetBlock(), rhs, sw_reserved_bits, contig).GetRawAttributes() == this->GetRawAttributes();
+            }
+    };
+
+    class L2PageTableEntry : public PageTableEntry {
+        public:
+            constexpr explicit ALWAYS_INLINE L2PageTableEntry(InvalidTag) : PageTableEntry(InvalidTag{}) { /* ... */ }
+
+            constexpr explicit ALWAYS_INLINE L2PageTableEntry(TableTag, KPhysicalAddress phys_addr, bool pxn)
+                : PageTableEntry((0x3ul << 60) | (static_cast<u64>(pxn) << 59) | GetInteger(phys_addr) | 0x3)
+            {
+                /* ... */
+            }
+
+            constexpr explicit ALWAYS_INLINE L2PageTableEntry(TableTag, KPhysicalAddress phys_addr, bool is_kernel, bool pxn)
+                : PageTableEntry(((is_kernel ? 0x3ul : 0) << 60) | (static_cast<u64>(pxn) << 59) | GetInteger(phys_addr) | 0x3)
+            {
+                /* ... */
+            }
+
+            constexpr explicit ALWAYS_INLINE L2PageTableEntry(BlockTag, KPhysicalAddress phys_addr, const PageTableEntry &attr, u8 sw_reserved_bits, bool contig)
+                : PageTableEntry(attr, (static_cast<u64>(sw_reserved_bits) << 55) | (static_cast<u64>(contig) << 52) | GetInteger(phys_addr) | 0x1)
+            {
+                /* ... */
+            }
+
+            constexpr ALWAYS_INLINE KPhysicalAddress GetBlock() const {
+                return this->SelectBits(21, 27);
+            }
+
+            constexpr ALWAYS_INLINE KPhysicalAddress GetTable() const {
+                return this->SelectBits(12, 36);
+            }
+
+            constexpr ALWAYS_INLINE bool GetTable(KPhysicalAddress &out) const {
+                if (this->IsTable()) {
+                    out = this->GetTable();
+                    return true;
+                } else {
+                    return false;
+                }
+            }
+
+            static constexpr ALWAYS_INLINE u64 GetEntryTemplateForL2BlockMask(size_t idx) {
+                constexpr u64 BaseMask = (0xFFF0000000000FFFul & ~static_cast<u64>((0x1ul << 52) | ExtensionFlag_TestTableMask | ExtensionFlag_DisableMergeHead | ExtensionFlag_DisableMergeHeadAndBody | ExtensionFlag_DisableMergeTail));
+                if (idx == 0) {
+                    return BaseMask | ExtensionFlag_DisableMergeHead | ExtensionFlag_DisableMergeHeadAndBody;
+                } else if (idx < (L2ContiguousBlockSize / L2BlockSize) - 1) {
+                    return BaseMask;
+                } else {
+                    return BaseMask | ExtensionFlag_DisableMergeTail;
+                }
+            }
+
+            constexpr ALWAYS_INLINE u64 GetEntryTemplateForL2Block(size_t idx) const {
+                return m_attributes & GetEntryTemplateForL2BlockMask(idx);
+            }
+
+            static constexpr ALWAYS_INLINE u64 GetEntryTemplateForL3BlockMask(size_t idx) {
+                constexpr u64 BaseMask = (0xFFF0000000000FFFul & ~static_cast<u64>((0x1ul << 52) | ExtensionFlag_TestTableMask | ExtensionFlag_DisableMergeHead | ExtensionFlag_DisableMergeHeadAndBody | ExtensionFlag_DisableMergeTail));
+                if (idx == 0) {
+                    return BaseMask | ExtensionFlag_DisableMergeHead | ExtensionFlag_DisableMergeHeadAndBody;
+                } else if (idx < L3ContiguousBlockSize / L3BlockSize) {
+                    return BaseMask | ExtensionFlag_DisableMergeHeadAndBody;
+                } else if (idx < (L2BlockSize - L3ContiguousBlockSize) / L3BlockSize) {
+                    return BaseMask;
+                } else {
+                    return BaseMask | ExtensionFlag_DisableMergeTail;
+                }
+            }
+
+            constexpr ALWAYS_INLINE u64 GetEntryTemplateForL3Block(size_t idx) const {
+                return m_attributes & GetEntryTemplateForL3BlockMask(idx);
+            }
+
+            constexpr ALWAYS_INLINE bool IsCompatibleWithAttribute(const PageTableEntry &rhs, u8 sw_reserved_bits, bool contig) const {
+                /* Check whether this has the same permission/etc as the desired attributes. */
+                return L2PageTableEntry(BlockTag{}, this->GetBlock(), rhs, sw_reserved_bits, contig).GetRawAttributes() == this->GetRawAttributes();
+            }
+    };
+
+    class L3PageTableEntry : public PageTableEntry {
+        public:
+            constexpr explicit ALWAYS_INLINE L3PageTableEntry(InvalidTag) : PageTableEntry(InvalidTag{}) { /* ... */ }
+
+            constexpr explicit ALWAYS_INLINE L3PageTableEntry(BlockTag, KPhysicalAddress phys_addr, const PageTableEntry &attr, u8 sw_reserved_bits, bool contig)
+                : PageTableEntry(attr, (static_cast<u64>(sw_reserved_bits) << 55) | (static_cast<u64>(contig) << 52) | GetInteger(phys_addr) | 0x3)
+            {
+                /* ... */
+            }
+
+            constexpr ALWAYS_INLINE bool IsBlock() const { return (GetRawAttributes() & ExtensionFlag_TestTableMask) == ExtensionFlag_TestTableMask; }
+            constexpr ALWAYS_INLINE bool IsMappedBlock() const { return this->GetBits(0, 2) == 3; }
+
+            constexpr ALWAYS_INLINE KPhysicalAddress GetBlock() const {
+                return this->SelectBits(12, 36);
+            }
+
+            static constexpr ALWAYS_INLINE u64 GetEntryTemplateForL3BlockMask(size_t idx) {
+                constexpr u64 BaseMask = (0xFFF0000000000FFFul & ~static_cast<u64>((0x1ul << 52) | ExtensionFlag_TestTableMask | ExtensionFlag_DisableMergeHead | ExtensionFlag_DisableMergeHeadAndBody | ExtensionFlag_DisableMergeTail));
+                if (idx == 0) {
+                    return BaseMask | ExtensionFlag_DisableMergeHead | ExtensionFlag_DisableMergeHeadAndBody;
+                } else if (idx < (L3ContiguousBlockSize / L3BlockSize) - 1) {
+                    return BaseMask;
+                } else {
+                    return BaseMask | ExtensionFlag_DisableMergeTail;
+                }
+            }
+
+            constexpr ALWAYS_INLINE u64 GetEntryTemplateForL3Block(size_t idx) const {
+                return m_attributes & GetEntryTemplateForL3BlockMask(idx);
+            }
+
+            constexpr ALWAYS_INLINE bool IsCompatibleWithAttribute(const PageTableEntry &rhs, u8 sw_reserved_bits, bool contig) const {
+                /* Check whether this has the same permission/etc as the desired attributes. */
+                return L3PageTableEntry(BlockTag{}, this->GetBlock(), rhs, sw_reserved_bits, contig).GetRawAttributes() == this->GetRawAttributes();
+            }
+    };
+
+    constexpr inline L1PageTableEntry InvalidL1PageTableEntry = L1PageTableEntry(PageTableEntry::InvalidTag{});
+    constexpr inline L2PageTableEntry InvalidL2PageTableEntry = L2PageTableEntry(PageTableEntry::InvalidTag{});
+    constexpr inline L3PageTableEntry InvalidL3PageTableEntry = L3PageTableEntry(PageTableEntry::InvalidTag{});
+
+}

Source/Atmosphere/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table_impl.hpp

@@ -0,0 +1,161 @@
+/*
+ * Copyright (c) Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_select_cpu.hpp>
+#include <mesosphere/kern_k_typed_address.hpp>
+#include <mesosphere/kern_k_memory_layout.hpp>
+#include <mesosphere/arch/arm64/kern_k_page_table_entry.hpp>
+
+namespace ams::kern::arch::arm64 {
+
+    class KPageTableImpl {
+        NON_COPYABLE(KPageTableImpl);
+        NON_MOVEABLE(KPageTableImpl);
+        public:
+            struct TraversalEntry {
+                KPhysicalAddress phys_addr;
+                size_t block_size;
+                u8 sw_reserved_bits;
+                u8 attr;
+
+                constexpr bool IsHeadMergeDisabled() const { return (this->sw_reserved_bits & PageTableEntry::SoftwareReservedBit_DisableMergeHead) != 0; }
+                constexpr bool IsHeadAndBodyMergeDisabled() const { return (this->sw_reserved_bits & PageTableEntry::SoftwareReservedBit_DisableMergeHeadAndBody) != 0; }
+                constexpr bool IsTailMergeDisabled() const { return (this->sw_reserved_bits & PageTableEntry::SoftwareReservedBit_DisableMergeHeadTail) != 0; }
+            };
+
+            enum EntryLevel : u32 {
+                EntryLevel_L3    = 0,
+                EntryLevel_L2    = 1,
+                EntryLevel_L1    = 2,
+                EntryLevel_Count = 3,
+            };
+
+            struct TraversalContext {
+                PageTableEntry *level_entries[EntryLevel_Count];
+                EntryLevel level;
+                bool is_contiguous;
+            };
+
+            using EntryUpdatedCallback = void (*)(const void *);
+        private:
+            static constexpr size_t PageBits  = util::CountTrailingZeros(PageSize);
+            static constexpr size_t NumLevels = 3;
+            static constexpr size_t LevelBits = 9;
+            static_assert(NumLevels > 0);
+
+            template<size_t Offset, size_t Count>
+            static constexpr ALWAYS_INLINE u64 GetBits(u64 value) {
+                return (value >> Offset) & ((1ul << Count) - 1);
+            }
+
+            static constexpr ALWAYS_INLINE u64 GetBits(u64 value, size_t offset, size_t count) {
+                return (value >> offset) & ((1ul << count) - 1);
+            }
+
+            template<size_t Offset, size_t Count>
+            static constexpr ALWAYS_INLINE u64 SelectBits(u64 value) {
+                return value & (((1ul << Count) - 1) << Offset);
+            }
+
+            static constexpr ALWAYS_INLINE u64 SelectBits(u64 value, size_t offset, size_t count) {
+                return value & (((1ul << count) - 1) << offset);
+            }
+
+            static constexpr ALWAYS_INLINE uintptr_t GetL0Index(KProcessAddress addr) { return GetBits<PageBits + LevelBits * (NumLevels - 0), LevelBits>(GetInteger(addr)); }
+            static constexpr ALWAYS_INLINE uintptr_t GetL1Index(KProcessAddress addr) { return GetBits<PageBits + LevelBits * (NumLevels - 1), LevelBits>(GetInteger(addr)); }
+            static constexpr ALWAYS_INLINE uintptr_t GetL2Index(KProcessAddress addr) { return GetBits<PageBits + LevelBits * (NumLevels - 2), LevelBits>(GetInteger(addr)); }
+            static constexpr ALWAYS_INLINE uintptr_t GetL3Index(KProcessAddress addr) { return GetBits<PageBits + LevelBits * (NumLevels - 3), LevelBits>(GetInteger(addr)); }
+
+            static constexpr ALWAYS_INLINE uintptr_t GetL1Offset(KProcessAddress addr) { return GetBits<0, PageBits + LevelBits * (NumLevels - 1)>(GetInteger(addr)); }
+            static constexpr ALWAYS_INLINE uintptr_t GetL2Offset(KProcessAddress addr) { return GetBits<0, PageBits + LevelBits * (NumLevels - 2)>(GetInteger(addr)); }
+            static constexpr ALWAYS_INLINE uintptr_t GetL3Offset(KProcessAddress addr) { return GetBits<0, PageBits + LevelBits * (NumLevels - 3)>(GetInteger(addr)); }
+            static constexpr ALWAYS_INLINE uintptr_t GetContiguousL1Offset(KProcessAddress addr) { return GetBits<0, PageBits + LevelBits * (NumLevels - 1) + 4>(GetInteger(addr)); }
+            static constexpr ALWAYS_INLINE uintptr_t GetContiguousL2Offset(KProcessAddress addr) { return GetBits<0, PageBits + LevelBits * (NumLevels - 2) + 4>(GetInteger(addr)); }
+            static constexpr ALWAYS_INLINE uintptr_t GetContiguousL3Offset(KProcessAddress addr) { return GetBits<0, PageBits + LevelBits * (NumLevels - 3) + 4>(GetInteger(addr)); }
+
+            static constexpr ALWAYS_INLINE uintptr_t GetBlock(const PageTableEntry *pte, EntryLevel level) { return SelectBits(pte->GetRawAttributesUnsafe(), PageBits + LevelBits * level, LevelBits * (NumLevels + 1 - level)); }
+            static constexpr ALWAYS_INLINE uintptr_t GetOffset(KProcessAddress addr, EntryLevel level) { return GetBits(GetInteger(addr), 0, PageBits + LevelBits * level); }
+
+            static ALWAYS_INLINE KVirtualAddress GetPageTableVirtualAddress(KPhysicalAddress addr) {
+                return KMemoryLayout::GetLinearVirtualAddress(addr);
+            }
+        public:
+            static constexpr ALWAYS_INLINE uintptr_t GetLevelIndex(KProcessAddress addr, EntryLevel level) { return GetBits(GetInteger(addr), PageBits + LevelBits * level, LevelBits); }
+        private:
+            L1PageTableEntry *m_table;
+            bool m_is_kernel;
+            u32  m_num_entries;
+        public:
+            ALWAYS_INLINE KVirtualAddress GetTableEntry(KVirtualAddress table, size_t index) const {
+                return table + index * sizeof(PageTableEntry);
+            }
+
+            ALWAYS_INLINE L1PageTableEntry *GetL1Entry(KProcessAddress address) const {
+                return GetPointer<L1PageTableEntry>(GetTableEntry(KVirtualAddress(m_table), GetL1Index(address) & (m_num_entries - 1)));
+            }
+
+            ALWAYS_INLINE L2PageTableEntry *GetL2EntryFromTable(KVirtualAddress table, KProcessAddress address) const {
+                return GetPointer<L2PageTableEntry>(GetTableEntry(table, GetL2Index(address)));
+            }
+
+            ALWAYS_INLINE L2PageTableEntry *GetL2Entry(const L1PageTableEntry *entry, KProcessAddress address) const {
+                return GetL2EntryFromTable(KMemoryLayout::GetLinearVirtualAddress(entry->GetTable()), address);
+            }
+
+            ALWAYS_INLINE L3PageTableEntry *GetL3EntryFromTable(KVirtualAddress table, KProcessAddress address) const {
+                return GetPointer<L3PageTableEntry>(GetTableEntry(table, GetL3Index(address)));
+            }
+
+            ALWAYS_INLINE L3PageTableEntry *GetL3Entry(const L2PageTableEntry *entry, KProcessAddress address) const {
+                return GetL3EntryFromTable(KMemoryLayout::GetLinearVirtualAddress(entry->GetTable()), address);
+            }
+
+            static constexpr size_t GetBlockSize(EntryLevel level, bool contiguous = false) {
+                return 1 << (PageBits + LevelBits * level + 4 * contiguous);
+            }
+        public:
+            constexpr explicit KPageTableImpl(util::ConstantInitializeTag) : m_table(), m_is_kernel(), m_num_entries() { /* ... */ }
+
+            explicit KPageTableImpl() { /* ... */ }
+
+            size_t GetNumL1Entries() const { return m_num_entries; }
+
+            NOINLINE void InitializeForKernel(void *tb, KVirtualAddress start, KVirtualAddress end);
+            NOINLINE void InitializeForProcess(void *tb, KVirtualAddress start, KVirtualAddress end);
+            L1PageTableEntry *Finalize();
+
+            void Dump(uintptr_t start, size_t size) const;
+            size_t CountPageTables() const;
+
+            bool BeginTraversal(TraversalEntry *out_entry, TraversalContext *out_context, KProcessAddress address) const;
+            bool ContinueTraversal(TraversalEntry *out_entry, TraversalContext *context) const;
+
+            bool GetPhysicalAddress(KPhysicalAddress *out, KProcessAddress virt_addr) const;
+
+            static bool MergePages(KVirtualAddress *out, TraversalContext *context, EntryUpdatedCallback on_entry_updated, const void *pt);
+            void SeparatePages(TraversalEntry *entry, TraversalContext *context, KProcessAddress address, PageTableEntry *pte, EntryUpdatedCallback on_entry_updated, const void *pt) const;
+
+            KProcessAddress GetAddressForContext(const TraversalContext *context) const {
+                KProcessAddress addr = m_is_kernel ? static_cast<uintptr_t>(-GetBlockSize(EntryLevel_L1)) * m_num_entries : 0;
+                for (u32 level = context->level; level <= EntryLevel_L1; ++level) {
+                    addr += ((reinterpret_cast<uintptr_t>(context->level_entries[level]) / sizeof(PageTableEntry)) & (BlocksPerTable - 1)) << (PageBits + LevelBits * level);
+                }
+                return addr;
+            }
+    };
+
+}

Source/Atmosphere/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_process_page_table.hpp

@@ -0,0 +1,348 @@
+/*
+ * Copyright (c) Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/arch/arm64/kern_k_page_table.hpp>
+
+namespace ams::kern::arch::arm64 {
+
+    class KProcessPageTable {
+        private:
+            KPageTable m_page_table;
+        public:
+            void Activate(size_t process_index, u64 id) {
+                /* Activate the page table with the specified contextidr. */
+                m_page_table.ActivateProcess(process_index, id);
+            }
+
+            Result Initialize(ams::svc::CreateProcessFlag flags, bool from_back, KMemoryManager::Pool pool, KProcessAddress code_address, size_t code_size, KSystemResource *system_resource, KResourceLimit *resource_limit, size_t process_index) {
+                R_RETURN(m_page_table.InitializeForProcess(flags, from_back, pool, code_address, code_size, system_resource, resource_limit, process_index));
+            }
+
+            void Finalize() { m_page_table.Finalize(); }
+
+            ALWAYS_INLINE KScopedLightLock AcquireDeviceMapLock() {
+                return m_page_table.AcquireDeviceMapLock();
+            }
+
+            Result SetMemoryPermission(KProcessAddress addr, size_t size, ams::svc::MemoryPermission perm) {
+                R_RETURN(m_page_table.SetMemoryPermission(addr, size, perm));
+            }
+
+            Result SetProcessMemoryPermission(KProcessAddress addr, size_t size, ams::svc::MemoryPermission perm) {
+                R_RETURN(m_page_table.SetProcessMemoryPermission(addr, size, perm));
+            }
+
+            Result SetMemoryAttribute(KProcessAddress addr, size_t size, u32 mask, u32 attr) {
+                R_RETURN(m_page_table.SetMemoryAttribute(addr, size, mask, attr));
+            }
+
+            Result SetHeapSize(KProcessAddress *out, size_t size) {
+                R_RETURN(m_page_table.SetHeapSize(out, size));
+            }
+
+            Result SetMaxHeapSize(size_t size) {
+                R_RETURN(m_page_table.SetMaxHeapSize(size));
+            }
+
+            Result QueryInfo(KMemoryInfo *out_info, ams::svc::PageInfo *out_page_info, KProcessAddress addr) const {
+                R_RETURN(m_page_table.QueryInfo(out_info, out_page_info, addr));
+            }
+
+            Result QueryPhysicalAddress(ams::svc::PhysicalMemoryInfo *out, KProcessAddress address) const {
+                R_RETURN(m_page_table.QueryPhysicalAddress(out, address));
+            }
+
+            Result QueryStaticMapping(KProcessAddress *out, KPhysicalAddress address, size_t size) const {
+                R_RETURN(m_page_table.QueryStaticMapping(out, address, size));
+            }
+
+            Result QueryIoMapping(KProcessAddress *out, KPhysicalAddress address, size_t size) const {
+                R_RETURN(m_page_table.QueryIoMapping(out, address, size));
+            }
+
+            Result MapMemory(KProcessAddress dst_address, KProcessAddress src_address, size_t size) {
+                R_RETURN(m_page_table.MapMemory(dst_address, src_address, size));
+            }
+
+            Result UnmapMemory(KProcessAddress dst_address, KProcessAddress src_address, size_t size) {
+                R_RETURN(m_page_table.UnmapMemory(dst_address, src_address, size));
+            }
+
+            Result MapCodeMemory(KProcessAddress dst_address, KProcessAddress src_address, size_t size) {
+                R_RETURN(m_page_table.MapCodeMemory(dst_address, src_address, size));
+            }
+
+            Result UnmapCodeMemory(KProcessAddress dst_address, KProcessAddress src_address, size_t size) {
+                R_RETURN(m_page_table.UnmapCodeMemory(dst_address, src_address, size));
+            }
+
+            Result MapIo(KPhysicalAddress phys_addr, size_t size, KMemoryPermission perm) {
+                R_RETURN(m_page_table.MapIo(phys_addr, size, perm));
+            }
+
+            Result MapIoRegion(KProcessAddress dst_address, KPhysicalAddress phys_addr, size_t size, ams::svc::MemoryMapping mapping, ams::svc::MemoryPermission perm) {
+                R_RETURN(m_page_table.MapIoRegion(dst_address, phys_addr, size, mapping, perm));
+            }
+
+            Result UnmapIoRegion(KProcessAddress dst_address, KPhysicalAddress phys_addr, size_t size, ams::svc::MemoryMapping mapping) {
+                R_RETURN(m_page_table.UnmapIoRegion(dst_address, phys_addr, size, mapping));
+            }
+
+            Result MapStatic(KPhysicalAddress phys_addr, size_t size, KMemoryPermission perm) {
+                R_RETURN(m_page_table.MapStatic(phys_addr, size, perm));
+            }
+
+            Result MapRegion(KMemoryRegionType region_type, KMemoryPermission perm) {
+                R_RETURN(m_page_table.MapRegion(region_type, perm));
+            }
+
+            Result MapInsecurePhysicalMemory(KProcessAddress address, size_t size) {
+                R_RETURN(m_page_table.MapInsecurePhysicalMemory(address, size));
+            }
+
+            Result UnmapInsecurePhysicalMemory(KProcessAddress address, size_t size) {
+                R_RETURN(m_page_table.UnmapInsecurePhysicalMemory(address, size));
+            }
+
+            Result MapPageGroup(KProcessAddress addr, const KPageGroup &pg, KMemoryState state, KMemoryPermission perm) {
+                R_RETURN(m_page_table.MapPageGroup(addr, pg, state, perm));
+            }
+
+            Result UnmapPageGroup(KProcessAddress address, const KPageGroup &pg, KMemoryState state) {
+                R_RETURN(m_page_table.UnmapPageGroup(address, pg, state));
+            }
+
+            Result MapPages(KProcessAddress *out_addr, size_t num_pages, size_t alignment, KPhysicalAddress phys_addr, KMemoryState state, KMemoryPermission perm) {
+                R_RETURN(m_page_table.MapPages(out_addr, num_pages, alignment, phys_addr, state, perm));
+            }
+
+            Result MapPages(KProcessAddress *out_addr, size_t num_pages, KMemoryState state, KMemoryPermission perm) {
+                R_RETURN(m_page_table.MapPages(out_addr, num_pages, state, perm));
+            }
+
+            Result MapPages(KProcessAddress address, size_t num_pages, KMemoryState state, KMemoryPermission perm) {
+                R_RETURN(m_page_table.MapPages(address, num_pages, state, perm));
+            }
+
+            Result UnmapPages(KProcessAddress addr, size_t num_pages, KMemoryState state) {
+                R_RETURN(m_page_table.UnmapPages(addr, num_pages, state));
+            }
+
+            Result MakeAndOpenPageGroup(KPageGroup *out, KProcessAddress address, size_t num_pages, u32 state_mask, u32 state, u32 perm_mask, u32 perm, u32 attr_mask, u32 attr) {
+                R_RETURN(m_page_table.MakeAndOpenPageGroup(out, address, num_pages, state_mask, state, perm_mask, perm, attr_mask, attr));
+            }
+
+            Result InvalidateProcessDataCache(KProcessAddress address, size_t size) {
+                R_RETURN(m_page_table.InvalidateProcessDataCache(address, size));
+            }
+
+            Result InvalidateCurrentProcessDataCache(KProcessAddress address, size_t size) {
+                R_RETURN(m_page_table.InvalidateCurrentProcessDataCache(address, size));
+            }
+
+            Result ReadDebugMemory(void *buffer, KProcessAddress address, size_t size, bool force_debug_prod) {
+                R_RETURN(m_page_table.ReadDebugMemory(buffer, address, size, force_debug_prod));
+            }
+
+            Result ReadDebugIoMemory(void *buffer, KProcessAddress address, size_t size, KMemoryState state) {
+                R_RETURN(m_page_table.ReadDebugIoMemory(buffer, address, size, state));
+            }
+
+            Result WriteDebugMemory(KProcessAddress address, const void *buffer, size_t size) {
+                R_RETURN(m_page_table.WriteDebugMemory(address, buffer, size));
+            }
+
+            Result WriteDebugIoMemory(KProcessAddress address, const void *buffer, size_t size, KMemoryState state) {
+                R_RETURN(m_page_table.WriteDebugIoMemory(address, buffer, size, state));
+            }
+
+            Result LockForMapDeviceAddressSpace(bool *out_is_io, KProcessAddress address, size_t size, KMemoryPermission perm, bool is_aligned, bool check_heap) {
+                R_RETURN(m_page_table.LockForMapDeviceAddressSpace(out_is_io, address, size, perm, is_aligned, check_heap));
+            }
+
+            Result LockForUnmapDeviceAddressSpace(KProcessAddress address, size_t size, bool check_heap) {
+                R_RETURN(m_page_table.LockForUnmapDeviceAddressSpace(address, size, check_heap));
+            }
+
+            Result UnlockForDeviceAddressSpace(KProcessAddress address, size_t size) {
+                R_RETURN(m_page_table.UnlockForDeviceAddressSpace(address, size));
+            }
+
+            Result UnlockForDeviceAddressSpacePartialMap(KProcessAddress address, size_t size) {
+                R_RETURN(m_page_table.UnlockForDeviceAddressSpacePartialMap(address, size));
+            }
+
+            Result OpenMemoryRangeForMapDeviceAddressSpace(KPageTableBase::MemoryRange *out, KProcessAddress address, size_t size, KMemoryPermission perm, bool is_aligned) {
+                R_RETURN(m_page_table.OpenMemoryRangeForMapDeviceAddressSpace(out, address, size, perm, is_aligned));
+            }
+
+            Result OpenMemoryRangeForUnmapDeviceAddressSpace(KPageTableBase::MemoryRange *out, KProcessAddress address, size_t size) {
+                R_RETURN(m_page_table.OpenMemoryRangeForUnmapDeviceAddressSpace(out, address, size));
+            }
+
+            Result LockForIpcUserBuffer(KPhysicalAddress *out, KProcessAddress address, size_t size) {
+                R_RETURN(m_page_table.LockForIpcUserBuffer(out, address, size));
+            }
+
+            Result UnlockForIpcUserBuffer(KProcessAddress address, size_t size) {
+                R_RETURN(m_page_table.UnlockForIpcUserBuffer(address, size));
+            }
+
+            Result LockForTransferMemory(KPageGroup *out, KProcessAddress address, size_t size, KMemoryPermission perm) {
+                R_RETURN(m_page_table.LockForTransferMemory(out, address, size, perm));
+            }
+
+            Result UnlockForTransferMemory(KProcessAddress address, size_t size, const KPageGroup &pg) {
+                R_RETURN(m_page_table.UnlockForTransferMemory(address, size, pg));
+            }
+
+            Result LockForCodeMemory(KPageGroup *out, KProcessAddress address, size_t size) {
+                R_RETURN(m_page_table.LockForCodeMemory(out, address, size));
+            }
+
+            Result UnlockForCodeMemory(KProcessAddress address, size_t size, const KPageGroup &pg) {
+                R_RETURN(m_page_table.UnlockForCodeMemory(address, size, pg));
+            }
+
+            Result OpenMemoryRangeForProcessCacheOperation(KPageTableBase::MemoryRange *out, KProcessAddress address, size_t size) {
+                R_RETURN(m_page_table.OpenMemoryRangeForProcessCacheOperation(out, address, size));
+            }
+
+            Result CopyMemoryFromLinearToUser(KProcessAddress dst_addr, size_t size, KProcessAddress src_addr, u32 src_state_mask, u32 src_state, KMemoryPermission src_test_perm, u32 src_attr_mask, u32 src_attr) {
+                R_RETURN(m_page_table.CopyMemoryFromLinearToUser(dst_addr, size, src_addr, src_state_mask, src_state, src_test_perm, src_attr_mask, src_attr));
+            }
+
+            Result CopyMemoryFromLinearToKernel(KProcessAddress dst_addr, size_t size, KProcessAddress src_addr, u32 src_state_mask, u32 src_state, KMemoryPermission src_test_perm, u32 src_attr_mask, u32 src_attr) {
+                R_RETURN(m_page_table.CopyMemoryFromLinearToKernel(dst_addr, size, src_addr, src_state_mask, src_state, src_test_perm, src_attr_mask, src_attr));
+            }
+
+            Result CopyMemoryFromUserToLinear(KProcessAddress dst_addr, size_t size, u32 dst_state_mask, u32 dst_state, KMemoryPermission dst_test_perm, u32 dst_attr_mask, u32 dst_attr, KProcessAddress src_addr) {
+                R_RETURN(m_page_table.CopyMemoryFromUserToLinear(dst_addr, size, dst_state_mask, dst_state, dst_test_perm, dst_attr_mask, dst_attr, src_addr));
+            }
+
+            Result CopyMemoryFromKernelToLinear(KProcessAddress dst_addr, size_t size, u32 dst_state_mask, u32 dst_state, KMemoryPermission dst_test_perm, u32 dst_attr_mask, u32 dst_attr, KProcessAddress src_addr) {
+                R_RETURN(m_page_table.CopyMemoryFromKernelToLinear(dst_addr, size, dst_state_mask, dst_state, dst_test_perm, dst_attr_mask, dst_attr, src_addr));
+            }
+
+            Result CopyMemoryFromHeapToHeap(KProcessPageTable &dst_page_table, KProcessAddress dst_addr, size_t size, u32 dst_state_mask, u32 dst_state, KMemoryPermission dst_test_perm, u32 dst_attr_mask, u32 dst_attr, KProcessAddress src_addr, u32 src_state_mask, u32 src_state, KMemoryPermission src_test_perm, u32 src_attr_mask, u32 src_attr) {
+                R_RETURN(m_page_table.CopyMemoryFromHeapToHeap(dst_page_table.m_page_table, dst_addr, size, dst_state_mask, dst_state, dst_test_perm, dst_attr_mask, dst_attr, src_addr, src_state_mask, src_state, src_test_perm, src_attr_mask, src_attr));
+            }
+
+            Result CopyMemoryFromHeapToHeapWithoutCheckDestination(KProcessPageTable &dst_page_table, KProcessAddress dst_addr, size_t size, u32 dst_state_mask, u32 dst_state, KMemoryPermission dst_test_perm, u32 dst_attr_mask, u32 dst_attr, KProcessAddress src_addr, u32 src_state_mask, u32 src_state, KMemoryPermission src_test_perm, u32 src_attr_mask, u32 src_attr) {
+                R_RETURN(m_page_table.CopyMemoryFromHeapToHeapWithoutCheckDestination(dst_page_table.m_page_table, dst_addr, size, dst_state_mask, dst_state, dst_test_perm, dst_attr_mask, dst_attr, src_addr, src_state_mask, src_state, src_test_perm, src_attr_mask, src_attr));
+            }
+
+            Result SetupForIpc(KProcessAddress *out_dst_addr, size_t size, KProcessAddress src_addr, KProcessPageTable &src_page_table, KMemoryPermission test_perm, KMemoryState dst_state, bool send) {
+                R_RETURN(m_page_table.SetupForIpc(out_dst_addr, size, src_addr, src_page_table.m_page_table, test_perm, dst_state, send));
+            }
+
+            Result CleanupForIpcServer(KProcessAddress address, size_t size, KMemoryState dst_state) {
+                R_RETURN(m_page_table.CleanupForIpcServer(address, size, dst_state));
+            }
+
+            Result CleanupForIpcClient(KProcessAddress address, size_t size, KMemoryState dst_state) {
+                R_RETURN(m_page_table.CleanupForIpcClient(address, size, dst_state));
+            }
+
+            Result MapPhysicalMemory(KProcessAddress address, size_t size) {
+                R_RETURN(m_page_table.MapPhysicalMemory(address, size));
+            }
+
+            Result UnmapPhysicalMemory(KProcessAddress address, size_t size) {
+                R_RETURN(m_page_table.UnmapPhysicalMemory(address, size));
+            }
+
+            Result MapPhysicalMemoryUnsafe(KProcessAddress address, size_t size) {
+                R_RETURN(m_page_table.MapPhysicalMemoryUnsafe(address, size));
+            }
+
+            Result UnmapPhysicalMemoryUnsafe(KProcessAddress address, size_t size) {
+                R_RETURN(m_page_table.UnmapPhysicalMemoryUnsafe(address, size));
+            }
+
+            Result UnmapProcessMemory(KProcessAddress dst_address, size_t size, KProcessPageTable &src_page_table, KProcessAddress src_address) {
+                R_RETURN(m_page_table.UnmapProcessMemory(dst_address, size, src_page_table.m_page_table, src_address));
+            }
+
+            void DumpMemoryBlocks() const {
+                return m_page_table.DumpMemoryBlocks();
+            }
+
+            void DumpPageTable() const {
+                return m_page_table.DumpPageTable();
+            }
+
+            size_t CountPageTables() const {
+                return m_page_table.CountPageTables();
+            }
+
+            bool GetPhysicalAddress(KPhysicalAddress *out, KProcessAddress address) const {
+                return m_page_table.GetPhysicalAddress(out, address);
+            }
+
+            bool Contains(KProcessAddress addr, size_t size) const { return m_page_table.Contains(addr, size); }
+
+            bool IsInAliasRegion(KProcessAddress addr, size_t size) const { return m_page_table.IsInAliasRegion(addr, size); }
+            bool IsInUnsafeAliasRegion(KProcessAddress addr, size_t size) const { return m_page_table.IsInUnsafeAliasRegion(addr, size); }
+
+            bool CanContain(KProcessAddress addr, size_t size, KMemoryState state) const { return m_page_table.CanContain(addr, size, state); }
+            bool CanContain(KProcessAddress addr, size_t size, ams::svc::MemoryState state) const { return m_page_table.CanContain(addr, size, state); }
+
+            KProcessAddress GetAddressSpaceStart()    const { return m_page_table.GetAddressSpaceStart(); }
+            KProcessAddress GetHeapRegionStart()      const { return m_page_table.GetHeapRegionStart(); }
+            KProcessAddress GetAliasRegionStart()     const { return m_page_table.GetAliasRegionStart(); }
+            KProcessAddress GetStackRegionStart()     const { return m_page_table.GetStackRegionStart(); }
+            KProcessAddress GetKernelMapRegionStart() const { return m_page_table.GetKernelMapRegionStart(); }
+            KProcessAddress GetAliasCodeRegionStart() const { return m_page_table.GetAliasCodeRegionStart(); }
+
+            size_t GetAddressSpaceSize()    const { return m_page_table.GetAddressSpaceSize(); }
+            size_t GetHeapRegionSize()      const { return m_page_table.GetHeapRegionSize(); }
+            size_t GetAliasRegionSize()     const { return m_page_table.GetAliasRegionSize(); }
+            size_t GetStackRegionSize()     const { return m_page_table.GetStackRegionSize(); }
+            size_t GetKernelMapRegionSize() const { return m_page_table.GetKernelMapRegionSize(); }
+            size_t GetAliasCodeRegionSize() const { return m_page_table.GetAliasCodeRegionSize(); }
+
+            size_t GetAliasRegionExtraSize() const { return m_page_table.GetAliasRegionExtraSize(); }
+
+            size_t GetNormalMemorySize() const { return m_page_table.GetNormalMemorySize(); }
+
+            size_t GetCodeSize() const { return m_page_table.GetCodeSize(); }
+            size_t GetCodeDataSize() const { return m_page_table.GetCodeDataSize(); }
+
+            size_t GetAliasCodeSize() const { return m_page_table.GetAliasCodeSize(); }
+            size_t GetAliasCodeDataSize() const { return m_page_table.GetAliasCodeDataSize(); }
+
+            u32 GetAllocateOption() const { return m_page_table.GetAllocateOption(); }
+
+            KPhysicalAddress GetHeapPhysicalAddress(KVirtualAddress address) const {
+                return m_page_table.GetHeapPhysicalAddress(address);
+            }
+
+            KVirtualAddress GetHeapVirtualAddress(KPhysicalAddress address) const {
+                return m_page_table.GetHeapVirtualAddress(address);
+            }
+
+            KBlockInfoManager *GetBlockInfoManager() {
+                return m_page_table.GetBlockInfoManager();
+            }
+
+            KPageTableBase &GetBasePageTable() {
+                return m_page_table;
+            }
+    };
+
+}

Source/Atmosphere/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_slab_heap_impl.hpp

@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <vapours.hpp>
+#include <mesosphere/kern_select_cpu.hpp>
+#include <mesosphere/kern_select_interrupt_manager.hpp>
+
+namespace ams::kern::arch::arm64 {
+
+    template<typename T>
+    concept SlabHeapNode = requires (T &t) {
+        { t.next } -> std::convertible_to<T *>;
+    };
+
+    ALWAYS_INLINE bool IsSlabAtomicValid() {
+        /* Without careful consideration, slab heaps atomics are vulnerable to */
+        /* the ABA problem, when doing compare and swap of node pointers. */
+        /* We resolve this by using the ARM exclusive monitor; we bundle the */
+        /* load and store of the relevant values into a single exclusive monitor */
+        /* hold, preventing the ABA problem. */
+        /* However, our assembly must do both a load and a store under a single */
+        /* hold, at different memory addresses. Considering the case where the */
+        /* addresses are distinct but resolve to the same cache set (by chance), */
+        /* we can note that under a 1-way associative (direct-mapped) cache */
+        /* we would have as a guarantee that the second access would evict the */
+        /* cache line from the first access, invalidating our exclusive monitor */
+        /* hold. Thus, we require that the cache is not 1-way associative, for */
+        /* our implementation to be correct. */
+        {
+            /* Disable interrupts. */
+            KScopedInterruptDisable di;
+
+            /* Select L1 cache. */
+            cpu::SetCsselrEl1(0);
+            cpu::InstructionMemoryBarrier();
+
+            /* Check that the L1 cache is not direct-mapped. */
+            return cpu::CacheSizeIdRegisterAccessor().GetAssociativity() != 0;
+        }
+    }
+
+    template<typename T> requires SlabHeapNode<T>
+    ALWAYS_INLINE T *AllocateFromSlabAtomic(T **head) {
+        u32 tmp;
+        T *node, *next;
+
+        __asm__ __volatile__(
+            "1:\n"
+            "    ldaxr  %[node], [%[head]]\n"
+            "    cbz    %[node], 2f\n"
+            "    ldr    %[next], [%[node]]\n"
+            "    stlxr  %w[tmp], %[next], [%[head]]\n"
+            "    cbnz   %w[tmp], 1b\n"
+            "2:\n"
+            : [tmp]"=&r"(tmp), [node]"=&r"(node), [next]"=&r"(next), [head]"+&r"(head)
+            :
+            : "cc", "memory"
+        );
+
+        return node;
+    }
+
+    template<typename T> requires SlabHeapNode<T>
+    ALWAYS_INLINE void FreeToSlabAtomic(T **head, T *node) {
+        u32 tmp;
+        T *next;
+
+        __asm__ __volatile__(
+            "1:\n"
+            "    ldaxr  %[next], [%[head]]\n"
+            "    str    %[next], [%[node]]\n"
+            "    stlxr  %w[tmp], %[node], [%[head]]\n"
+            "    cbnz   %w[tmp], 1b\n"
+            : [tmp]"=&r"(tmp), [node]"+&r"(node), [next]"=&r"(next), [head]"+&r"(head)
+            :
+            : "cc", "memory"
+        );
+    }
+
+}

Source/Atmosphere/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_spin_lock.hpp

@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <vapours.hpp>
+#include <mesosphere/kern_select_cpu.hpp>
+
+namespace ams::kern::arch::arm64 {
+
+    class KNotAlignedSpinLock {
+        private:
+            u32 m_packed_tickets;
+        public:
+            constexpr KNotAlignedSpinLock() : m_packed_tickets(0) { /* ... */ }
+
+            ALWAYS_INLINE void Lock() {
+                u32 tmp0, tmp1, tmp2;
+
+                __asm__ __volatile__(
+                    "    prfm   pstl1keep, %[m_packed_tickets]\n"
+                    "1:\n"
+                    "    ldaxr  %w[tmp0], %[m_packed_tickets]\n"
+                    "    add    %w[tmp2], %w[tmp0], #0x10000\n"
+                    "    stxr   %w[tmp1], %w[tmp2], %[m_packed_tickets]\n"
+                    "    cbnz   %w[tmp1], 1b\n"
+                    "    \n"
+                    "    and    %w[tmp1], %w[tmp0], #0xFFFF\n"
+                    "    cmp    %w[tmp1], %w[tmp0], lsr #16\n"
+                    "    b.eq   3f\n"
+                    "    sevl\n"
+                    "2:\n"
+                    "    wfe\n"
+                    "    ldaxrh %w[tmp1], %[m_packed_tickets]\n"
+                    "    cmp    %w[tmp1], %w[tmp0], lsr #16\n"
+                    "    b.ne   2b\n"
+                    "3:\n"
+                    : [tmp0]"=&r"(tmp0), [tmp1]"=&r"(tmp1), [tmp2]"=&r"(tmp2), [m_packed_tickets]"+Q"(m_packed_tickets)
+                    :
+                    : "cc", "memory"
+                );
+            }
+
+            ALWAYS_INLINE void Unlock() {
+                const u32 value = m_packed_tickets + 1;
+                __asm__ __volatile__(
+                    "    stlrh %w[value], %[m_packed_tickets]\n"
+                    : [m_packed_tickets]"+Q"(m_packed_tickets)
+                    : [value]"r"(value)
+                    : "memory"
+                );
+            }
+    };
+    static_assert(sizeof(KNotAlignedSpinLock) == sizeof(u32));
+
+    class KAlignedSpinLock {
+        private:
+            alignas(cpu::DataCacheLineSize) u16 m_current_ticket;
+            alignas(cpu::DataCacheLineSize) u16 m_next_ticket;
+        public:
+            constexpr KAlignedSpinLock() : m_current_ticket(0), m_next_ticket(0) { /* ... */ }
+
+            ALWAYS_INLINE void Lock() {
+                u32 tmp0, tmp1, got_lock;
+
+                __asm__ __volatile__(
+                    "    prfm   pstl1keep, %[m_next_ticket]\n"
+                    "1:\n"
+                    "    ldxrh  %w[tmp0], %[m_next_ticket]\n"
+                    "    add    %w[tmp1], %w[tmp0], #0x1\n"
+                    "    stxrh  %w[got_lock], %w[tmp1], %[m_next_ticket]\n"
+                    "    cbnz   %w[got_lock], 1b\n"
+                    "    \n"
+                    "    sevl\n"
+                    "2:\n"
+                    "    wfe\n"
+                    "    ldaxrh %w[tmp1], %[m_current_ticket]\n"
+                    "    cmp    %w[tmp1], %w[tmp0]\n"
+                    "    b.ne   2b\n"
+                    : [tmp0]"=&r"(tmp0), [tmp1]"=&r"(tmp1), [got_lock]"=&r"(got_lock), [m_next_ticket]"+Q"(m_next_ticket)
+                    : [m_current_ticket]"Q"(m_current_ticket)
+                    : "cc", "memory"
+                );
+            }
+
+            ALWAYS_INLINE void Unlock() {
+                const u32 value = m_current_ticket + 1;
+                __asm__ __volatile__(
+                    "    stlrh %w[value], %[m_current_ticket]\n"
+                    : [m_current_ticket]"+Q"(m_current_ticket)
+                    : [value]"r"(value)
+                    : "memory"
+                );
+            }
+    };
+    static_assert(sizeof(KAlignedSpinLock) == 2 * cpu::DataCacheLineSize);
+
+    using KSpinLock = KNotAlignedSpinLock;
+
+}

Source/Atmosphere/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_supervisor_page_table.hpp

@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_select_cpu.hpp>
+#include <mesosphere/arch/arm64/kern_k_page_table.hpp>
+
+namespace ams::kern::arch::arm64 {
+
+    class KSupervisorPageTable {
+        private:
+            KPageTable m_page_table;
+        public:
+            constexpr KSupervisorPageTable() : m_page_table(util::ConstantInitialize) { /* ... */ }
+
+            NOINLINE void Initialize(s32 core_id);
+
+            void Activate() {
+                m_page_table.ActivateKernel();
+            }
+
+            void ActivateForInit() {
+                this->Activate();
+
+                /* Invalidate entire TLB. */
+                cpu::InvalidateEntireTlb();
+            }
+
+            Result MapPages(KProcessAddress *out_addr, size_t num_pages, size_t alignment, KPhysicalAddress phys_addr, KProcessAddress region_start, size_t region_num_pages, KMemoryState state, KMemoryPermission perm) {
+                R_RETURN(m_page_table.MapPages(out_addr, num_pages, alignment, phys_addr, region_start, region_num_pages, state, perm));
+            }
+
+            Result UnmapPages(KProcessAddress address, size_t num_pages, KMemoryState state) {
+                R_RETURN(m_page_table.UnmapPages(address, num_pages, state));
+            }
+
+            Result MapPageGroup(KProcessAddress *out_addr, const KPageGroup &pg, KProcessAddress region_start, size_t region_num_pages, KMemoryState state, KMemoryPermission perm) {
+                R_RETURN(m_page_table.MapPageGroup(out_addr, pg, region_start, region_num_pages, state, perm));
+            }
+
+            Result UnmapPageGroup(KProcessAddress address, const KPageGroup &pg, KMemoryState state) {
+                R_RETURN(m_page_table.UnmapPageGroup(address, pg, state));
+            }
+
+            bool GetPhysicalAddress(KPhysicalAddress *out, KProcessAddress address) const {
+                return m_page_table.GetPhysicalAddress(out, address);
+            }
+
+            void DumpMemoryBlocks() const {
+                return m_page_table.DumpMemoryBlocks();
+            }
+
+            void DumpPageTable() const {
+                return m_page_table.DumpPageTable();
+            }
+
+            size_t CountPageTables() const {
+                return m_page_table.CountPageTables();
+            }
+    };
+
+}

Source/Atmosphere/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_thread_context.hpp

@@ -0,0 +1,316 @@
+/*
+ * Copyright (c) Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_k_typed_address.hpp>
+
+namespace ams::kern {
+
+    class KThread;
+
+}
+
+namespace ams::kern::arch::arm64 {
+
+    class KThreadContext {
+        public:
+            static constexpr size_t NumCalleeSavedRegisters    = (29 - 19) + 1;
+            static constexpr size_t NumCalleeSavedFpuRegisters = 8;
+            static constexpr size_t NumCallerSavedFpuRegisters = 24;
+            static constexpr size_t NumFpuRegisters            = NumCalleeSavedFpuRegisters + NumCallerSavedFpuRegisters;
+        public:
+            union CalleeSaveRegisters {
+                u64 registers[NumCalleeSavedRegisters];
+                struct {
+                    u64 x19;
+                    u64 x20;
+                    u64 x21;
+                    u64 x22;
+                    u64 x23;
+                    u64 x24;
+                    u64 x25;
+                    u64 x26;
+                    u64 x27;
+                    u64 x28;
+                    u64 x29;
+                };
+            };
+
+            union CalleeSaveFpu64Registers {
+                u128 v[NumCalleeSavedFpuRegisters];
+                struct {
+                    u128 q8;
+                    u128 q9;
+                    u128 q10;
+                    u128 q11;
+                    u128 q12;
+                    u128 q13;
+                    u128 q14;
+                    u128 q15;
+                };
+            };
+
+            union CalleeSaveFpu32Registers {
+                u128 v[NumCalleeSavedFpuRegisters / 2];
+                struct {
+                    u128 q4;
+                    u128 q5;
+                    u128 q6;
+                    u128 q7;
+                };
+            };
+
+            union CalleeSaveFpuRegisters {
+                CalleeSaveFpu64Registers fpu64;
+                CalleeSaveFpu32Registers fpu32;
+            };
+
+            union CallerSaveFpu64Registers {
+                u128 v[NumCallerSavedFpuRegisters];
+                struct {
+                    union {
+                        u128 v0_7[NumCallerSavedFpuRegisters / 3];
+                        struct {
+                            u128 q0;
+                            u128 q1;
+                            u128 q2;
+                            u128 q3;
+                            u128 q4;
+                            u128 q5;
+                            u128 q6;
+                            u128 q7;
+                        };
+                    };
+                    union {
+                        u128 v16_31[2 * NumCallerSavedFpuRegisters / 3];
+                        struct {
+                            u128 q16;
+                            u128 q17;
+                            u128 q18;
+                            u128 q19;
+                            u128 q20;
+                            u128 q21;
+                            u128 q22;
+                            u128 q23;
+                            u128 q24;
+                            u128 q25;
+                            u128 q26;
+                            u128 q27;
+                            u128 q28;
+                            u128 q29;
+                            u128 q30;
+                            u128 q31;
+                        };
+                    };
+                };
+            };
+
+            union CallerSaveFpu32Registers {
+                u128 v[NumCallerSavedFpuRegisters / 2];
+                struct {
+                    union {
+                        u128 v0_3[(NumCallerSavedFpuRegisters / 3) / 2];
+                        struct {
+                            u128 q0;
+                            u128 q1;
+                            u128 q2;
+                            u128 q3;
+                        };
+                    };
+                    union {
+                        u128 v8_15[(2 * NumCallerSavedFpuRegisters / 3) / 2];
+                        struct {
+                            u128 q8;
+                            u128 q9;
+                            u128 q10;
+                            u128 q11;
+                            u128 q12;
+                            u128 q13;
+                            u128 q14;
+                            u128 q15;
+                        };
+                    };
+                };
+            };
+
+            union CallerSaveFpuRegisters {
+                CallerSaveFpu64Registers fpu64;
+                CallerSaveFpu32Registers fpu32;
+            };
+        private:
+            CalleeSaveRegisters m_callee_saved;
+            u64 m_lr;
+            u64 m_sp;
+            u32 m_fpcr;
+            u32 m_fpsr;
+            alignas(0x10) CalleeSaveFpuRegisters m_callee_saved_fpu;
+            bool m_locked;
+        private:
+            static void RestoreFpuRegisters64(const KThreadContext &);
+            static void RestoreFpuRegisters32(const KThreadContext &);
+        public:
+            constexpr explicit KThreadContext(util::ConstantInitializeTag) : m_callee_saved(), m_lr(), m_sp(), m_fpcr(), m_fpsr(), m_callee_saved_fpu(), m_locked() { /* ... */ }
+            explicit KThreadContext() { /* ... */ }
+
+            Result Initialize(KVirtualAddress u_pc, KVirtualAddress k_sp, KVirtualAddress u_sp, uintptr_t arg, bool is_user, bool is_64_bit, bool is_main);
+
+            void SetArguments(uintptr_t arg0, uintptr_t arg1);
+
+            static void FpuContextSwitchHandler(KThread *thread);
+
+            u32 GetFpcr() const { return m_fpcr; }
+            u32 GetFpsr() const { return m_fpsr; }
+
+            void SetFpcr(u32 v) { m_fpcr = v; }
+            void SetFpsr(u32 v) { m_fpsr = v; }
+
+            void CloneFpuStatus();
+
+            const auto &GetCalleeSaveFpuRegisters() const { return m_callee_saved_fpu; }
+                  auto &GetCalleeSaveFpuRegisters()       { return m_callee_saved_fpu; }
+        public:
+            static void OnThreadTerminating(const KThread *thread);
+        public:
+            static consteval bool ValidateOffsets();
+
+            template<typename CallerSave, typename CalleeSave> requires ((std::same_as<CallerSave, CallerSaveFpu64Registers> && std::same_as<CalleeSave, CalleeSaveFpu64Registers>) || (std::same_as<CallerSave, CallerSaveFpu32Registers> && std::same_as<CalleeSave, CalleeSaveFpu32Registers>))
+            static void GetFpuRegisters(u128 *out, const CallerSave &caller_save, const CalleeSave &callee_save) {
+                /* Check that the register counts are correct. */
+                constexpr size_t RegisterUnitCount = util::size(CalleeSave{}.v);
+                static_assert(util::size(CalleeSave{}.v) == 1 * RegisterUnitCount);
+                static_assert(util::size(CallerSave{}.v) == 3 * RegisterUnitCount);
+
+                /* Copy the low caller-save registers. */
+                for (size_t i = 0; i < RegisterUnitCount; ++i) {
+                    *(out++) = caller_save.v[i];
+                }
+
+                /* Copy the callee-save registers. */
+                for (size_t i = 0; i < RegisterUnitCount; ++i) {
+                    *(out++) = callee_save.v[i];
+                }
+
+                /* Copy the remaining caller-save registers. */
+                for (size_t i = 0; i < 2 * RegisterUnitCount; ++i) {
+                    *(out++) = caller_save.v[RegisterUnitCount + i];
+                }
+            }
+
+            template<typename CallerSave, typename CalleeSave> requires ((std::same_as<CallerSave, CallerSaveFpu64Registers> && std::same_as<CalleeSave, CalleeSaveFpu64Registers>) || (std::same_as<CallerSave, CallerSaveFpu32Registers> && std::same_as<CalleeSave, CalleeSaveFpu32Registers>))
+            static ALWAYS_INLINE void SetFpuRegisters(CallerSave &caller_save, CalleeSave &callee_save, const u128 *v) {
+                /* Check that the register counts are correct. */
+                constexpr size_t RegisterUnitCount = util::size(CalleeSave{}.v);
+                static_assert(util::size(CalleeSave{}.v) == 1 * RegisterUnitCount);
+                static_assert(util::size(CallerSave{}.v) == 3 * RegisterUnitCount);
+
+                /* Copy the low caller-save registers. */
+                for (size_t i = 0; i < RegisterUnitCount; ++i) {
+                    caller_save.v[i] = *(v++);
+                }
+
+                /* Copy the callee-save registers. */
+                for (size_t i = 0; i < RegisterUnitCount; ++i) {
+                    callee_save.v[i] = *(v++);
+                }
+
+                /* Copy the remaining caller-save registers. */
+                for (size_t i = 0; i < 2 * RegisterUnitCount; ++i) {
+                    caller_save.v[RegisterUnitCount + i] = *(v++);
+                }
+            }
+    };
+
+    consteval bool KThreadContext::ValidateOffsets() {
+        static_assert(sizeof(KThreadContext) == THREAD_CONTEXT_SIZE);
+
+        static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved.registers) == THREAD_CONTEXT_CPU_REGISTERS);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved.x19)       == THREAD_CONTEXT_X19);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved.x20)       == THREAD_CONTEXT_X20);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved.x21)       == THREAD_CONTEXT_X21);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved.x22)       == THREAD_CONTEXT_X22);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved.x23)       == THREAD_CONTEXT_X23);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved.x24)       == THREAD_CONTEXT_X24);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved.x25)       == THREAD_CONTEXT_X25);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved.x26)       == THREAD_CONTEXT_X26);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved.x27)       == THREAD_CONTEXT_X27);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved.x28)       == THREAD_CONTEXT_X28);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved.x29)       == THREAD_CONTEXT_X29);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_lr)                     == THREAD_CONTEXT_LR);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_sp)                     == THREAD_CONTEXT_SP);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_fpcr)                   == THREAD_CONTEXT_FPCR);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_fpsr)                   == THREAD_CONTEXT_FPSR);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved_fpu)       == THREAD_CONTEXT_FPU_REGISTERS);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_locked)                 == THREAD_CONTEXT_LOCKED);
+
+        static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved_fpu.fpu64.q8 ) == THREAD_CONTEXT_FPU64_Q8 );
+        static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved_fpu.fpu64.q9 ) == THREAD_CONTEXT_FPU64_Q9 );
+        static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved_fpu.fpu64.q10) == THREAD_CONTEXT_FPU64_Q10);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved_fpu.fpu64.q11) == THREAD_CONTEXT_FPU64_Q11);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved_fpu.fpu64.q12) == THREAD_CONTEXT_FPU64_Q12);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved_fpu.fpu64.q13) == THREAD_CONTEXT_FPU64_Q13);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved_fpu.fpu64.q14) == THREAD_CONTEXT_FPU64_Q14);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved_fpu.fpu64.q15) == THREAD_CONTEXT_FPU64_Q15);
+
+        static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved_fpu.fpu32.q4 ) == THREAD_CONTEXT_FPU32_Q4 );
+        static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved_fpu.fpu32.q5 ) == THREAD_CONTEXT_FPU32_Q5 );
+        static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved_fpu.fpu32.q6 ) == THREAD_CONTEXT_FPU32_Q6 );
+        static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved_fpu.fpu32.q7 ) == THREAD_CONTEXT_FPU32_Q7 );
+
+        static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q0 ) == THREAD_FPU64_CONTEXT_Q0 );
+        static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q1 ) == THREAD_FPU64_CONTEXT_Q1 );
+        static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q2 ) == THREAD_FPU64_CONTEXT_Q2 );
+        static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q3 ) == THREAD_FPU64_CONTEXT_Q3 );
+        static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q4 ) == THREAD_FPU64_CONTEXT_Q4 );
+        static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q5 ) == THREAD_FPU64_CONTEXT_Q5 );
+        static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q6 ) == THREAD_FPU64_CONTEXT_Q6 );
+        static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q7 ) == THREAD_FPU64_CONTEXT_Q7 );
+        static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q16) == THREAD_FPU64_CONTEXT_Q16);
+        static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q17) == THREAD_FPU64_CONTEXT_Q17);
+        static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q18) == THREAD_FPU64_CONTEXT_Q18);
+        static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q19) == THREAD_FPU64_CONTEXT_Q19);
+        static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q20) == THREAD_FPU64_CONTEXT_Q20);
+        static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q21) == THREAD_FPU64_CONTEXT_Q21);
+        static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q22) == THREAD_FPU64_CONTEXT_Q22);
+        static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q23) == THREAD_FPU64_CONTEXT_Q23);
+        static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q24) == THREAD_FPU64_CONTEXT_Q24);
+        static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q25) == THREAD_FPU64_CONTEXT_Q25);
+        static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q26) == THREAD_FPU64_CONTEXT_Q26);
+        static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q27) == THREAD_FPU64_CONTEXT_Q27);
+        static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q28) == THREAD_FPU64_CONTEXT_Q28);
+        static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q29) == THREAD_FPU64_CONTEXT_Q29);
+        static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q30) == THREAD_FPU64_CONTEXT_Q30);
+        static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q31) == THREAD_FPU64_CONTEXT_Q31);
+
+        static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu32.q0 ) == THREAD_FPU32_CONTEXT_Q0 );
+        static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu32.q1 ) == THREAD_FPU32_CONTEXT_Q1 );
+        static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu32.q2 ) == THREAD_FPU32_CONTEXT_Q2 );
+        static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu32.q3 ) == THREAD_FPU32_CONTEXT_Q3 );
+        static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu32.q8 ) == THREAD_FPU32_CONTEXT_Q8 );
+        static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu32.q9 ) == THREAD_FPU32_CONTEXT_Q9 );
+        static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu32.q10) == THREAD_FPU32_CONTEXT_Q10);
+        static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu32.q11) == THREAD_FPU32_CONTEXT_Q11);
+        static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu32.q12) == THREAD_FPU32_CONTEXT_Q12);
+        static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu32.q13) == THREAD_FPU32_CONTEXT_Q13);
+        static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu32.q14) == THREAD_FPU32_CONTEXT_Q14);
+        static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu32.q15) == THREAD_FPU32_CONTEXT_Q15);
+
+        return true;
+    }
+    static_assert(KThreadContext::ValidateOffsets());
+
+    void GetUserContext(ams::svc::ThreadContext *out, const KThread *thread);
+
+}

Source/Atmosphere/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_secure_monitor_base.hpp

@@ -0,0 +1,78 @@
+/*
+ * Copyright (c) Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_select_cpu.hpp>
+
+namespace ams::kern::arch::arm64::smc {
+
+    template<int SmcId>
+    void SecureMonitorCall(u64 *buf) {
+        /* Load arguments into registers. */
+        register u64 x0 asm("x0") = buf[0];
+        register u64 x1 asm("x1") = buf[1];
+        register u64 x2 asm("x2") = buf[2];
+        register u64 x3 asm("x3") = buf[3];
+        register u64 x4 asm("x4") = buf[4];
+        register u64 x5 asm("x5") = buf[5];
+        register u64 x6 asm("x6") = buf[6];
+        register u64 x7 asm("x7") = buf[7];
+
+        /* Backup the current thread pointer. */
+        const uintptr_t current_thread_pointer_value = cpu::GetCurrentThreadPointerValue();
+
+        /* Perform the call. */
+        __asm__ __volatile__("smc %c[smc_id]"
+                            : "+r"(x0), "+r"(x1), "+r"(x2), "+r"(x3), "+r"(x4), "+r"(x5), "+r"(x6), "+r"(x7)
+                            : [smc_id]"i"(SmcId)
+                            : "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x18", "cc", "memory"
+                            );
+
+        /* Restore the current thread pointer into X18. */
+        cpu::SetCurrentThreadPointerValue(current_thread_pointer_value);
+
+        /* Store arguments to output. */
+        buf[0] = x0;
+        buf[1] = x1;
+        buf[2] = x2;
+        buf[3] = x3;
+        buf[4] = x4;
+        buf[5] = x5;
+        buf[6] = x6;
+        buf[7] = x7;
+    }
+
+    enum PsciFunction {
+        PsciFunction_CpuSuspend = 0xC4000001,
+        PsciFunction_CpuOff     = 0x84000002,
+        PsciFunction_CpuOn      = 0xC4000003,
+    };
+
+    template<int SmcId>
+    u64 PsciCall(PsciFunction function, u64 x1 = 0, u64 x2 = 0, u64 x3 = 0, u64 x4 = 0, u64 x5 = 0, u64 x6 = 0, u64 x7 = 0) {
+        ams::svc::lp64::SecureMonitorArguments args = { { function, x1, x2, x3, x4, x5, x6, x7 } };
+
+        SecureMonitorCall<SmcId>(args.r);
+
+        return args.r[0];
+    }
+
+    template<int SmcId>
+    u64 CpuOn(u64 core_id, uintptr_t entrypoint, uintptr_t arg) {
+        return PsciCall<SmcId>(PsciFunction_CpuOn, core_id, entrypoint, arg);
+    }
+
+}

Source/Atmosphere/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_userspace_memory_access.hpp

@@ -0,0 +1,163 @@
+/*
+ * Copyright (c) Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+
+namespace ams::kern::arch::arm64 {
+
+    void UserspaceAccessFunctionAreaBegin();
+
+    class UserspaceAccess {
+        private:
+            class Impl {
+                public:
+                    static bool CopyMemoryFromUser(void *dst, const void *src, size_t size);
+                    static bool CopyMemoryFromUserAligned32Bit(void *dst, const void *src, size_t size);
+                    static bool CopyMemoryFromUserAligned64Bit(void *dst, const void *src, size_t size);
+                    static bool CopyMemoryFromUserSize64Bit(void *dst, const void *src);
+                    static bool CopyMemoryFromUserSize32Bit(void *dst, const void *src);
+                    static bool CopyMemoryFromUserSize32BitWithSupervisorAccess(void *dst, const void *src);
+                    static s32  CopyStringFromUser(void *dst, const void *src, size_t size);
+
+                    static bool CopyMemoryToUser(void *dst, const void *src, size_t size);
+                    static bool CopyMemoryToUserAligned32Bit(void *dst, const void *src, size_t size);
+                    static bool CopyMemoryToUserAligned64Bit(void *dst, const void *src, size_t size);
+                    static bool CopyMemoryToUserSize32Bit(void *dst, const void *src);
+                    static s32  CopyStringToUser(void *dst, const void *src, size_t size);
+
+                    static bool UpdateLockAtomic(u32 *out, u32 *address, u32 if_zero, u32 new_orr_mask);
+                    static bool UpdateIfEqualAtomic(s32 *out, s32 *address, s32 compare_value, s32 new_value);
+                    static bool DecrementIfLessThanAtomic(s32 *out, s32 *address, s32 compare);
+
+                    static bool StoreDataCache(uintptr_t start, uintptr_t end);
+                    static bool FlushDataCache(uintptr_t start, uintptr_t end);
+                    static bool InvalidateDataCache(uintptr_t start, uintptr_t end);
+
+                    static bool ReadIoMemory32Bit(void *dst, const void *src, size_t size);
+                    static bool ReadIoMemory16Bit(void *dst, const void *src, size_t size);
+                    static bool ReadIoMemory8Bit(void *dst, const void *src, size_t size);
+                    static bool WriteIoMemory32Bit(void *dst, const void *src, size_t size);
+                    static bool WriteIoMemory16Bit(void *dst, const void *src, size_t size);
+                    static bool WriteIoMemory8Bit(void *dst, const void *src, size_t size);
+            };
+        public:
+            static bool CopyMemoryFromUser(void *dst, const void *src, size_t size) {
+                return Impl::CopyMemoryFromUser(dst, src, size);
+            }
+
+            static bool CopyMemoryFromUserAligned32Bit(void *dst, const void *src, size_t size) {
+                return Impl::CopyMemoryFromUserAligned32Bit(dst, src, size);
+            }
+
+            static bool CopyMemoryFromUserAligned64Bit(void *dst, const void *src, size_t size) {
+                return Impl::CopyMemoryFromUserAligned64Bit(dst, src, size);
+            }
+
+            static bool CopyMemoryFromUserSize64Bit(void *dst, const void *src) {
+                return Impl::CopyMemoryFromUserSize64Bit(dst, src);
+            }
+
+            static bool CopyMemoryFromUserSize32Bit(void *dst, const void *src) {
+                return Impl::CopyMemoryFromUserSize32Bit(dst, src);
+            }
+
+
+            static bool CopyMemoryFromUserSize32BitWithSupervisorAccess(void *dst, const void *src) {
+                /* Check that the address is within the valid userspace range. */
+                if (const uintptr_t src_uptr = reinterpret_cast<uintptr_t>(src); src_uptr < ams::svc::AddressNullGuard32Size || (src_uptr + sizeof(u32) - 1) >= ams::svc::AddressMemoryRegion39Size) {
+                    return false;
+                }
+
+                return Impl::CopyMemoryFromUserSize32BitWithSupervisorAccess(dst, src);
+            }
+
+            static s32  CopyStringFromUser(void *dst, const void *src, size_t size) {
+                return Impl::CopyStringFromUser(dst, src, size);
+            }
+
+            static bool CopyMemoryToUser(void *dst, const void *src, size_t size) {
+                return Impl::CopyMemoryToUser(dst, src, size);
+            }
+
+            static bool CopyMemoryToUserAligned32Bit(void *dst, const void *src, size_t size) {
+                return Impl::CopyMemoryToUserAligned32Bit(dst, src, size);
+            }
+
+            static bool CopyMemoryToUserAligned64Bit(void *dst, const void *src, size_t size) {
+                return Impl::CopyMemoryToUserAligned64Bit(dst, src, size);
+            }
+
+            static bool CopyMemoryToUserSize32Bit(void *dst, const void *src) {
+                return Impl::CopyMemoryToUserSize32Bit(dst, src);
+            }
+
+            static s32  CopyStringToUser(void *dst, const void *src, size_t size) {
+                return Impl::CopyStringToUser(dst, src, size);
+            }
+
+            static bool UpdateLockAtomic(u32 *out, u32 *address, u32 if_zero, u32 new_orr_mask) {
+                return Impl::UpdateLockAtomic(out, address, if_zero, new_orr_mask);
+            }
+
+            static bool UpdateIfEqualAtomic(s32 *out, s32 *address, s32 compare_value, s32 new_value) {
+                return Impl::UpdateIfEqualAtomic(out, address, compare_value, new_value);
+            }
+
+            static bool DecrementIfLessThanAtomic(s32 *out, s32 *address, s32 compare) {
+                return Impl::DecrementIfLessThanAtomic(out, address, compare);
+            }
+
+            static bool StoreDataCache(uintptr_t start, uintptr_t end) {
+                return Impl::StoreDataCache(start, end);
+            }
+
+            static bool FlushDataCache(uintptr_t start, uintptr_t end) {
+                return Impl::FlushDataCache(start, end);
+            }
+
+            static bool InvalidateDataCache(uintptr_t start, uintptr_t end) {
+                return Impl::InvalidateDataCache(start, end);
+            }
+
+            static bool ReadIoMemory32Bit(void *dst, const void *src, size_t size) {
+                return Impl::ReadIoMemory32Bit(dst, src, size);
+            }
+
+            static bool ReadIoMemory16Bit(void *dst, const void *src, size_t size) {
+                return Impl::ReadIoMemory16Bit(dst, src, size);
+            }
+
+            static bool ReadIoMemory8Bit(void *dst, const void *src, size_t size) {
+                return Impl::ReadIoMemory8Bit(dst, src, size);
+            }
+
+            static bool WriteIoMemory32Bit(void *dst, const void *src, size_t size) {
+                return Impl::WriteIoMemory32Bit(dst, src, size);
+            }
+
+            static bool WriteIoMemory16Bit(void *dst, const void *src, size_t size) {
+                return Impl::WriteIoMemory16Bit(dst, src, size);
+            }
+
+            static bool WriteIoMemory8Bit(void *dst, const void *src, size_t size) {
+                return Impl::WriteIoMemory8Bit(dst, src, size);
+            }
+    };
+
+
+    void UserspaceAccessFunctionAreaEnd();
+
+}