kern: simplify random bitmap selection to match latest Nintendo logic

This is an optimization that saves the most common type of virtual call in the kernel (DynamicCast)
by storing class token as a member, rather than getting it via virtual call every time.

This does not currently cost any memory space on 64-bit targets, due to pre-existing padding space.

This optimization can be turned off via a compile-time flag for accuracy.

exosphere/program/source/secmon_exception_handler.cpp

@@ -22,7 +22,7 @@ namespace ams::secmon {
 
         constexpr inline uintptr_t PMC = MemoryRegionVirtualDevicePmc.GetAddress();
 
-        constinit std::atomic_bool g_is_locked = false;
+        constinit util::Atomic<bool> g_is_locked = false;
 
     }
 
@@ -72,7 +72,7 @@ namespace ams::secmon {
         }
 
         /* Acquire exclusive access to exception handling logic. */
-        if (!g_is_locked.exchange(true)) {
+        if (!g_is_locked.Exchange(true)) {
             /* Invoke the exception handler impl. */
             ExceptionHandlerImpl(lr, sp);
 

exosphere/program/source/smc/secmon_smc_se_lock.cpp

@@ -21,35 +21,47 @@ namespace ams::secmon::smc {
 
     namespace {
 
-        constinit std::atomic_bool g_is_locked = false;
+        constinit util::Atomic<bool> g_is_locked = false;
+
+        ALWAYS_INLINE bool TryLockSecurityEngineImpl() {
+            bool value = false;
+            return g_is_locked.CompareExchangeStrong(value, true);
+        }
+
+        ALWAYS_INLINE void UnlockSecurityEngineImpl() {
+            g_is_locked = false;
+        }
+
+        ALWAYS_INLINE bool IsSecurityEngineLockedImpl() {
+            return g_is_locked.Load();
+        }
 
     }
 
     bool TryLockSecurityEngine() {
-        bool value = false;
-        return g_is_locked.compare_exchange_strong(value, true);
+        return TryLockSecurityEngineImpl();
     }
 
     void UnlockSecurityEngine() {
-        g_is_locked = false;
+        return UnlockSecurityEngineImpl();
     }
 
     bool IsSecurityEngineLocked() {
-        return g_is_locked;
+        return IsSecurityEngineLockedImpl();
     }
 
     SmcResult LockSecurityEngineAndInvoke(SmcArguments &args, SmcHandler impl) {
         /* Try to lock the security engine. */
-        SMC_R_UNLESS(TryLockSecurityEngine(), Busy);
-        ON_SCOPE_EXIT { UnlockSecurityEngine(); };
+        SMC_R_UNLESS(TryLockSecurityEngineImpl(), Busy);
+        ON_SCOPE_EXIT { UnlockSecurityEngineImpl(); };
 
         return impl(args);
     }
 
     SmcResult LockSecurityEngineAndInvokeAsync(SmcArguments &args, SmcHandler impl, GetResultHandler result_handler) {
         /* Try to lock the security engine. */
-        SMC_R_UNLESS(TryLockSecurityEngine(), Busy);
-        auto se_guard = SCOPE_GUARD { UnlockSecurityEngine(); };
+        SMC_R_UNLESS(TryLockSecurityEngineImpl(), Busy);
+        auto se_guard = SCOPE_GUARD { UnlockSecurityEngineImpl(); };
 
         /* Try to start an async operation. */
         const u64 async_key = BeginAsyncOperation(result_handler);

fusee/program/source/fusee_overlay_manager.cpp

@@ -33,11 +33,11 @@ namespace ams::nxboot {
             u32 verif_hash;
             u32 store_hash;
             if (fuse::GetSocType() == fuse::SocType_Erista) {
-                result = fs::ReadFile(archive_file, __builtin_offsetof(ExternalPackage, ovl_mtc_erista), GetOverlayDestination(), sizeof(ExternalPackage{}.ovl_mtc_erista));
+                result = fs::ReadFile(archive_file, AMS_OFFSETOF(ExternalPackage, ovl_mtc_erista), GetOverlayDestination(), sizeof(ExternalPackage{}.ovl_mtc_erista));
                 verif_hash = reinterpret_cast<const u32 *>(GetOverlayDestination())[-2];
                 store_hash = reinterpret_cast<const u32 *>(GetOverlayDestination())[(sizeof(ExternalPackage{}.ovl_mtc_erista) / sizeof(u32)) - 1];
             } else /* if (fuse::GetSocType() == fuse::SocType_Mariko) */ {
-                result = fs::ReadFile(archive_file, __builtin_offsetof(ExternalPackage, ovl_mtc_mariko), GetOverlayDestination(), sizeof(ExternalPackage{}.ovl_mtc_mariko));
+                result = fs::ReadFile(archive_file, AMS_OFFSETOF(ExternalPackage, ovl_mtc_mariko), GetOverlayDestination(), sizeof(ExternalPackage{}.ovl_mtc_mariko));
                 verif_hash = reinterpret_cast<const u32 *>(GetOverlayDestination())[-1];
                 store_hash = reinterpret_cast<const u32 *>(GetOverlayDestination())[(sizeof(ExternalPackage{}.ovl_mtc_mariko) / sizeof(u32)) - 1];
             }

fusee/program/source/fusee_setup_horizon.cpp

@@ -146,7 +146,7 @@ namespace ams::nxboot {
                         /* Handle individual fields. */
                         for (const auto &entry : section.kv_list) {
                             if (std::strcmp(entry.key, "enabled") == 0) {
-                                enabled = entry.value[0] == '1';
+                                enabled = entry.value[0] != '0';
                             } else if (std::strcmp(entry.key, "id") == 0) {
                                 id = ParseHexInteger(entry.value);
                             } else if (std::strcmp(entry.key, "sector") == 0) {
@@ -270,7 +270,7 @@ namespace ams::nxboot {
             u8 *package2;
             size_t package2_size;
             {
-                constexpr s64 Package2Offset = __builtin_offsetof(pkg2::StorageLayout, package2_header);
+                constexpr s64 Package2Offset = AMS_OFFSETOF(pkg2::StorageLayout, package2_header);
 
                 pkg2::Package2Header header;
                 if (R_FAILED((result = ReadPackage2(Package2Offset, std::addressof(header), sizeof(header))))) {

libraries/config/board/qemu/virt/board.mk

@@ -0,0 +1,5 @@
+export ATMOSPHERE_DEFINES  += -DATMOSPHERE_BOARD_QEMU_VIRT -D__SWITCH__
+export ATMOSPHERE_SETTINGS +=
+export ATMOSPHERE_CFLAGS   +=
+export ATMOSPHERE_CXXFLAGS +=
+export ATMOSPHERE_ASFLAGS  +=

libraries/config/common.mk

@@ -23,7 +23,8 @@ export ATMOSPHERE_CFLAGS   := -Wall -ffunction-sections -fdata-sections -fno-str
                               -fno-asynchronous-unwind-tables -fno-unwind-tables -fno-stack-protector \
                               -Wno-format-truncation -Wno-format-zero-length -Wno-stringop-truncation
 
-export ATMOSPHERE_CXXFLAGS := -fno-rtti -fno-exceptions -std=gnu++20
+
+export ATMOSPHERE_CXXFLAGS := -fno-rtti -fno-exceptions -std=gnu++20 -Wno-invalid-offsetof
 export ATMOSPHERE_ASFLAGS  :=
 
 
@@ -51,6 +52,21 @@ export ATMOSPHERE_OS_NAME    := horizon
 export ATMOSPHERE_CPU_EXTENSIONS :=
 endif
 
+else ifeq ($(ATMOSPHERE_BOARD),qemu-virt)
+
+
+ifeq ($(ATMOSPHERE_CPU),arm-cortex-a57)
+export ATMOSPHERE_ARCH_DIR   := arm64
+export ATMOSPHERE_BOARD_DIR  := qemu/virt
+export ATMOSPHERE_OS_DIR     := horizon
+
+export ATMOSPHERE_ARCH_NAME  := arm64
+export ATMOSPHERE_BOARD_NAME := qemu_virt
+export ATMOSPHERE_OS_NAME    := horizon
+
+export ATMOSPHERE_CPU_EXTENSIONS := arm_crypto_extension aarch64_crypto_extension
+endif
+
 endif
 
 ifeq ($(ATMOSPHERE_CPU),arm-cortex-a57)

libraries/libexosphere/include/exosphere/pkg1/pkg1_bootloader_parameters.hpp

@@ -55,9 +55,9 @@ namespace ams::pkg1 {
     static_assert(util::is_pod<SecureMonitorParameters>::value);
     static_assert(sizeof(SecureMonitorParameters) == 0x200);
 
-    static_assert(offsetof(SecureMonitorParameters, bct_params)       == 0x10);
-    static_assert(offsetof(SecureMonitorParameters, bootloader_state) == 0xF8);
-    static_assert(offsetof(SecureMonitorParameters, secmon_state) == 0xFC);
+    static_assert(AMS_OFFSETOF(SecureMonitorParameters, bct_params)       == 0x10);
+    static_assert(AMS_OFFSETOF(SecureMonitorParameters, bootloader_state) == 0xF8);
+    static_assert(AMS_OFFSETOF(SecureMonitorParameters, secmon_state) == 0xFC);
 
     enum BootloaderAttribute {
         BootloaderAttribute_None         = (0u << 0),

libraries/libexosphere/include/exosphere/secmon/secmon_volatile_context.hpp

@@ -128,10 +128,10 @@ namespace ams::secmon {
 
     }
 
-    constexpr inline const Address WarmbootStackAddress   = MemoryRegionVirtualTzramVolatileData.GetAddress() + offsetof(VolatileData, warmboot_stack)    + sizeof(VolatileData::warmboot_stack);
-    constexpr inline const Address Core012SmcStackAddress = MemoryRegionVirtualTzramVolatileData.GetAddress() + offsetof(VolatileData, core012_smc_stack) + sizeof(VolatileData::core012_smc_stack);
+    constexpr inline const Address WarmbootStackAddress   = MemoryRegionVirtualTzramVolatileData.GetAddress() + AMS_OFFSETOF(VolatileData, warmboot_stack)    + sizeof(VolatileData::warmboot_stack);
+    constexpr inline const Address Core012SmcStackAddress = MemoryRegionVirtualTzramVolatileData.GetAddress() + AMS_OFFSETOF(VolatileData, core012_smc_stack) + sizeof(VolatileData::core012_smc_stack);
 
-    constexpr inline const Address Core0ExceptionStackAddress = MemoryRegionVirtualTzramVolatileData.GetAddress() + offsetof(VolatileData, core_exception_stacks) + CoreExceptionStackSize;
+    constexpr inline const Address Core0ExceptionStackAddress = MemoryRegionVirtualTzramVolatileData.GetAddress() + AMS_OFFSETOF(VolatileData, core_exception_stacks) + CoreExceptionStackSize;
     constexpr inline const Address Core1ExceptionStackAddress = Core0ExceptionStackAddress + CoreExceptionStackSize;
     constexpr inline const Address Core2ExceptionStackAddress = Core1ExceptionStackAddress + CoreExceptionStackSize;
 

libraries/libexosphere/source/gic/gic_api.cpp

@@ -191,27 +191,27 @@ namespace ams::gic {
     }
 
     void SetPriority(int interrupt_id, int priority) {
-        ReadWrite(g_distributor_address + offsetof(GicDistributor, ipriorityr), BITSIZEOF(u8), interrupt_id, priority);
+        ReadWrite(g_distributor_address + AMS_OFFSETOF(GicDistributor, ipriorityr), BITSIZEOF(u8), interrupt_id, priority);
     }
 
     void SetInterruptGroup(int interrupt_id, int group) {
-        ReadWrite(g_distributor_address + offsetof(GicDistributor, igroupr), 1, interrupt_id, group);
+        ReadWrite(g_distributor_address + AMS_OFFSETOF(GicDistributor, igroupr), 1, interrupt_id, group);
     }
 
     void SetEnable(int interrupt_id, bool enable) {
-        Write(g_distributor_address + offsetof(GicDistributor, isenabler), 1, interrupt_id, enable);
+        Write(g_distributor_address + AMS_OFFSETOF(GicDistributor, isenabler), 1, interrupt_id, enable);
     }
 
     void SetSpiTargetCpu(int interrupt_id, u32 cpu_mask) {
-        ReadWrite(g_distributor_address + offsetof(GicDistributor, itargetsr), BITSIZEOF(u8), interrupt_id, cpu_mask);
+        ReadWrite(g_distributor_address + AMS_OFFSETOF(GicDistributor, itargetsr), BITSIZEOF(u8), interrupt_id, cpu_mask);
     }
 
     void SetSpiMode(int interrupt_id, InterruptMode mode) {
-        ReadWrite(g_distributor_address + offsetof(GicDistributor, icfgr), 2, interrupt_id, static_cast<u32>(mode) << 1);
+        ReadWrite(g_distributor_address + AMS_OFFSETOF(GicDistributor, icfgr), 2, interrupt_id, static_cast<u32>(mode) << 1);
     }
 
     void SetPending(int interrupt_id) {
-        Write(g_distributor_address + offsetof(GicDistributor, ispendr), 1, interrupt_id, 1);
+        Write(g_distributor_address + AMS_OFFSETOF(GicDistributor, ispendr), 1, interrupt_id, 1);
     }
 
     int GetInterruptRequestId() {

libraries/libmesosphere/Makefile

@@ -104,6 +104,18 @@ $(eval $(call ATMOSPHERE_ADD_TARGET, audit, $(TARGET)_audit.a, \
 	ATMOSPHERE_BUILD_SETTINGS="-DMESOSPHERE_BUILD_FOR_AUDITING" \
 ))
 
+$(eval $(call ATMOSPHERE_ADD_TARGET, qemu_virt_release, $(TARGET)_qemu_virt.a, \
+	ATMOSPHERE_BUILD_SETTINGS="" \
+))
+
+$(eval $(call ATMOSPHERE_ADD_TARGET, qemu_virt_debug, $(TARGET)_qemu_virt_debug.a, \
+	ATMOSPHERE_BUILD_SETTINGS="-DMESOSPHERE_BUILD_FOR_DEBUGGING" \
+))
+
+$(eval $(call ATMOSPHERE_ADD_TARGET, qemu_virt_audit, $(TARGET)_qemu_virt_audit.a, \
+	ATMOSPHERE_BUILD_SETTINGS="-DMESOSPHERE_BUILD_FOR_AUDITING" \
+))
+
 #---------------------------------------------------------------------------------
 
 -include $(ATMOSPHERE_BOARD_NAME)_$(ATMOSPHERE_ARCH_NAME).mk
@@ -142,8 +154,6 @@ $(OFILES_SRC)	: $(HFILES_BIN)
 
 kern_libc_generic.o: CFLAGS += -fno-builtin
 
-kern_k_auto_object.o kern_k_debug_base_process_holder.o: CXXFLAGS += -fno-lto
-
 #---------------------------------------------------------------------------------
 %_bin.h %.bin.o	:	%.bin
 #---------------------------------------------------------------------------------

libraries/libmesosphere/include/mesosphere.hpp

@@ -90,3 +90,6 @@
 
 /* Main functionality. */
 #include <mesosphere/kern_main.hpp>
+
+/* Deferred includes. */
+#include <mesosphere/kern_k_auto_object_impls.hpp>

libraries/libmesosphere/include/mesosphere/arch/arm/kern_generic_interrupt_controller.hpp

@@ -248,8 +248,9 @@ namespace ams::kern::arch::arm {
                 return id;
             }
         private:
-            static constexpr size_t PriorityShift = BITSIZEOF(u8) - __builtin_ctz(NumPriorityLevels);
+            static constexpr size_t PriorityShift = BITSIZEOF(u8) - util::CountTrailingZeros(NumPriorityLevels);
             static_assert(PriorityShift < BITSIZEOF(u8));
+            static_assert(util::IsPowerOfTwo(NumPriorityLevels));
 
             static constexpr ALWAYS_INLINE u8 ToGicPriorityValue(s32 level) {
                 return (level << PriorityShift) | ((1 << PriorityShift) - 1);

libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_arguments.hpp

@@ -35,17 +35,17 @@ namespace ams::kern::init {
     static_assert(alignof(KInitArguments) == util::CeilingPowerOfTwo(INIT_ARGUMENTS_SIZE));
     static_assert(sizeof(KInitArguments) == std::max(INIT_ARGUMENTS_SIZE, util::CeilingPowerOfTwo(INIT_ARGUMENTS_SIZE)));
 
-    static_assert(__builtin_offsetof(KInitArguments, ttbr0)           == INIT_ARGUMENTS_TTBR0);
-    static_assert(__builtin_offsetof(KInitArguments, ttbr1)           == INIT_ARGUMENTS_TTBR1);
-    static_assert(__builtin_offsetof(KInitArguments, tcr)             == INIT_ARGUMENTS_TCR);
-    static_assert(__builtin_offsetof(KInitArguments, mair)            == INIT_ARGUMENTS_MAIR);
-    static_assert(__builtin_offsetof(KInitArguments, cpuactlr)        == INIT_ARGUMENTS_CPUACTLR);
-    static_assert(__builtin_offsetof(KInitArguments, cpuectlr)        == INIT_ARGUMENTS_CPUECTLR);
-    static_assert(__builtin_offsetof(KInitArguments, sctlr)           == INIT_ARGUMENTS_SCTLR);
-    static_assert(__builtin_offsetof(KInitArguments, sp)              == INIT_ARGUMENTS_SP);
-    static_assert(__builtin_offsetof(KInitArguments, entrypoint)      == INIT_ARGUMENTS_ENTRYPOINT);
-    static_assert(__builtin_offsetof(KInitArguments, argument)        == INIT_ARGUMENTS_ARGUMENT);
-    static_assert(__builtin_offsetof(KInitArguments, setup_function)  == INIT_ARGUMENTS_SETUP_FUNCTION);
-    static_assert(__builtin_offsetof(KInitArguments, exception_stack) == INIT_ARGUMENTS_EXCEPTION_STACK);
+    static_assert(AMS_OFFSETOF(KInitArguments, ttbr0)           == INIT_ARGUMENTS_TTBR0);
+    static_assert(AMS_OFFSETOF(KInitArguments, ttbr1)           == INIT_ARGUMENTS_TTBR1);
+    static_assert(AMS_OFFSETOF(KInitArguments, tcr)             == INIT_ARGUMENTS_TCR);
+    static_assert(AMS_OFFSETOF(KInitArguments, mair)            == INIT_ARGUMENTS_MAIR);
+    static_assert(AMS_OFFSETOF(KInitArguments, cpuactlr)        == INIT_ARGUMENTS_CPUACTLR);
+    static_assert(AMS_OFFSETOF(KInitArguments, cpuectlr)        == INIT_ARGUMENTS_CPUECTLR);
+    static_assert(AMS_OFFSETOF(KInitArguments, sctlr)           == INIT_ARGUMENTS_SCTLR);
+    static_assert(AMS_OFFSETOF(KInitArguments, sp)              == INIT_ARGUMENTS_SP);
+    static_assert(AMS_OFFSETOF(KInitArguments, entrypoint)      == INIT_ARGUMENTS_ENTRYPOINT);
+    static_assert(AMS_OFFSETOF(KInitArguments, argument)        == INIT_ARGUMENTS_ARGUMENT);
+    static_assert(AMS_OFFSETOF(KInitArguments, setup_function)  == INIT_ARGUMENTS_SETUP_FUNCTION);
+    static_assert(AMS_OFFSETOF(KInitArguments, exception_stack) == INIT_ARGUMENTS_EXCEPTION_STACK);
 
 }

libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp

@@ -34,19 +34,22 @@ namespace ams::kern::arch::arm64::init {
         }
     }
 
-    class KInitialPageTable {
+    /* NOTE: Nintendo uses virtual functions, rather than a concept + template. */
+    template<typename T>
+    concept IsInitialPageAllocator = requires (T &t, KPhysicalAddress phys_addr, size_t size) {
+        { t.Allocate(size) }        -> std::same_as<KPhysicalAddress>;
+        { t.Free(phys_addr, size) } -> std::same_as<void>;
+    };
+
+    template<IsInitialPageAllocator _PageAllocator>
+    class KInitialPageTableTemplate {
         public:
-            class IPageAllocator {
-                public:
-                    virtual KPhysicalAddress Allocate(size_t size) = 0;
-                    virtual void Free(KPhysicalAddress phys_addr, size_t size) = 0;
-            };
+            using PageAllocator = _PageAllocator;
         private:
             KPhysicalAddress m_l1_tables[2];
             u32 m_num_entries[2];
-
         public:
-            KInitialPageTable(KVirtualAddress start_address, KVirtualAddress end_address, IPageAllocator &allocator) {
+            KInitialPageTableTemplate(KVirtualAddress start_address, KVirtualAddress end_address, PageAllocator &allocator) {
                 /* Set tables. */
                 m_l1_tables[0] = AllocateNewPageTable(allocator);
                 m_l1_tables[1] = AllocateNewPageTable(allocator);
@@ -56,7 +59,7 @@ namespace ams::kern::arch::arm64::init {
                 m_num_entries[1] = ((end_address / L1BlockSize) & (MaxPageTableEntries - 1)) - ((start_address / L1BlockSize) & (MaxPageTableEntries - 1)) + 1;
             }
 
-            KInitialPageTable() {
+            KInitialPageTableTemplate() {
                 /* Set tables. */
                 m_l1_tables[0] = util::AlignDown(cpu::GetTtbr0El1(), PageSize);
                 m_l1_tables[1] = util::AlignDown(cpu::GetTtbr1El1(), PageSize);
@@ -95,7 +98,7 @@ namespace ams::kern::arch::arm64::init {
                 return l3_table + ((GetInteger(address) / L3BlockSize) & (MaxPageTableEntries - 1));
             }
 
-            static ALWAYS_INLINE KPhysicalAddress AllocateNewPageTable(IPageAllocator &allocator) {
+            static ALWAYS_INLINE KPhysicalAddress AllocateNewPageTable(PageAllocator &allocator) {
                 auto address = allocator.Allocate(PageSize);
                 ClearNewPageTable(address);
                 return address;
@@ -323,7 +326,7 @@ namespace ams::kern::arch::arm64::init {
                 }
             }
         public:
-            void NOINLINE Map(KVirtualAddress virt_addr, size_t size, KPhysicalAddress phys_addr, const PageTableEntry &attr, IPageAllocator &allocator) {
+            void NOINLINE Map(KVirtualAddress virt_addr, size_t size, KPhysicalAddress phys_addr, const PageTableEntry &attr, PageAllocator &allocator) {
                 /* Ensure that addresses and sizes are page aligned. */
                 MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr),    PageSize));
                 MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(phys_addr),    PageSize));
@@ -700,10 +703,9 @@ namespace ams::kern::arch::arm64::init {
                 this->PhysicallyRandomize(virt_addr, size, L3BlockSize, do_copy);
                 cpu::StoreEntireCacheForInit();
             }
-
     };
 
-    class KInitialPageAllocator final : public KInitialPageTable::IPageAllocator {
+    class KInitialPageAllocator final {
         private:
             static constexpr inline size_t FreeUnitSize = BITSIZEOF(u64) * PageSize;
             struct FreeListEntry {
@@ -807,11 +809,11 @@ namespace ams::kern::arch::arm64::init {
                 }
             }
 
-            virtual KPhysicalAddress Allocate(size_t size) override {
+            KPhysicalAddress Allocate(size_t size) {
                 return this->Allocate(size, size);
             }
 
-            virtual void Free(KPhysicalAddress phys_addr, size_t size) override {
+            void Free(KPhysicalAddress phys_addr, size_t size) {
                 auto **prev_next = std::addressof(m_state.free_head);
                 auto *new_chunk  = reinterpret_cast<FreeListEntry *>(GetInteger(phys_addr));
                 if (auto *cur = m_state.free_head; cur != nullptr) {
@@ -863,5 +865,8 @@ namespace ams::kern::arch::arm64::init {
                 *prev_next = new_chunk;
             }
     };
+    static_assert(IsInitialPageAllocator<KInitialPageAllocator>);
+
+    using KInitialPageTable = KInitialPageTableTemplate<KInitialPageAllocator>;
 
 }

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu.hpp

@@ -30,6 +30,8 @@ namespace ams::kern::arch::arm64::cpu {
 
 #if defined(ATMOSPHERE_BOARD_NINTENDO_NX)
     constexpr inline size_t NumCores = 4;
+#elif defined(ATMOSPHERE_BOARD_QEMU_VIRT)
+    constexpr inline size_t NumCores = 4;
 #else
     #error "Unknown Board for cpu::NumCores"
 #endif

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_debug.hpp

@@ -34,8 +34,9 @@ namespace ams::kern::arch::arm64 {
 
             static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
         public:
-            virtual Result GetThreadContextImpl(ams::svc::ThreadContext *out, KThread *thread, u32 context_flags) override;
-            virtual Result SetThreadContextImpl(const ams::svc::ThreadContext &ctx, KThread *thread, u32 context_flags) override;
+            /* NOTE: These are virtual in Nintendo's kernel. */
+            Result GetThreadContextImpl(ams::svc::ThreadContext *out, KThread *thread, u32 context_flags);
+            Result SetThreadContextImpl(const ams::svc::ThreadContext &ctx, KThread *thread, u32 context_flags);
         private:
             Result GetFpuContext(ams::svc::ThreadContext *out, KThread *thread, u32 context_flags);
             Result SetFpuContext(const ams::svc::ThreadContext &ctx, KThread *thread, u32 context_flags);

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_exception_context.hpp

@@ -45,40 +45,40 @@ namespace ams::kern::arch::arm64 {
     };
     static_assert(sizeof(KExceptionContext) == EXCEPTION_CONTEXT_SIZE);
 
-    static_assert(__builtin_offsetof(KExceptionContext, x[ 0]) == EXCEPTION_CONTEXT_X0);
-    static_assert(__builtin_offsetof(KExceptionContext, x[ 1]) == EXCEPTION_CONTEXT_X1);
-    static_assert(__builtin_offsetof(KExceptionContext, x[ 2]) == EXCEPTION_CONTEXT_X2);
-    static_assert(__builtin_offsetof(KExceptionContext, x[ 3]) == EXCEPTION_CONTEXT_X3);
-    static_assert(__builtin_offsetof(KExceptionContext, x[ 4]) == EXCEPTION_CONTEXT_X4);
-    static_assert(__builtin_offsetof(KExceptionContext, x[ 5]) == EXCEPTION_CONTEXT_X5);
-    static_assert(__builtin_offsetof(KExceptionContext, x[ 6]) == EXCEPTION_CONTEXT_X6);
-    static_assert(__builtin_offsetof(KExceptionContext, x[ 7]) == EXCEPTION_CONTEXT_X7);
-    static_assert(__builtin_offsetof(KExceptionContext, x[ 8]) == EXCEPTION_CONTEXT_X8);
-    static_assert(__builtin_offsetof(KExceptionContext, x[ 9]) == EXCEPTION_CONTEXT_X9);
-    static_assert(__builtin_offsetof(KExceptionContext, x[10]) == EXCEPTION_CONTEXT_X10);
-    static_assert(__builtin_offsetof(KExceptionContext, x[11]) == EXCEPTION_CONTEXT_X11);
-    static_assert(__builtin_offsetof(KExceptionContext, x[12]) == EXCEPTION_CONTEXT_X12);
-    static_assert(__builtin_offsetof(KExceptionContext, x[13]) == EXCEPTION_CONTEXT_X13);
-    static_assert(__builtin_offsetof(KExceptionContext, x[14]) == EXCEPTION_CONTEXT_X14);
-    static_assert(__builtin_offsetof(KExceptionContext, x[15]) == EXCEPTION_CONTEXT_X15);
-    static_assert(__builtin_offsetof(KExceptionContext, x[16]) == EXCEPTION_CONTEXT_X16);
-    static_assert(__builtin_offsetof(KExceptionContext, x[17]) == EXCEPTION_CONTEXT_X17);
-    static_assert(__builtin_offsetof(KExceptionContext, x[18]) == EXCEPTION_CONTEXT_X18);
-    static_assert(__builtin_offsetof(KExceptionContext, x[19]) == EXCEPTION_CONTEXT_X19);
-    static_assert(__builtin_offsetof(KExceptionContext, x[20]) == EXCEPTION_CONTEXT_X20);
-    static_assert(__builtin_offsetof(KExceptionContext, x[21]) == EXCEPTION_CONTEXT_X21);
-    static_assert(__builtin_offsetof(KExceptionContext, x[22]) == EXCEPTION_CONTEXT_X22);
-    static_assert(__builtin_offsetof(KExceptionContext, x[23]) == EXCEPTION_CONTEXT_X23);
-    static_assert(__builtin_offsetof(KExceptionContext, x[24]) == EXCEPTION_CONTEXT_X24);
-    static_assert(__builtin_offsetof(KExceptionContext, x[25]) == EXCEPTION_CONTEXT_X25);
-    static_assert(__builtin_offsetof(KExceptionContext, x[26]) == EXCEPTION_CONTEXT_X26);
-    static_assert(__builtin_offsetof(KExceptionContext, x[27]) == EXCEPTION_CONTEXT_X27);
-    static_assert(__builtin_offsetof(KExceptionContext, x[28]) == EXCEPTION_CONTEXT_X28);
-    static_assert(__builtin_offsetof(KExceptionContext, x[29]) == EXCEPTION_CONTEXT_X29);
-    static_assert(__builtin_offsetof(KExceptionContext, x[30]) == EXCEPTION_CONTEXT_X30);
-    static_assert(__builtin_offsetof(KExceptionContext, sp)    == EXCEPTION_CONTEXT_SP);
-    static_assert(__builtin_offsetof(KExceptionContext, pc)    == EXCEPTION_CONTEXT_PC);
-    static_assert(__builtin_offsetof(KExceptionContext, psr)   == EXCEPTION_CONTEXT_PSR);
-    static_assert(__builtin_offsetof(KExceptionContext, tpidr) == EXCEPTION_CONTEXT_TPIDR);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[ 0]) == EXCEPTION_CONTEXT_X0);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[ 1]) == EXCEPTION_CONTEXT_X1);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[ 2]) == EXCEPTION_CONTEXT_X2);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[ 3]) == EXCEPTION_CONTEXT_X3);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[ 4]) == EXCEPTION_CONTEXT_X4);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[ 5]) == EXCEPTION_CONTEXT_X5);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[ 6]) == EXCEPTION_CONTEXT_X6);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[ 7]) == EXCEPTION_CONTEXT_X7);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[ 8]) == EXCEPTION_CONTEXT_X8);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[ 9]) == EXCEPTION_CONTEXT_X9);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[10]) == EXCEPTION_CONTEXT_X10);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[11]) == EXCEPTION_CONTEXT_X11);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[12]) == EXCEPTION_CONTEXT_X12);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[13]) == EXCEPTION_CONTEXT_X13);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[14]) == EXCEPTION_CONTEXT_X14);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[15]) == EXCEPTION_CONTEXT_X15);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[16]) == EXCEPTION_CONTEXT_X16);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[17]) == EXCEPTION_CONTEXT_X17);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[18]) == EXCEPTION_CONTEXT_X18);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[19]) == EXCEPTION_CONTEXT_X19);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[20]) == EXCEPTION_CONTEXT_X20);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[21]) == EXCEPTION_CONTEXT_X21);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[22]) == EXCEPTION_CONTEXT_X22);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[23]) == EXCEPTION_CONTEXT_X23);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[24]) == EXCEPTION_CONTEXT_X24);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[25]) == EXCEPTION_CONTEXT_X25);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[26]) == EXCEPTION_CONTEXT_X26);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[27]) == EXCEPTION_CONTEXT_X27);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[28]) == EXCEPTION_CONTEXT_X28);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[29]) == EXCEPTION_CONTEXT_X29);
+    static_assert(AMS_OFFSETOF(KExceptionContext, x[30]) == EXCEPTION_CONTEXT_X30);
+    static_assert(AMS_OFFSETOF(KExceptionContext, sp)    == EXCEPTION_CONTEXT_SP);
+    static_assert(AMS_OFFSETOF(KExceptionContext, pc)    == EXCEPTION_CONTEXT_PC);
+    static_assert(AMS_OFFSETOF(KExceptionContext, psr)   == EXCEPTION_CONTEXT_PSR);
+    static_assert(AMS_OFFSETOF(KExceptionContext, tpidr) == EXCEPTION_CONTEXT_TPIDR);
 
 }

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_interrupt_name.hpp

@@ -36,6 +36,10 @@ namespace ams::kern::arch::arm64 {
             KInterruptName_SecurePhysicalTimer    = 29,
             KInterruptName_NonSecurePhysicalTimer = 30,
             KInterruptName_LegacyNIrq             = 31,
+    #elif defined(ATMOSPHERE_BOARD_QEMU_VIRT)
+            KInterruptName_VirtualTimer           = 27,
+            KInterruptName_SecurePhysicalTimer    = 29,
+            KInterruptName_NonSecurePhysicalTimer = 30,
     #endif
 
     #if defined(ATMOSPHERE_BOARD_NINTENDO_NX)

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table.hpp

@@ -168,7 +168,8 @@ namespace ams::kern::arch::arm64 {
                 return entry;
             }
         public:
-            constexpr KPageTable() : KPageTableBase(), m_manager(), m_ttbr(), m_asid() { /* ... */ }
+            constexpr explicit KPageTable(util::ConstantInitializeTag) : KPageTableBase(util::ConstantInitialize), m_manager(), m_ttbr(), m_asid() { /* ... */ }
+            explicit KPageTable() { /* ... */ }
 
             static NOINLINE void Initialize(s32 core_id);
 

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table_impl.hpp

@@ -42,7 +42,7 @@ namespace ams::kern::arch::arm64 {
                 const L3PageTableEntry *l3_entry;
             };
         private:
-            static constexpr size_t PageBits  = __builtin_ctzll(PageSize);
+            static constexpr size_t PageBits  = util::CountTrailingZeros(PageSize);
             static constexpr size_t NumLevels = 3;
             static constexpr size_t LevelBits = 9;
             static_assert(NumLevels > 0);
@@ -105,7 +105,9 @@ namespace ams::kern::arch::arm64 {
                 return GetL3EntryFromTable(KMemoryLayout::GetLinearVirtualAddress(entry->GetTable()), address);
             }
         public:
-            constexpr KPageTableImpl() : m_table(), m_is_kernel(), m_num_entries() { /* ... */ }
+            constexpr explicit KPageTableImpl(util::ConstantInitializeTag) : m_table(), m_is_kernel(), m_num_entries() { /* ... */ }
+
+            explicit KPageTableImpl() { /* ... */ }
 
             NOINLINE void InitializeForKernel(void *tb, KVirtualAddress start, KVirtualAddress end);
             NOINLINE void InitializeForProcess(void *tb, KVirtualAddress start, KVirtualAddress end);

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_process_page_table.hpp

@@ -23,8 +23,6 @@ namespace ams::kern::arch::arm64 {
         private:
             KPageTable m_page_table;
         public:
-            constexpr KProcessPageTable() : m_page_table() { /* ... */ }
-
             void Activate(u64 id) {
                 /* Activate the page table with the specified contextidr. */
                 m_page_table.Activate(id);

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_supervisor_page_table.hpp

@@ -25,7 +25,7 @@ namespace ams::kern::arch::arm64 {
             KPageTable m_page_table;
             u64 m_ttbr0_identity[cpu::NumCores];
         public:
-            constexpr KSupervisorPageTable() : m_page_table(), m_ttbr0_identity() { /* ... */ }
+            constexpr KSupervisorPageTable() : m_page_table(util::ConstantInitialize), m_ttbr0_identity() { /* ... */ }
 
             NOINLINE void Initialize(s32 core_id);
 

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_thread_context.hpp

@@ -57,7 +57,8 @@ namespace ams::kern::arch::arm64 {
             static void RestoreFpuRegisters64(const KThreadContext &);
             static void RestoreFpuRegisters32(const KThreadContext &);
         public:
-            constexpr explicit KThreadContext() : m_callee_saved(), m_lr(), m_sp(), m_cpacr(), m_fpcr(), m_fpsr(), m_fpu_registers(), m_locked() { /* ... */ }
+            constexpr explicit KThreadContext(util::ConstantInitializeTag) : m_callee_saved(), m_lr(), m_sp(), m_cpacr(), m_fpcr(), m_fpsr(), m_fpu_registers(), m_locked() { /* ... */ }
+            explicit KThreadContext() { /* ... */ }
 
             Result Initialize(KVirtualAddress u_pc, KVirtualAddress k_sp, KVirtualAddress u_sp, uintptr_t arg, bool is_user, bool is_64_bit, bool is_main);
             Result Finalize();
@@ -86,25 +87,25 @@ namespace ams::kern::arch::arm64 {
     consteval bool KThreadContext::ValidateOffsets() {
         static_assert(sizeof(KThreadContext) == THREAD_CONTEXT_SIZE);
 
-        static_assert(__builtin_offsetof(KThreadContext, m_callee_saved.registers) == THREAD_CONTEXT_CPU_REGISTERS);
-        static_assert(__builtin_offsetof(KThreadContext, m_callee_saved.x19)       == THREAD_CONTEXT_X19);
-        static_assert(__builtin_offsetof(KThreadContext, m_callee_saved.x20)       == THREAD_CONTEXT_X20);
-        static_assert(__builtin_offsetof(KThreadContext, m_callee_saved.x21)       == THREAD_CONTEXT_X21);
-        static_assert(__builtin_offsetof(KThreadContext, m_callee_saved.x22)       == THREAD_CONTEXT_X22);
-        static_assert(__builtin_offsetof(KThreadContext, m_callee_saved.x23)       == THREAD_CONTEXT_X23);
-        static_assert(__builtin_offsetof(KThreadContext, m_callee_saved.x24)       == THREAD_CONTEXT_X24);
-        static_assert(__builtin_offsetof(KThreadContext, m_callee_saved.x25)       == THREAD_CONTEXT_X25);
-        static_assert(__builtin_offsetof(KThreadContext, m_callee_saved.x26)       == THREAD_CONTEXT_X26);
-        static_assert(__builtin_offsetof(KThreadContext, m_callee_saved.x27)       == THREAD_CONTEXT_X27);
-        static_assert(__builtin_offsetof(KThreadContext, m_callee_saved.x28)       == THREAD_CONTEXT_X28);
-        static_assert(__builtin_offsetof(KThreadContext, m_callee_saved.x29)       == THREAD_CONTEXT_X29);
-        static_assert(__builtin_offsetof(KThreadContext, m_lr)                     == THREAD_CONTEXT_LR);
-        static_assert(__builtin_offsetof(KThreadContext, m_sp)                     == THREAD_CONTEXT_SP);
-        static_assert(__builtin_offsetof(KThreadContext, m_cpacr)                  == THREAD_CONTEXT_CPACR);
-        static_assert(__builtin_offsetof(KThreadContext, m_fpcr)                   == THREAD_CONTEXT_FPCR);
-        static_assert(__builtin_offsetof(KThreadContext, m_fpsr)                   == THREAD_CONTEXT_FPSR);
-        static_assert(__builtin_offsetof(KThreadContext, m_fpu_registers)          == THREAD_CONTEXT_FPU_REGISTERS);
-        static_assert(__builtin_offsetof(KThreadContext, m_locked)                 == THREAD_CONTEXT_LOCKED);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved.registers) == THREAD_CONTEXT_CPU_REGISTERS);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved.x19)       == THREAD_CONTEXT_X19);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved.x20)       == THREAD_CONTEXT_X20);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved.x21)       == THREAD_CONTEXT_X21);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved.x22)       == THREAD_CONTEXT_X22);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved.x23)       == THREAD_CONTEXT_X23);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved.x24)       == THREAD_CONTEXT_X24);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved.x25)       == THREAD_CONTEXT_X25);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved.x26)       == THREAD_CONTEXT_X26);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved.x27)       == THREAD_CONTEXT_X27);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved.x28)       == THREAD_CONTEXT_X28);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved.x29)       == THREAD_CONTEXT_X29);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_lr)                     == THREAD_CONTEXT_LR);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_sp)                     == THREAD_CONTEXT_SP);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_cpacr)                  == THREAD_CONTEXT_CPACR);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_fpcr)                   == THREAD_CONTEXT_FPCR);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_fpsr)                   == THREAD_CONTEXT_FPSR);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_fpu_registers)          == THREAD_CONTEXT_FPU_REGISTERS);
+        static_assert(AMS_OFFSETOF(KThreadContext, m_locked)                 == THREAD_CONTEXT_LOCKED);
 
         return true;
     }

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_secure_monitor_base.hpp

@@ -0,0 +1,95 @@
+/*
+ * Copyright (c) Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_select_cpu.hpp>
+#include <mesosphere/kern_select_interrupt_manager.hpp>
+
+namespace ams::kern::arch::arm64::smc {
+
+    template<int SmcId, bool DisableInterrupt>
+    void SecureMonitorCall(u64 *buf) {
+        /* Load arguments into registers. */
+        register u64 x0 asm("x0") = buf[0];
+        register u64 x1 asm("x1") = buf[1];
+        register u64 x2 asm("x2") = buf[2];
+        register u64 x3 asm("x3") = buf[3];
+        register u64 x4 asm("x4") = buf[4];
+        register u64 x5 asm("x5") = buf[5];
+        register u64 x6 asm("x6") = buf[6];
+        register u64 x7 asm("x7") = buf[7];
+
+        /* Perform the call. */
+        if constexpr (DisableInterrupt) {
+            KScopedInterruptDisable di;
+
+            /* Backup the current thread pointer. */
+            const uintptr_t current_thread_pointer_value = cpu::GetCurrentThreadPointerValue();
+
+            __asm__ __volatile__("smc %c[smc_id]"
+                                : "+r"(x0), "+r"(x1), "+r"(x2), "+r"(x3), "+r"(x4), "+r"(x5), "+r"(x6), "+r"(x7)
+                                : [smc_id]"i"(SmcId)
+                                : "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x18", "cc", "memory"
+                                );
+
+            /* Restore the current thread pointer into X18. */
+            cpu::SetCurrentThreadPointerValue(current_thread_pointer_value);
+        } else {
+            /* Backup the current thread pointer. */
+            const uintptr_t current_thread_pointer_value = cpu::GetCurrentThreadPointerValue();
+
+            __asm__ __volatile__("smc %c[smc_id]"
+                                : "+r"(x0), "+r"(x1), "+r"(x2), "+r"(x3), "+r"(x4), "+r"(x5), "+r"(x6), "+r"(x7)
+                                : [smc_id]"i"(SmcId)
+                                : "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x18", "cc", "memory"
+                                );
+
+            /* Restore the current thread pointer into X18. */
+            cpu::SetCurrentThreadPointerValue(current_thread_pointer_value);
+        }
+
+        /* Store arguments to output. */
+        buf[0] = x0;
+        buf[1] = x1;
+        buf[2] = x2;
+        buf[3] = x3;
+        buf[4] = x4;
+        buf[5] = x5;
+        buf[6] = x6;
+        buf[7] = x7;
+    }
+
+    enum PsciFunction {
+        PsciFunction_CpuSuspend = 0xC4000001,
+        PsciFunction_CpuOff     = 0x84000002,
+        PsciFunction_CpuOn      = 0xC4000003,
+    };
+
+    template<int SmcId, bool DisableInterrupt>
+    u64 PsciCall(PsciFunction function, u64 x1 = 0, u64 x2 = 0, u64 x3 = 0, u64 x4 = 0, u64 x5 = 0, u64 x6 = 0, u64 x7 = 0) {
+        ams::svc::lp64::SecureMonitorArguments args = { { function, x1, x2, x3, x4, x5, x6, x7 } };
+
+        SecureMonitorCall<SmcId, DisableInterrupt>(args.r);
+
+        return args.r[0];
+    }
+
+    template<int SmcId, bool DisableInterrupt>
+    u64 CpuOn(u64 core_id, uintptr_t entrypoint, uintptr_t arg) {
+        return PsciCall<SmcId, DisableInterrupt>(PsciFunction_CpuOn, core_id, entrypoint, arg);
+    }
+
+}

libraries/libmesosphere/include/mesosphere/board/generic/kern_k_device_page_table.hpp

@@ -44,13 +44,13 @@ namespace ams::kern::board::generic {
                 return ams::kern::svc::ResultNotImplemented();
             }
 
-            Result ALWAYS_INLINE Map(size_t *out_mapped_size, const KPageGroup &pg, KDeviceVirtualAddress device_address, ams::svc::MemoryPermission device_perm, bool refresh_mappings) {
-                MESOSPHERE_UNUSED(out_mapped_size, pg, device_address, device_perm, refresh_mappings);
+            Result ALWAYS_INLINE Map(KProcessPageTable *page_table, KProcessAddress process_address, size_t size, KDeviceVirtualAddress device_address, ams::svc::MemoryPermission device_perm, bool is_aligned) {
+                MESOSPHERE_UNUSED(page_table, process_address, size, device_address, device_perm, is_aligned);
                 return ams::kern::svc::ResultNotImplemented();
             }
 
-            Result ALWAYS_INLINE Unmap(const KPageGroup &pg, KDeviceVirtualAddress device_address) {
-                MESOSPHERE_UNUSED(pg, device_address);
+            Result ALWAYS_INLINE Unmap(KProcessPageTable *page_table, KProcessAddress process_address, size_t size, KDeviceVirtualAddress device_address) {
+                MESOSPHERE_UNUSED(page_table, process_address, size, device_address);
                 return ams::kern::svc::ResultNotImplemented();
             }
 

libraries/libmesosphere/include/mesosphere/board/nintendo/nx/kern_k_device_page_table.hpp

@@ -61,7 +61,12 @@ namespace ams::kern::board::nintendo::nx {
                 return KPageTable::GetPageTablePhysicalAddress(addr);
             }
         public:
-            constexpr KDevicePageTable() : m_tables(), m_table_asids(), m_attached_device(), m_attached_value(), m_detached_value(), m_hs_attached_value(), m_hs_detached_value() { /* ... */ }
+            constexpr KDevicePageTable()
+                : m_tables{Null<KVirtualAddress>, Null<KVirtualAddress>, Null<KVirtualAddress>, Null<KVirtualAddress>},
+                  m_table_asids(), m_attached_device(), m_attached_value(), m_detached_value(), m_hs_attached_value(), m_hs_detached_value()
+            {
+                /* ... */
+            }
 
             Result Initialize(u64 space_address, u64 space_size);
             void Finalize();

libraries/libmesosphere/include/mesosphere/board/nintendo/nx/kern_k_memory_layout.hpp

@@ -15,9 +15,12 @@
  */
 #pragma once
 #include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_k_typed_address.hpp>
 
 namespace ams::kern {
 
+    constexpr inline KPhysicalAddress MainMemoryAddress = 0x80000000;
+
     constexpr inline size_t MainMemorySize    = 4_GB;
     constexpr inline size_t MainMemorySizeMax = 8_GB;
 

libraries/libmesosphere/include/mesosphere/board/nintendo/nx/kern_k_system_control.hpp

@@ -15,23 +15,17 @@
  */
 #pragma once
 #include <mesosphere/kern_common.hpp>
-
-namespace ams::kern {
-
-    struct InitialProcessBinaryLayout;
-
-}
+#include <mesosphere/kern_k_system_control_base.hpp>
 
 namespace ams::kern::board::nintendo::nx {
 
-    class KSystemControl {
+    class KSystemControl : public KSystemControlBase {
         public:
-            class Init {
+            class Init : public KSystemControlBase::Init {
                 public:
                     /* Initialization. */
+                    static size_t GetRealMemorySize();
                     static size_t GetIntendedMemorySize();
-                    static KPhysicalAddress GetKernelPhysicalBaseAddress(uintptr_t base_address);
-                    static void GetInitialProcessBinaryLayout(InitialProcessBinaryLayout *out);
                     static bool ShouldIncreaseThreadResourceLimit();
                     static void CpuOn(u64 core_id, uintptr_t entrypoint, uintptr_t arg);
                     static size_t GetApplicationPoolSize();
@@ -40,7 +34,7 @@ namespace ams::kern::board::nintendo::nx {
                     static u8 GetDebugLogUartPort();
 
                     /* Randomness. */
-                    static void GenerateRandomBytes(void *dst, size_t size);
+                    static void GenerateRandom(u64 *dst, size_t count);
                     static u64  GenerateRandomRange(u64 min, u64 max);
             };
         public:
@@ -50,7 +44,7 @@ namespace ams::kern::board::nintendo::nx {
             static NOINLINE u32 GetCreateProcessMemoryPool();
 
             /* Randomness. */
-            static void GenerateRandomBytes(void *dst, size_t size);
+            static void GenerateRandom(u64 *dst, size_t count);
             static u64  GenerateRandomRange(u64 min, u64 max);
             static u64  GenerateRandomU64();
 
@@ -58,23 +52,12 @@ namespace ams::kern::board::nintendo::nx {
             static void ReadWriteRegisterPrivileged(u32 *out, ams::svc::PhysicalAddress address, u32 mask, u32 value);
             static Result ReadWriteRegister(u32 *out, ams::svc::PhysicalAddress address, u32 mask, u32 value);
 
-            static ALWAYS_INLINE u32 ReadRegisterPrivileged(ams::svc::PhysicalAddress address) {
-                u32 v;
-                ReadWriteRegisterPrivileged(std::addressof(v), address, 0x00000000u, 0);
-                return v;
-            }
-
-            static ALWAYS_INLINE void WriteRegisterPrivileged(ams::svc::PhysicalAddress address, u32 value) {
-                u32 v;
-                ReadWriteRegisterPrivileged(std::addressof(v), address, 0xFFFFFFFFu, value);
-            }
-
             /* Power management. */
             static void SleepSystem();
             static NORETURN void StopSystem(void *arg = nullptr);
 
             /* User access. */
-            static void CallSecureMonitorFromUser(ams::svc::lp64::SecureMonitorArguments *args);
+            static void CallSecureMonitorFromUserImpl(ams::svc::lp64::SecureMonitorArguments *args);
 
             /* Secure Memory. */
             static size_t CalculateRequiredSecureMemorySize(size_t size, u32 pool);

libraries/libmesosphere/include/mesosphere/board/qemu/virt/kern_cpu_map.hpp

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+
+namespace ams::kern::board::qemu::virt::impl::cpu {
+
+    /* Virtual to Physical core map. */
+    constexpr inline const s32 VirtualToPhysicalCoreMap[BITSIZEOF(u64)] = {
+        0, 1, 2, 3, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 3,
+    };
+
+}

libraries/libmesosphere/include/mesosphere/board/qemu/virt/kern_k_memory_layout.hpp

@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_k_typed_address.hpp>
+
+namespace ams::kern {
+
+    constexpr inline KPhysicalAddress MainMemoryAddress = 0x40000000;
+
+    constexpr inline size_t MainMemorySize    = 4_GB;
+    constexpr inline size_t MainMemorySizeMax = 8_GB;
+
+}

libraries/libmesosphere/include/mesosphere/board/qemu/virt/kern_k_memory_region_device_types.inc

@@ -0,0 +1,20 @@
+/*
+ * Copyright (c) Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/* All architectures must define NumBoardDeviceRegions. */
+constexpr inline const auto NumBoardDeviceRegions = 0;
+                                                                                                /* UNUSED: .Derive(NumBoardDeviceRegions, 0); */
+

libraries/libmesosphere/include/mesosphere/board/qemu/virt/kern_k_system_control.hpp

@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_k_system_control_base.hpp>
+
+namespace ams::kern::board::qemu::virt {
+
+    class KSystemControl : public KSystemControlBase {
+        public:
+            /* User access. */
+            static void CallSecureMonitorFromUser(ams::svc::lp64::SecureMonitorArguments *args);
+    };
+
+}

libraries/libmesosphere/include/mesosphere/kern_build_config.hpp

@@ -32,3 +32,19 @@
 //#define MESOSPHERE_BUILD_FOR_TRACING
 #define MESOSPHERE_ENABLE_PANIC_REGISTER_DUMP
 #define MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP
+
+/* NOTE: This enables fast class token storage using a class member. */
+/* This saves a virtual call when doing KAutoObject->DynCast<>(), */
+/* at the cost of storing class tokens inside the class object. */
+/* However, as of (10/16/2021) KAutoObject has an unused class member */
+/* of the right side, and so this does not actually cost any space. */
+#define MESOSPHERE_ENABLE_DEVIRTUALIZED_DYNAMIC_CAST
+
+/* NOTE: This uses currently-reserved bits inside the MapRange capability */
+/* in order to support large physical addresses (40-bit instead of 36). */
+/* This is toggleable in order to disable it if N ever uses those bits. */
+#if defined(ATMOSPHERE_BOARD_NINTENDO_NX)
+//#define MESOSPHERE_ENABLE_LARGE_PHYSICAL_ADDRESS_CAPABILITIES
+#else
+#define MESOSPHERE_ENABLE_LARGE_PHYSICAL_ADDRESS_CAPABILITIES
+#endif

libraries/libmesosphere/include/mesosphere/kern_debug_log.hpp

@@ -43,6 +43,8 @@ namespace ams::kern {
 
     #ifdef ATMOSPHERE_BOARD_NINTENDO_NX
         #define MESOSPHERE_DEBUG_LOG_USE_UART
+    #elif defined(ATMOSPHERE_BOARD_QEMU_VIRT)
+        #define MESOSPHERE_DEBUG_LOG_USE_SEMIHOSTING
     #else
         #error "Unknown board for Default Debug Log Source"
     #endif

libraries/libmesosphere/include/mesosphere/kern_k_address_arbiter.hpp

@@ -25,7 +25,7 @@ namespace ams::kern {
         private:
             ThreadTree m_tree;
         public:
-            constexpr KAddressArbiter() : m_tree() { /* ... */ }
+            constexpr KAddressArbiter() = default;
 
             Result SignalToAddress(uintptr_t addr, ams::svc::SignalType type, s32 value, s32 count) {
                 switch (type) {

libraries/libmesosphere/include/mesosphere/kern_k_auto_object.hpp

@@ -22,27 +22,73 @@ namespace ams::kern {
 
     class KProcess;
 
+    #if defined(MESOSPHERE_BUILD_FOR_DEBUGGING) || defined(MESOSPHERE_BUILD_FOR_AUDITING)
+    #define MESOSPHERE_AUTO_OBJECT_TYPENAME_IMPL(CLASS) #CLASS
+    #else
+    #define MESOSPHERE_AUTO_OBJECT_TYPENAME_IMPL(CLASS) ""
+    #endif
+
     #define MESOSPHERE_AUTOOBJECT_TRAITS(CLASS, BASE_CLASS)                                                     \
         NON_COPYABLE(CLASS);                                                                                    \
         NON_MOVEABLE(CLASS);                                                                                    \
         private:                                                                                                \
             friend class ::ams::kern::KClassTokenGenerator;                                                     \
             static constexpr inline auto ObjectType = ::ams::kern::KClassTokenGenerator::ObjectType::CLASS;     \
-            static constexpr inline const char * const TypeName = #CLASS;                                       \
+            static constexpr inline const char * const TypeName = MESOSPHERE_AUTO_OBJECT_TYPENAME_IMPL(CLASS);  \
             static constexpr inline ClassTokenType ClassToken() { return ::ams::kern::ClassToken<CLASS>; }      \
         public:                                                                                                 \
             using BaseClass = BASE_CLASS;                                                                       \
-            static constexpr ALWAYS_INLINE TypeObj GetStaticTypeObj() {                                         \
+            static consteval ALWAYS_INLINE TypeObj GetStaticTypeObj() {                                         \
                 constexpr ClassTokenType Token = ClassToken();                                                  \
                 return TypeObj(TypeName, Token);                                                                \
             }                                                                                                   \
-            static constexpr ALWAYS_INLINE const char *GetStaticTypeName() { return TypeName; }                 \
+            static consteval ALWAYS_INLINE const char *GetStaticTypeName() { return TypeName; }                 \
             virtual TypeObj GetTypeObj() const { return GetStaticTypeObj(); }                                   \
             virtual const char *GetTypeName() { return GetStaticTypeName(); }                                   \
         private:
 
-
     class KAutoObject {
+        public:
+            class ReferenceCount {
+                NON_COPYABLE(ReferenceCount);
+                NON_MOVEABLE(ReferenceCount);
+                private:
+                    using Storage = u32;
+                private:
+                    util::Atomic<Storage> m_value;
+                public:
+                    ALWAYS_INLINE explicit ReferenceCount() { /* ... */ }
+                    constexpr ALWAYS_INLINE explicit ReferenceCount(Storage v) : m_value(v) { /* ... */ }
+
+                    ALWAYS_INLINE void operator=(Storage v) { m_value = v; }
+
+                    ALWAYS_INLINE Storage GetValue() const { return m_value.Load(); }
+
+                    ALWAYS_INLINE bool Open() {
+                        /* Atomically increment the reference count, only if it's positive. */
+                        u32 cur = m_value.Load<std::memory_order_relaxed>();
+                        do {
+                            if (AMS_UNLIKELY(cur == 0)) {
+                                MESOSPHERE_AUDIT(cur != 0);
+                                return false;
+                            }
+                            MESOSPHERE_ABORT_UNLESS(cur < cur + 1);
+                        } while (AMS_UNLIKELY(!m_value.CompareExchangeWeak<std::memory_order_relaxed>(cur, cur + 1)));
+
+                        return true;
+                    }
+
+                    ALWAYS_INLINE bool Close() {
+                        /* Atomically decrement the reference count, not allowing it to become negative. */
+                        u32 cur = m_value.Load<std::memory_order_relaxed>();
+                        do {
+                            MESOSPHERE_ABORT_UNLESS(cur > 0);
+                        } while (AMS_UNLIKELY(!m_value.CompareExchangeWeak<std::memory_order_relaxed>(cur, cur - 1)));
+
+                        /* Return whether the object was closed. */
+                        return cur - 1 == 0;
+                    }
+            };
         protected:
             class TypeObj {
                 private:
@@ -63,37 +109,61 @@ namespace ams::kern {
                     }
 
                     constexpr ALWAYS_INLINE bool IsDerivedFrom(const TypeObj &rhs) {
-                        return (this->GetClassToken() | rhs.GetClassToken()) == this->GetClassToken();
+                        return IsClassTokenDerivedFrom(this->GetClassToken(), rhs.GetClassToken());
+                    }
+
+                    static constexpr ALWAYS_INLINE bool IsClassTokenDerivedFrom(ClassTokenType derived, ClassTokenType base) {
+                        return (derived | base) == derived;
                     }
             };
         private:
             MESOSPHERE_AUTOOBJECT_TRAITS(KAutoObject, KAutoObject);
         private:
             KAutoObject *m_next_closed_object;
-            std::atomic<u32> m_ref_count;
+            ReferenceCount m_ref_count;
+            #if defined(MESOSPHERE_ENABLE_DEVIRTUALIZED_DYNAMIC_CAST)
+            ClassTokenType m_class_token;
+            #endif
         public:
-            static KAutoObject *Create(KAutoObject *ptr);
-        public:
-            constexpr ALWAYS_INLINE explicit KAutoObject() : m_next_closed_object(nullptr), m_ref_count(0) { MESOSPHERE_ASSERT_THIS(); }
+            constexpr ALWAYS_INLINE explicit KAutoObject(util::ConstantInitializeTag) : m_next_closed_object(nullptr), m_ref_count(0)
+            #if defined(MESOSPHERE_ENABLE_DEVIRTUALIZED_DYNAMIC_CAST)
+                , m_class_token(0)
+            #endif
+            {
+                MESOSPHERE_ASSERT_THIS();
+            }
+
+            ALWAYS_INLINE explicit KAutoObject() : m_ref_count(0) { MESOSPHERE_ASSERT_THIS(); }
 
             /* Destroy is responsible for destroying the auto object's resources when ref_count hits zero. */
             virtual void Destroy() { MESOSPHERE_ASSERT_THIS(); }
 
             /* Finalize is responsible for cleaning up resource, but does not destroy the object. */
-            virtual void Finalize() { MESOSPHERE_ASSERT_THIS(); }
+            /* NOTE: This is a virtual function in official kernel, but because everything which uses it */
+            /* is already using CRTP for slab heap, we have devirtualized it for performance gain. */
+            /* virtual void Finalize() { MESOSPHERE_ASSERT_THIS(); } */
 
-            virtual KProcess *GetOwner() const { return nullptr; }
+            /* NOTE: This is a virtual function which is unused-except-for-debug in Nintendo's kernel. */
+            /* virtual KProcess *GetOwner() const { return nullptr; } */
 
             u32 GetReferenceCount() const {
-                return m_ref_count.load();
+                return m_ref_count.GetValue();
             }
 
             ALWAYS_INLINE bool IsDerivedFrom(const TypeObj &rhs) const {
-                return this->GetTypeObj().IsDerivedFrom(rhs);
+                #if defined(MESOSPHERE_ENABLE_DEVIRTUALIZED_DYNAMIC_CAST)
+                    return TypeObj::IsClassTokenDerivedFrom(m_class_token, rhs.GetClassToken());
+                #else
+                    return this->GetTypeObj().IsDerivedFrom(rhs);
+                #endif
             }
 
             ALWAYS_INLINE bool IsDerivedFrom(const KAutoObject &rhs) const {
-                return this->IsDerivedFrom(rhs.GetTypeObj());
+                #if defined(MESOSPHERE_ENABLE_DEVIRTUALIZED_DYNAMIC_CAST)
+                    return TypeObj::IsClassTokenDerivedFrom(m_class_token, rhs.m_class_token);
+                #else
+                    return this->IsDerivedFrom(rhs.GetTypeObj());
+                #endif
             }
 
             template<typename Derived>
@@ -120,8 +190,19 @@ namespace ams::kern {
                 }
             }
 
-            bool Open();
-            void Close();
+            NOINLINE bool Open() {
+                MESOSPHERE_ASSERT_THIS();
+
+                return m_ref_count.Open();
+            }
+
+            NOINLINE void Close() {
+                MESOSPHERE_ASSERT_THIS();
+
+                if (m_ref_count.Close()) {
+                    this->ScheduleDestruction();
+                }
+            }
         private:
             /* NOTE: This has to be defined *after* KThread is defined. */
             /* Nintendo seems to handle this by defining Open/Close() in a cpp, but we'd like them to remain in headers. */
@@ -130,34 +211,46 @@ namespace ams::kern {
         public:
             /* Getter, for KThread. */
             ALWAYS_INLINE KAutoObject *GetNextClosedObject() { return m_next_closed_object; }
+        public:
+            template<typename Derived> requires (std::derived_from<Derived, KAutoObject>)
+            static ALWAYS_INLINE void Create(typename std::type_identity<Derived>::type *obj) {
+                /* Get auto object pointer. */
+                KAutoObject &auto_object = *static_cast<KAutoObject *>(obj);
+
+                /* If we should, set our class token. */
+                #if defined(MESOSPHERE_ENABLE_DEVIRTUALIZED_DYNAMIC_CAST)
+                {
+                    constexpr auto Token = Derived::GetStaticTypeObj().GetClassToken();
+                    auto_object.m_class_token = Token;
+                }
+                #endif
+
+                /* Initialize reference count to 1. */
+                auto_object.m_ref_count = 1;
+            }
     };
 
-    class KAutoObjectWithListContainer;
-
-    class KAutoObjectWithList : public KAutoObject {
+    class KAutoObjectWithListBase : public KAutoObject {
         private:
+            void *m_alignment_forcer_unused[0];
+        public:
+            constexpr ALWAYS_INLINE explicit KAutoObjectWithListBase(util::ConstantInitializeTag) : KAutoObject(util::ConstantInitialize), m_alignment_forcer_unused{} { /* ... */ }
+
+            ALWAYS_INLINE explicit KAutoObjectWithListBase() { /* ... */ }
+    };
+
+    class KAutoObjectWithList : public KAutoObjectWithListBase {
+        private:
+            template<typename>
             friend class KAutoObjectWithListContainer;
         private:
             util::IntrusiveRedBlackTreeNode m_list_node;
         public:
-            constexpr ALWAYS_INLINE KAutoObjectWithList() : m_list_node() { /* ... */ }
-
-            static ALWAYS_INLINE int Compare(const KAutoObjectWithList &lhs, const KAutoObjectWithList &rhs) {
-                const u64 lid = lhs.GetId();
-                const u64 rid = rhs.GetId();
-
-                if (lid < rid) {
-                    return -1;
-                } else if (lid > rid) {
-                    return 1;
-                } else {
-                    return 0;
-                }
-            }
+            constexpr ALWAYS_INLINE KAutoObjectWithList(util::ConstantInitializeTag) : KAutoObjectWithListBase(util::ConstantInitialize), m_list_node(util::ConstantInitialize) { /* ... */ }
+            ALWAYS_INLINE explicit KAutoObjectWithList() { /* ... */ }
         public:
-            virtual u64 GetId() const {
-                return reinterpret_cast<u64>(this);
-            }
+            /* NOTE: This is virtual in Nintendo's kernel. */
+            u64 GetId() const;
     };
 
     template<typename T> requires std::derived_from<T, KAutoObject>
@@ -173,7 +266,6 @@ namespace ams::kern {
                 std::swap(m_obj, rhs.m_obj);
             }
         public:
-            constexpr ALWAYS_INLINE KScopedAutoObject() : m_obj(nullptr) { /* ... */ }
             constexpr ALWAYS_INLINE KScopedAutoObject(T *o) : m_obj(o) {
                 if (m_obj != nullptr) {
                     m_obj->Open();
@@ -228,5 +320,52 @@ namespace ams::kern {
             constexpr ALWAYS_INLINE bool IsNotNull() const { return m_obj != nullptr; }
     };
 
+    template<typename T> requires std::derived_from<T, KAutoObject>
+    class KSharedAutoObject {
+        private:
+            T *m_object;
+            KAutoObject::ReferenceCount m_ref_count;
+        public:
+            explicit KSharedAutoObject() : m_object(nullptr) { /* ... */ }
+
+            void Attach(T *obj) {
+                MESOSPHERE_ASSERT(m_object == nullptr);
+
+                /* Set our object. */
+                m_object = obj;
+
+                /* Open reference to our object. */
+                m_object->Open();
+
+                /* Set our reference count. */
+                m_ref_count = 1;
+            }
+
+            bool Open() {
+                return m_ref_count.Open();
+            }
+
+            void Close() {
+                if (m_ref_count.Close()) {
+                    this->Detach();
+                }
+            }
+
+            ALWAYS_INLINE T *Get() const {
+                return m_object;
+            }
+        private:
+            void Detach() {
+                /* Close our object, if we have one. */
+                if (T * const object = m_object; AMS_LIKELY(object != nullptr)) {
+                    /* Set our object to a debug sentinel value, which will cause crash if accessed. */
+                    m_object = reinterpret_cast<T *>(1);
+
+                    /* Close reference to our object. */
+                    object->Close();
+                }
+            }
+    };
+
 
 }

libraries/libmesosphere/include/mesosphere/kern_k_auto_object_container.hpp

@@ -20,45 +20,131 @@
 
 namespace ams::kern {
 
-    class KAutoObjectWithListContainer {
-        NON_COPYABLE(KAutoObjectWithListContainer);
-        NON_MOVEABLE(KAutoObjectWithListContainer);
-        public:
-            using ListType = util::IntrusiveRedBlackTreeMemberTraits<&KAutoObjectWithList::m_list_node>::TreeType<KAutoObjectWithList>;
-        public:
-            class ListAccessor : public KScopedLightLock {
-                private:
-                    ListType &m_list;
-                public:
-                    explicit ListAccessor(KAutoObjectWithListContainer *container) : KScopedLightLock(container->m_lock), m_list(container->m_object_list) { /* ... */ }
-                    explicit ListAccessor(KAutoObjectWithListContainer &container) : KScopedLightLock(container.m_lock), m_list(container.m_object_list) { /* ... */ }
+    namespace impl {
 
-                    typename ListType::iterator begin() const {
-                        return m_list.begin();
+        template<typename T>
+        struct GetAutoObjectWithListComparator;
+
+        class KAutoObjectWithListContainerBase {
+            NON_COPYABLE(KAutoObjectWithListContainerBase);
+            NON_MOVEABLE(KAutoObjectWithListContainerBase);
+            protected:
+                template<typename ListType>
+                class ListAccessorImpl {
+                    NON_COPYABLE(ListAccessorImpl);
+                    NON_MOVEABLE(ListAccessorImpl);
+                    private:
+                        KScopedLightLock m_lk;
+                        ListType &m_list;
+                    public:
+                        explicit ALWAYS_INLINE ListAccessorImpl(KAutoObjectWithListContainerBase *container, ListType &list) : m_lk(container->m_lock), m_list(list) { /* ... */ }
+                        explicit ALWAYS_INLINE ListAccessorImpl(KAutoObjectWithListContainerBase &container, ListType &list) : m_lk(container.m_lock), m_list(list) { /* ... */ }
+
+                        ALWAYS_INLINE ~ListAccessorImpl() { /* ... */ }
+
+                        ALWAYS_INLINE typename ListType::iterator begin() const {
+                            return m_list.begin();
+                        }
+
+                        ALWAYS_INLINE typename ListType::iterator end() const {
+                            return m_list.end();
+                        }
+
+                        ALWAYS_INLINE typename ListType::iterator find(typename ListType::const_reference ref) const {
+                            return m_list.find(ref);
+                        }
+
+                        ALWAYS_INLINE typename ListType::iterator find_key(typename ListType::const_key_reference ref) const {
+                            return m_list.find_key(ref);
+                        }
+                };
+
+                template<typename ListType>
+                friend class ListAccessorImpl;
+            private:
+                KLightLock m_lock;
+            protected:
+                constexpr KAutoObjectWithListContainerBase() : m_lock() { /* ... */ }
+
+                ALWAYS_INLINE void InitializeImpl() { MESOSPHERE_ASSERT_THIS(); }
+                ALWAYS_INLINE void FinalizeImpl() { MESOSPHERE_ASSERT_THIS(); }
+
+                template<typename ListType>
+                void RegisterImpl(KAutoObjectWithList *obj, ListType &list) {
+                    MESOSPHERE_ASSERT_THIS();
+
+                    KScopedLightLock lk(m_lock);
+
+                    list.insert(*obj);
+                }
+
+                template<typename ListType>
+                void UnregisterImpl(KAutoObjectWithList *obj, ListType &list) {
+                    MESOSPHERE_ASSERT_THIS();
+
+                    KScopedLightLock lk(m_lock);
+
+                    list.erase(list.iterator_to(*obj));
+                }
+
+                template<typename U, typename ListType>
+                size_t GetOwnedCountImpl(const KProcess *owner, ListType &list) {
+                    MESOSPHERE_ASSERT_THIS();
+
+                    KScopedLightLock lk(m_lock);
+
+                    size_t count = 0;
+
+                    for (const auto &obj : list) {
+                        MESOSPHERE_AUDIT(obj.DynamicCast<const U *>() != nullptr);
+                        if (static_cast<const U &>(obj).GetOwner() == owner) {
+                            ++count;
+                        }
                     }
 
-                    typename ListType::iterator end() const {
-                        return m_list.end();
-                    }
+                    return count;
+                }
+        };
 
-                    typename ListType::iterator find(typename ListType::const_reference ref) const {
-                        return m_list.find(ref);
-                    }
-            };
+        struct DummyKAutoObjectWithListComparator {
+            static NOINLINE int Compare(KAutoObjectWithList &lhs, KAutoObjectWithList &rhs) {
+                MESOSPHERE_UNUSED(lhs, rhs);
+                MESOSPHERE_PANIC("DummyKAutoObjectWithListComparator invoked");
+            }
+        };
 
-            friend class ListAccessor;
+    }
+
+    template<typename T>
+    class KAutoObjectWithListContainer : public impl::KAutoObjectWithListContainerBase {
         private:
-            KLightLock m_lock;
-            ListType m_object_list;
+            using Base = impl::KAutoObjectWithListContainerBase;
         public:
-            constexpr KAutoObjectWithListContainer() : m_lock(), m_object_list() { MESOSPHERE_ASSERT_THIS(); }
+            class ListAccessor;
+            friend class ListAccessor;
 
-            void Initialize() { MESOSPHERE_ASSERT_THIS(); }
-            void Finalize() { MESOSPHERE_ASSERT_THIS(); }
+            template<typename Comparator>
+            using ListType = util::IntrusiveRedBlackTreeMemberTraits<&KAutoObjectWithList::m_list_node>::TreeType<Comparator>;
 
-            void Register(KAutoObjectWithList *obj);
-            void Unregister(KAutoObjectWithList *obj);
-            size_t GetOwnedCount(KProcess *owner);
+            using DummyListType = ListType<impl::DummyKAutoObjectWithListComparator>;
+        private:
+            DummyListType m_dummy_object_list;
+        public:
+            constexpr ALWAYS_INLINE KAutoObjectWithListContainer() : Base(), m_dummy_object_list() { static_assert(std::derived_from<T, KAutoObjectWithList>); }
+
+            ALWAYS_INLINE void Initialize() { return this->InitializeImpl(); }
+            ALWAYS_INLINE void Finalize() { return this->FinalizeImpl(); }
+
+            void Register(T *obj);
+            void Unregister(T *obj);
+
+        private:
+            size_t GetOwnedCountChecked(const KProcess *owner);
+        public:
+            template<typename U> requires (std::same_as<U, T> && requires (const U &u) { { u.GetOwner() } -> std::convertible_to<const KProcess *>; })
+            ALWAYS_INLINE size_t GetOwnedCount(const KProcess *owner) {
+                return this->GetOwnedCountChecked(owner);
+            }
     };
 
 

libraries/libmesosphere/include/mesosphere/kern_k_auto_object_impls.hpp

@@ -0,0 +1,163 @@
+/*
+ * Copyright (c) Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_k_class_token.hpp>
+
+namespace ams::kern {
+
+    /* NOTE: This header is included after all other KAutoObjects. */
+    namespace impl {
+
+        template<typename T> requires std::derived_from<T, KAutoObject>
+        consteval bool IsAutoObjectInheritanceValidImpl() {
+            #define CLASS_TOKEN_HANDLER(CLASSNAME)                                                                               \
+                if constexpr (std::same_as<T, CLASSNAME>) {                                                                      \
+                    if (T::GetStaticTypeObj().GetClassToken() != ::ams::kern::ClassToken<CLASSNAME>) {                           \
+                        return false;                                                                                            \
+                    }                                                                                                            \
+                } else {                                                                                                         \
+                    if (T::GetStaticTypeObj().IsDerivedFrom(CLASSNAME::GetStaticTypeObj()) != std::derived_from<T, CLASSNAME>) { \
+                        return false;                                                                                            \
+                    }                                                                                                            \
+                }
+
+            FOR_EACH_K_CLASS_TOKEN_OBJECT_TYPE(CLASS_TOKEN_HANDLER)
+            #undef CLASS_TOKEN_HANDLER
+
+            return true;
+        }
+
+        consteval bool IsEveryAutoObjectInheritanceValid() {
+            #define CLASS_TOKEN_HANDLER(CLASSNAME) if (!IsAutoObjectInheritanceValidImpl<CLASSNAME>()) { return false; }
+            FOR_EACH_K_CLASS_TOKEN_OBJECT_TYPE(CLASS_TOKEN_HANDLER)
+            #undef CLASS_TOKEN_HANDLER
+
+            return true;
+        }
+
+        static_assert(IsEveryAutoObjectInheritanceValid());
+
+        template<typename T>
+        concept IsAutoObjectWithSpecializedGetId = std::derived_from<T, KAutoObjectWithList> && requires (const T &t, const KAutoObjectWithList &l) {
+            { t.GetIdImpl() } -> std::same_as<decltype(l.GetId())>;
+        };
+
+        template<typename T>
+        struct AutoObjectWithListComparatorImpl {
+            using RedBlackKeyType = u64;
+
+            static ALWAYS_INLINE RedBlackKeyType GetRedBlackKey(const RedBlackKeyType &v) { return v; }
+
+            static ALWAYS_INLINE RedBlackKeyType GetRedBlackKey(const KAutoObjectWithList &v) {
+                if constexpr (IsAutoObjectWithSpecializedGetId<T>) {
+                    return static_cast<const T &>(v).GetIdImpl();
+                } else {
+                    return reinterpret_cast<u64>(std::addressof(v));
+                }
+            }
+
+            template<typename U> requires (std::same_as<U, KAutoObjectWithList> || std::same_as<U, RedBlackKeyType>)
+            static ALWAYS_INLINE int Compare(const U &lhs, const KAutoObjectWithList &rhs) {
+                const u64 lid = GetRedBlackKey(lhs);
+                const u64 rid = GetRedBlackKey(rhs);
+
+                if (lid < rid) {
+                    return -1;
+                } else if (lid > rid) {
+                    return 1;
+                } else {
+                    return 0;
+                }
+            }
+        };
+
+        template<typename T>
+        using AutoObjectWithListComparator = AutoObjectWithListComparatorImpl<typename std::conditional<IsAutoObjectWithSpecializedGetId<T>, T, KAutoObjectWithList>::type>;
+
+        template<typename T>
+        using TrueObjectContainerListType = typename KAutoObjectWithListContainer<T>::ListType<AutoObjectWithListComparator<T>>;
+
+        template<typename T>
+        ALWAYS_INLINE TrueObjectContainerListType<T> &GetTrueObjectContainerList(typename KAutoObjectWithListContainer<T>::DummyListType &l) {
+            static_assert(alignof(l) == alignof(impl::TrueObjectContainerListType<T>));
+            static_assert(sizeof(l)  == sizeof(impl::TrueObjectContainerListType<T>));
+            return *reinterpret_cast<TrueObjectContainerListType<T> *>(std::addressof(l));
+        }
+
+    }
+
+    ALWAYS_INLINE void KAutoObject::ScheduleDestruction() {
+        MESOSPHERE_ASSERT_THIS();
+
+        /* Set our object to destroy. */
+        m_next_closed_object = GetCurrentThread().GetClosedObject();
+
+        /* Set ourselves as the thread's next object to destroy. */
+        GetCurrentThread().SetClosedObject(this);
+    }
+
+    template<typename T>
+    class KAutoObjectWithListContainer<T>::ListAccessor : public impl::KAutoObjectWithListContainerBase::ListAccessorImpl<impl::TrueObjectContainerListType<T>> {
+        NON_COPYABLE(ListAccessor);
+        NON_MOVEABLE(ListAccessor);
+        private:
+            using BaseListAccessor = impl::KAutoObjectWithListContainerBase::ListAccessorImpl<impl::TrueObjectContainerListType<T>>;
+        public:
+            explicit ALWAYS_INLINE ListAccessor(KAutoObjectWithListContainer *container) : BaseListAccessor(container, impl::GetTrueObjectContainerList<T>(container->m_dummy_object_list)) { /* ... */ }
+            explicit ALWAYS_INLINE ListAccessor(KAutoObjectWithListContainer &container) : BaseListAccessor(container, impl::GetTrueObjectContainerList<T>(container.m_dummy_object_list)) { /* ... */ }
+
+            ALWAYS_INLINE ~ListAccessor() { /* ... */ }
+    };
+
+
+    template<typename T>
+    ALWAYS_INLINE void KAutoObjectWithListContainer<T>::Register(T *obj) {
+        return this->RegisterImpl(obj, impl::GetTrueObjectContainerList<T>(m_dummy_object_list));
+    }
+
+    template<typename T>
+    ALWAYS_INLINE void KAutoObjectWithListContainer<T>::Unregister(T *obj) {
+        return this->UnregisterImpl(obj, impl::GetTrueObjectContainerList<T>(m_dummy_object_list));
+    }
+
+    template<typename T>
+    ALWAYS_INLINE size_t KAutoObjectWithListContainer<T>::GetOwnedCountChecked(const KProcess *owner) {
+        return this->GetOwnedCountImpl<T>(owner, impl::GetTrueObjectContainerList<T>(m_dummy_object_list));
+    }
+
+    inline u64 KAutoObjectWithList::GetId() const {
+        #define CLASS_TOKEN_HANDLER(CLASSNAME)                                                                      \
+            if constexpr (impl::IsAutoObjectWithSpecializedGetId<CLASSNAME>) {                                      \
+                if (const CLASSNAME * const derived = this->DynamicCast<const CLASSNAME *>(); derived != nullptr) { \
+                    return []<typename T>(const T * const t_derived) ALWAYS_INLINE_LAMBDA -> u64 {                  \
+                        static_assert(std::same_as<T, CLASSNAME>);                                                  \
+                        if constexpr (impl::IsAutoObjectWithSpecializedGetId<CLASSNAME>) {                          \
+                            return impl::AutoObjectWithListComparator<CLASSNAME>::GetRedBlackKey(*t_derived);       \
+                        } else {                                                                                    \
+                            AMS_ASSUME(false);                                                                      \
+                        }                                                                                           \
+                    }(derived);                                                                                     \
+                }                                                                                                   \
+            }
+
+        FOR_EACH_K_CLASS_TOKEN_OBJECT_TYPE(CLASS_TOKEN_HANDLER)
+        #undef CLASS_TOKEN_HANDLER
+
+        return impl::AutoObjectWithListComparator<KAutoObjectWithList>::GetRedBlackKey(*this);
+    }
+
+}

libraries/libmesosphere/include/mesosphere/kern_k_capabilities.hpp

@@ -55,48 +55,16 @@ namespace ams::kern {
                 return static_cast<u32>(type) + 1;
             }
 
-            static constexpr u32 CountTrailingZero(u32 flag) {
-                for (u32 i = 0; i < BITSIZEOF(u32); i++) {
-                    if (flag & (1u << i)) {
-                        return i;
-                    }
-                }
-                return BITSIZEOF(u32);
-            }
-
-            static constexpr u32 GetCapabilityId(CapabilityType type) {
-                const u32 flag = GetCapabilityFlag(type);
-                if (std::is_constant_evaluated()) {
-                    return CountTrailingZero(flag);
-                } else {
-                    return static_cast<u32>(__builtin_ctz(flag));
-                }
-            }
-
             template<size_t Index, size_t Count, typename T = u32>
             using Field = util::BitPack32::Field<Index, Count, T>;
 
             #define DEFINE_FIELD(name, prev, ...) using name = Field<prev::Next, __VA_ARGS__>
 
             template<CapabilityType Type>
-            static constexpr inline u32 CapabilityFlag = []() -> u32 {
-                return static_cast<u32>(Type) + 1;
-            }();
+            static constexpr inline u32 CapabilityFlag = static_cast<u32>(Type) + 1;
 
             template<CapabilityType Type>
-            static constexpr inline u32 CapabilityId = []() -> u32 {
-                const u32 flag = static_cast<u32>(Type) + 1;
-                if (std::is_constant_evaluated()) {
-                    for (u32 i = 0; i < BITSIZEOF(u32); i++) {
-                        if (flag & (1u << i)) {
-                            return i;
-                        }
-                    }
-                    return BITSIZEOF(u32);
-                } else {
-                    return __builtin_ctz(flag);
-                }
-            }();
+            static constexpr inline u32 CapabilityId = util::CountTrailingZeros<u32>(CapabilityFlag<Type>);
 
             struct CorePriority {
                 using IdBits = Field<0, CapabilityId<CapabilityType::CorePriority> + 1>;
@@ -114,7 +82,11 @@ namespace ams::kern {
                 DEFINE_FIELD(Index, Mask,    3);
             };
 
+            #if defined(MESOSPHERE_ENABLE_LARGE_PHYSICAL_ADDRESS_CAPABILITIES)
+            static constexpr u64 PhysicalMapAllowedMask = (1ul << 40) - 1;
+            #else
             static constexpr u64 PhysicalMapAllowedMask = (1ul << 36) - 1;
+            #endif
 
             struct MapRange {
                 using IdBits = Field<0, CapabilityId<CapabilityType::MapRange> + 1>;
@@ -126,9 +98,15 @@ namespace ams::kern {
             struct MapRangeSize {
                 using IdBits = Field<0, CapabilityId<CapabilityType::MapRange> + 1>;
 
-                DEFINE_FIELD(Pages,    IdBits,   20);
+                DEFINE_FIELD(Pages, IdBits, 20);
+
+                #if defined(MESOSPHERE_ENABLE_LARGE_PHYSICAL_ADDRESS_CAPABILITIES)
+                DEFINE_FIELD(AddressHigh, Pages,        4);
+                DEFINE_FIELD(Normal,      AddressHigh,  1, bool);
+                #else
                 DEFINE_FIELD(Reserved, Pages,     4);
                 DEFINE_FIELD(Normal,   Reserved,  1, bool);
+                #endif
             };
 
             struct MapIoPage {
@@ -203,14 +181,14 @@ namespace ams::kern {
                                                        CapabilityFlag<CapabilityType::HandleTable>   |
                                                        CapabilityFlag<CapabilityType::DebugFlags>;
         private:
-            svc::SvcAccessFlagSet m_svc_access_flags{};
-            InterruptFlagSet m_irq_access_flags{};
-            u64 m_core_mask{};
-            u64 m_priority_mask{};
-            util::BitPack32 m_debug_capabilities{0};
-            s32 m_handle_table_size{};
-            util::BitPack32 m_intended_kernel_version{0};
-            u32 m_program_type{};
+            svc::SvcAccessFlagSet m_svc_access_flags;
+            InterruptFlagSet m_irq_access_flags;
+            u64 m_core_mask;
+            u64 m_priority_mask;
+            util::BitPack32 m_debug_capabilities;
+            s32 m_handle_table_size;
+            util::BitPack32 m_intended_kernel_version;
+            u32 m_program_type;
         private:
             constexpr bool SetSvcAllowed(u32 id) {
                 if (AMS_LIKELY(id < m_svc_access_flags.GetCount())) {
@@ -245,7 +223,8 @@ namespace ams::kern {
             Result SetCapabilities(const u32 *caps, s32 num_caps, KProcessPageTable *page_table);
             Result SetCapabilities(svc::KUserPointer<const u32 *> user_caps, s32 num_caps, KProcessPageTable *page_table);
         public:
-            constexpr KCapabilities() = default;
+            constexpr explicit KCapabilities(util::ConstantInitializeTag) : m_svc_access_flags{}, m_irq_access_flags{}, m_core_mask{}, m_priority_mask{}, m_debug_capabilities{0}, m_handle_table_size{}, m_intended_kernel_version{}, m_program_type{} { /* ... */ }
+            KCapabilities() { /* ... */ }
 
             Result Initialize(const u32 *caps, s32 num_caps, KProcessPageTable *page_table);
             Result Initialize(svc::KUserPointer<const u32 *> user_caps, s32 num_caps, KProcessPageTable *page_table);

libraries/libmesosphere/include/mesosphere/kern_k_class_token.hpp

@@ -21,6 +21,35 @@ namespace ams::kern {
 
     class KAutoObject;
 
+    #define FOR_EACH_K_CLASS_TOKEN_OBJECT_TYPE(HANDLER) \
+                HANDLER(KAutoObject)                    \
+                                                        \
+                HANDLER(KSynchronizationObject)         \
+                HANDLER(KReadableEvent)                 \
+                                                        \
+                HANDLER(KInterruptEvent)                \
+                HANDLER(KDebug)                         \
+                HANDLER(KThread)                        \
+                HANDLER(KServerPort)                    \
+                HANDLER(KServerSession)                 \
+                HANDLER(KClientPort)                    \
+                HANDLER(KClientSession)                 \
+                HANDLER(KProcess)                       \
+                HANDLER(KResourceLimit)                 \
+                HANDLER(KLightSession)                  \
+                HANDLER(KPort)                          \
+                HANDLER(KSession)                       \
+                HANDLER(KSharedMemory)                  \
+                HANDLER(KEvent)                         \
+                HANDLER(KLightClientSession)            \
+                HANDLER(KLightServerSession)            \
+                HANDLER(KTransferMemory)                \
+                HANDLER(KDeviceAddressSpace)            \
+                HANDLER(KSessionRequest)                \
+                HANDLER(KCodeMemory)                    \
+                HANDLER(KIoPool)                        \
+                HANDLER(KIoRegion)
+
     class KClassTokenGenerator {
         public:
             using TokenBaseType = u16;
@@ -113,8 +142,11 @@ namespace ams::kern {
                 KIoPool,
                 KIoRegion,
 
+                FinalClassesLast,
+
                 FinalClassesEnd = FinalClassesStart + NumFinalClasses,
             };
+            static_assert(ObjectType::FinalClassesLast <= ObjectType::FinalClassesEnd);
 
             template<typename T>
             static constexpr inline TokenBaseType ClassToken = GetClassToken<T>();
@@ -125,4 +157,37 @@ namespace ams::kern {
     template<typename T>
     static constexpr inline ClassTokenType ClassToken = KClassTokenGenerator::ClassToken<T>;
 
+    namespace impl {
+
+        consteval bool IsKClassTokenGeneratorForEachMacroValid() {
+            auto IsObjectTypeIncludedByMacro = [](KClassTokenGenerator::ObjectType object_type) -> bool {
+                #define CLASS_TOKEN_HANDLER(CLASSNAME) if (object_type == KClassTokenGenerator::ObjectType::CLASSNAME) { return true; }
+                FOR_EACH_K_CLASS_TOKEN_OBJECT_TYPE(CLASS_TOKEN_HANDLER)
+                #undef CLASS_TOKEN_HANDLER
+                return false;
+            };
+
+            if (!IsObjectTypeIncludedByMacro(KClassTokenGenerator::ObjectType::KAutoObject)) {
+                return false;
+            }
+
+            for (auto base = util::ToUnderlying(KClassTokenGenerator::ObjectType::BaseClassesStart); base < util::ToUnderlying(KClassTokenGenerator::ObjectType::BaseClassesEnd); ++base) {
+                if (!IsObjectTypeIncludedByMacro(static_cast<KClassTokenGenerator::ObjectType>(base))) {
+                    return false;
+                }
+            }
+
+            for (auto fin = util::ToUnderlying(KClassTokenGenerator::ObjectType::FinalClassesStart); fin < util::ToUnderlying(KClassTokenGenerator::ObjectType::FinalClassesLast); ++fin) {
+                if (!IsObjectTypeIncludedByMacro(static_cast<KClassTokenGenerator::ObjectType>(fin))) {
+                    return false;
+                }
+            }
+
+            return true;
+        }
+
+        static_assert(IsKClassTokenGeneratorForEachMacroValid());
+
+    }
+
 }

libraries/libmesosphere/include/mesosphere/kern_k_client_port.hpp

@@ -28,12 +28,14 @@ namespace ams::kern {
     class KClientPort final : public KSynchronizationObject {
         MESOSPHERE_AUTOOBJECT_TRAITS(KClientPort, KSynchronizationObject);
         private:
-            std::atomic<s32> m_num_sessions;
-            std::atomic<s32> m_peak_sessions;
+            util::Atomic<s32> m_num_sessions;
+            util::Atomic<s32> m_peak_sessions;
             s32 m_max_sessions;
             KPort *m_parent;
         public:
-            constexpr KClientPort() : m_num_sessions(), m_peak_sessions(), m_max_sessions(), m_parent() { /* ... */ }
+            constexpr explicit KClientPort(util::ConstantInitializeTag) : KSynchronizationObject(util::ConstantInitialize), m_num_sessions(0), m_peak_sessions(0), m_max_sessions(), m_parent() { /* ... */ }
+
+            explicit KClientPort() { /* ... */ }
 
             void Initialize(KPort *parent, s32 max_sessions);
             void OnSessionFinalized();
@@ -41,8 +43,8 @@ namespace ams::kern {
 
             constexpr const KPort *GetParent() const { return m_parent; }
 
-            ALWAYS_INLINE s32 GetNumSessions()  const { return m_num_sessions; }
-            ALWAYS_INLINE s32 GetPeakSessions() const { return m_peak_sessions; }
+            ALWAYS_INLINE s32 GetNumSessions()  const { return m_num_sessions.Load(); }
+            ALWAYS_INLINE s32 GetPeakSessions() const { return m_peak_sessions.Load(); }
             ALWAYS_INLINE s32 GetMaxSessions()  const { return m_max_sessions; }
 
             bool IsLight() const;

libraries/libmesosphere/include/mesosphere/kern_k_client_session.hpp

@@ -27,7 +27,8 @@ namespace ams::kern {
         private:
             KSession *m_parent;
         public:
-            constexpr KClientSession() : m_parent() { /* ... */ }
+            constexpr explicit KClientSession(util::ConstantInitializeTag) : KAutoObject(util::ConstantInitialize), m_parent() { /* ... */ }
+            explicit KClientSession() { /* ... */ }
 
             void Initialize(KSession *parent) {
                 /* Set member variables. */
@@ -35,7 +36,6 @@ namespace ams::kern {
             }
 
             virtual void Destroy() override;
-            static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
 
             constexpr KSession *GetParent() const { return m_parent; }
 

libraries/libmesosphere/include/mesosphere/kern_k_code_memory.hpp

@@ -36,14 +36,14 @@ namespace ams::kern {
             }
 
             Result Initialize(KProcessAddress address, size_t size);
-            virtual void Finalize() override;
+            void Finalize();
 
             Result Map(KProcessAddress address, size_t size);
             Result Unmap(KProcessAddress address, size_t size);
             Result MapToOwner(KProcessAddress address, size_t size, ams::svc::MemoryPermission perm);
             Result UnmapFromOwner(KProcessAddress address, size_t size);
 
-            virtual bool IsInitialized() const override { return m_is_initialized; }
+            bool IsInitialized() const { return m_is_initialized; }
             static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
 
             KProcess *GetOwner() const { return m_owner; }

libraries/libmesosphere/include/mesosphere/kern_k_condition_variable.hpp

@@ -26,7 +26,7 @@ namespace ams::kern {
         private:
             ThreadTree m_tree;
         public:
-            constexpr KConditionVariable() : m_tree() { /* ... */ }
+            constexpr KConditionVariable() = default;
 
             /* Arbitration. */
             static Result SignalToAddress(KProcessAddress addr);

libraries/libmesosphere/include/mesosphere/kern_k_debug_base.hpp

@@ -25,35 +25,20 @@ namespace ams::kern {
     class KDebugBase : public KSynchronizationObject {
         protected:
             using DebugEventList = util::IntrusiveListBaseTraits<KEventInfo>::ListType;
-        private:
-            class ProcessHolder {
-                private:
-                    friend class KDebugBase;
-                private:
-                    KProcess *m_process;
-                    std::atomic<u32> m_ref_count;
-                private:
-                    explicit ProcessHolder() : m_process(nullptr) { /* ... */ }
-
-                    void Attach(KProcess *process);
-                    void Detach();
-
-                    bool Open();
-                    void Close();
-            };
         private:
             DebugEventList m_event_info_list;
             u32 m_continue_flags;
-            ProcessHolder m_process_holder;
+            KSharedAutoObject<KProcess> m_process_holder;
             KLightLock m_lock;
             KProcess::State m_old_process_state;
             bool m_is_attached;
         public:
-            explicit KDebugBase() : m_event_info_list(), m_process_holder(), m_lock() { /* ... */ }
+            explicit KDebugBase() { /* ... */ }
         protected:
             bool Is64Bit() const;
         public:
             void Initialize();
+            void Finalize();
 
             Result Attach(KProcess *process);
             Result BreakProcess();
@@ -68,9 +53,6 @@ namespace ams::kern {
             Result GetThreadContext(ams::svc::ThreadContext *out, u64 thread_id, u32 context_flags);
             Result SetThreadContext(const ams::svc::ThreadContext &ctx, u64 thread_id, u32 context_flags);
 
-            virtual Result GetThreadContextImpl(ams::svc::ThreadContext *out, KThread *thread, u32 context_flags) = 0;
-            virtual Result SetThreadContextImpl(const ams::svc::ThreadContext &ctx, KThread *thread, u32 context_flags) = 0;
-
             Result GetRunningThreadInfo(ams::svc::LastThreadContext *out_context, u64 *out_thread_id);
 
             Result GetDebugEventInfo(ams::svc::lp64::DebugEventInfo *out);
@@ -89,7 +71,7 @@ namespace ams::kern {
             }
 
             ALWAYS_INLINE KProcess *GetProcessUnsafe() const {
-                return m_process_holder.m_process;
+                return m_process_holder.Get();
             }
         private:
             void PushDebugEvent(ams::svc::DebugEvent event, uintptr_t param0 = 0, uintptr_t param1 = 0, uintptr_t param2 = 0, uintptr_t param3 = 0, uintptr_t param4 = 0);
@@ -98,8 +80,10 @@ namespace ams::kern {
             template<typename T> requires (std::same_as<T, ams::svc::lp64::DebugEventInfo> || std::same_as<T, ams::svc::ilp32::DebugEventInfo>)
             Result GetDebugEventInfoImpl(T *out);
         public:
-            virtual void OnFinalizeSynchronizationObject() override;
             virtual bool IsSignaled() const override;
+        private:
+            /* NOTE: This is public/virtual override in Nintendo's kernel. */
+            void OnFinalizeSynchronizationObject();
         private:
             static Result ProcessDebugEvent(ams::svc::DebugEvent event, uintptr_t param0, uintptr_t param1, uintptr_t param2, uintptr_t param3, uintptr_t param4);
         public:

libraries/libmesosphere/include/mesosphere/kern_k_device_address_space.hpp

@@ -30,12 +30,12 @@ namespace ams::kern {
             u64 m_space_size;
             bool m_is_initialized;
         public:
-            constexpr KDeviceAddressSpace() : m_lock(), m_table(), m_space_address(), m_space_size(), m_is_initialized() { /* ... */ }
+            explicit KDeviceAddressSpace() : m_is_initialized(false) { /* ... */ }
 
             Result Initialize(u64 address, u64 size);
-            virtual void Finalize() override;
+            void Finalize();
 
-            virtual bool IsInitialized() const override { return m_is_initialized; }
+            bool IsInitialized() const { return m_is_initialized; }
             static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
 
             Result Attach(ams::svc::DeviceName device_name);

libraries/libmesosphere/include/mesosphere/kern_k_dynamic_page_manager.hpp

@@ -40,7 +40,7 @@ namespace ams::kern {
             KVirtualAddress m_address;
             size_t m_size;
         public:
-            KDynamicPageManager() : m_lock(), m_page_bitmap(), m_used(), m_peak(), m_count(), m_address(), m_size() { /* ... */ }
+            KDynamicPageManager() : m_lock(), m_page_bitmap(), m_used(), m_peak(), m_count(), m_address(Null<KVirtualAddress>), m_size() { /* ... */ }
 
             Result Initialize(KVirtualAddress memory, size_t sz) {
                 /* We need to have positive size. */

libraries/libmesosphere/include/mesosphere/kern_k_dynamic_slab_heap.hpp

@@ -29,19 +29,19 @@ namespace ams::kern {
         private:
             using PageBuffer = KDynamicPageManager::PageBuffer;
         private:
-            std::atomic<size_t> m_used{};
-            std::atomic<size_t> m_peak{};
-            std::atomic<size_t> m_count{};
-            KVirtualAddress m_address{};
+            util::Atomic<size_t> m_used{0};
+            util::Atomic<size_t> m_peak{0};
+            util::Atomic<size_t> m_count{0};
+            KVirtualAddress m_address{Null<KVirtualAddress>};
             size_t m_size{};
         public:
             constexpr KDynamicSlabHeap() = default;
 
             constexpr ALWAYS_INLINE KVirtualAddress GetAddress() const { return m_address; }
             constexpr ALWAYS_INLINE size_t GetSize() const { return m_size; }
-            constexpr ALWAYS_INLINE size_t GetUsed() const { return m_used.load(); }
-            constexpr ALWAYS_INLINE size_t GetPeak() const { return m_peak.load(); }
-            constexpr ALWAYS_INLINE size_t GetCount() const { return m_count.load(); }
+            constexpr ALWAYS_INLINE size_t GetUsed() const { return m_used.Load(); }
+            constexpr ALWAYS_INLINE size_t GetPeak() const { return m_peak.Load(); }
+            constexpr ALWAYS_INLINE size_t GetCount() const { return m_count.Load(); }
 
             constexpr ALWAYS_INLINE bool IsInRange(KVirtualAddress addr) const {
                 return this->GetAddress() <= addr && addr <= this->GetAddress() + this->GetSize() - 1;
@@ -58,7 +58,7 @@ namespace ams::kern {
                 KSlabHeapImpl::Initialize();
 
                 /* Allocate until we have the correct number of objects. */
-                while (m_count.load() < num_objects) {
+                while (m_count.Load() < num_objects) {
                     auto *allocated = reinterpret_cast<T *>(page_allocator->Allocate());
                     MESOSPHERE_ABORT_UNLESS(allocated != nullptr);
 
@@ -66,7 +66,7 @@ namespace ams::kern {
                         KSlabHeapImpl::Free(allocated + i);
                     }
 
-                    m_count.fetch_add(sizeof(PageBuffer) / sizeof(T));
+                    m_count += sizeof(PageBuffer) / sizeof(T);
                 }
             }
 
@@ -89,7 +89,7 @@ namespace ams::kern {
                             for (size_t i = 1; i < sizeof(PageBuffer) / sizeof(T); i++) {
                                 KSlabHeapImpl::Free(allocated + i);
                             }
-                            m_count.fetch_add(sizeof(PageBuffer) / sizeof(T));
+                            m_count += sizeof(PageBuffer) / sizeof(T);
                         }
                     }
                 }
@@ -99,10 +99,10 @@ namespace ams::kern {
                     std::construct_at(allocated);
 
                     /* Update our tracking. */
-                    size_t used = m_used.fetch_add(1) + 1;
-                    size_t peak = m_peak.load();
+                    const size_t used = ++m_used;
+                    size_t peak = m_peak.Load();
                     while (peak < used) {
-                        if (m_peak.compare_exchange_weak(peak, used, std::memory_order_relaxed)) {
+                        if (m_peak.CompareExchangeWeak<std::memory_order_relaxed>(peak, used)) {
                             break;
                         }
                     }
@@ -113,7 +113,7 @@ namespace ams::kern {
 
             ALWAYS_INLINE void Free(T *t) {
                 KSlabHeapImpl::Free(t);
-                m_used.fetch_sub(1);
+                --m_used;
             }
     };
 

libraries/libmesosphere/include/mesosphere/kern_k_event.hpp

@@ -29,21 +29,24 @@ namespace ams::kern {
             bool m_initialized;
             bool m_readable_event_destroyed;
         public:
-            constexpr KEvent()
-                : m_readable_event(), m_owner(), m_initialized(), m_readable_event_destroyed()
+            constexpr explicit KEvent(util::ConstantInitializeTag)
+                : KAutoObjectWithSlabHeapAndContainer<KEvent, KAutoObjectWithList, true>(util::ConstantInitialize),
+                  m_readable_event(util::ConstantInitialize), m_owner(), m_initialized(), m_readable_event_destroyed()
             {
                 /* ... */
             }
 
-            void Initialize();
-            virtual void Finalize() override;
+            explicit KEvent() : m_readable_event(), m_owner(), m_initialized(), m_readable_event_destroyed() { /* ... */ }
 
-            virtual bool IsInitialized() const override { return m_initialized; }
-            virtual uintptr_t GetPostDestroyArgument() const override { return reinterpret_cast<uintptr_t>(m_owner); }
+            void Initialize();
+            void Finalize();
+
+            bool IsInitialized() const { return m_initialized; }
+            uintptr_t GetPostDestroyArgument() const { return reinterpret_cast<uintptr_t>(m_owner); }
 
             static void PostDestroy(uintptr_t arg);
 
-            virtual KProcess *GetOwner() const override { return m_owner; }
+            KProcess *GetOwner() const { return m_owner; }
 
             KReadableEvent &GetReadableEvent() { return m_readable_event; }
 

libraries/libmesosphere/include/mesosphere/kern_k_handle_table.hpp

@@ -63,16 +63,16 @@ namespace ams::kern {
         private:
             EntryInfo m_entry_infos[MaxTableSize];
             KAutoObject *m_objects[MaxTableSize];
+            mutable KSpinLock m_lock;
             s32 m_free_head_index;
             u16 m_table_size;
             u16 m_max_count;
             u16 m_next_linear_id;
             u16 m_count;
-            mutable KSpinLock m_lock;
         public:
-            constexpr KHandleTable() :
-                m_entry_infos(), m_objects(), m_free_head_index(-1), m_table_size(0), m_max_count(0), m_next_linear_id(MinLinearId), m_count(0), m_lock()
-            { MESOSPHERE_ASSERT_THIS(); }
+            constexpr explicit KHandleTable(util::ConstantInitializeTag) : m_entry_infos(), m_objects(), m_lock(), m_free_head_index(-1), m_table_size(), m_max_count(), m_next_linear_id(MinLinearId), m_count() { /* ... */ }
+
+            explicit KHandleTable() : m_lock(), m_free_head_index(-1), m_count() { MESOSPHERE_ASSERT_THIS(); }
 
             constexpr NOINLINE Result Initialize(s32 size) {
                 MESOSPHERE_ASSERT_THIS();

libraries/libmesosphere/include/mesosphere/kern_k_interrupt_event.hpp

@@ -31,14 +31,25 @@ namespace ams::kern {
             s32 m_core_id;
             bool m_is_initialized;
         public:
-            constexpr KInterruptEvent() : m_interrupt_id(-1), m_core_id(-1), m_is_initialized(false) { /* ... */ }
+            constexpr explicit KInterruptEvent(util::ConstantInitializeTag) : KAutoObjectWithSlabHeapAndContainer<KInterruptEvent, KReadableEvent>(util::ConstantInitialize), m_interrupt_id(-1), m_core_id(-1), m_is_initialized(false) { /* ... */ }
+
+            explicit KInterruptEvent() : m_interrupt_id(-1), m_is_initialized(false) { /* ... */ }
 
             Result Initialize(int32_t interrupt_name, ams::svc::InterruptType type);
-            virtual void Finalize() override;
+            void Finalize();
 
-            virtual Result Reset() override;
+            Result Reset();
 
-            virtual bool IsInitialized() const override { return m_is_initialized; }
+            Result Clear() {
+                MESOSPHERE_ASSERT_THIS();
+
+                /* Try to perform a reset, succeeding unconditionally. */
+                this->Reset();
+
+                return ResultSuccess();
+            }
+
+            bool IsInitialized() const { return m_is_initialized; }
 
             static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
 

libraries/libmesosphere/include/mesosphere/kern_k_io_pool.hpp

@@ -33,14 +33,14 @@ namespace ams::kern {
         public:
             static bool IsValidIoPoolType(ams::svc::IoPoolType pool_type);
         public:
-            explicit KIoPool() : m_lock(), m_io_region_list(), m_is_initialized(false) {
+            explicit KIoPool() : m_is_initialized(false) {
                 /* ... */
             }
 
             Result Initialize(ams::svc::IoPoolType pool_type);
-            virtual void Finalize() override;
+            void Finalize();
 
-            virtual bool IsInitialized() const override { return m_is_initialized; }
+            bool IsInitialized() const { return m_is_initialized; }
             static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
 
             Result AddIoRegion(KIoRegion *region);

libraries/libmesosphere/include/mesosphere/kern_k_io_region.hpp

@@ -40,16 +40,12 @@ namespace ams::kern {
             util::IntrusiveListNode m_process_list_node;
             util::IntrusiveListNode m_pool_list_node;
         public:
-            explicit KIoRegion()
-                : m_lock(), m_pool(nullptr), m_is_initialized(false), m_process_list_node(), m_pool_list_node()
-            {
-                /* ... */
-            }
+            explicit KIoRegion() : m_pool(nullptr), m_is_initialized(false) { /* ... */ }
 
             Result Initialize(KIoPool *pool, KPhysicalAddress phys_addr, size_t size, ams::svc::MemoryMapping mapping, ams::svc::MemoryPermission perm);
-            virtual void Finalize() override;
+            void Finalize();
 
-            virtual bool IsInitialized() const override { return m_is_initialized; }
+            bool IsInitialized() const { return m_is_initialized; }
             static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
 
             Result Map(KProcessAddress address, size_t size, ams::svc::MemoryPermission map_perm);

libraries/libmesosphere/include/mesosphere/kern_k_light_client_session.hpp

@@ -26,7 +26,7 @@ namespace ams::kern {
         private:
             KLightSession *m_parent;
         public:
-            constexpr KLightClientSession() : m_parent() { /* ... */ }
+            explicit KLightClientSession() { /* ... */ }
 
             void Initialize(KLightSession *parent) {
                 /* Set member variables. */
@@ -34,7 +34,6 @@ namespace ams::kern {
             }
 
             virtual void Destroy() override;
-            static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
 
             constexpr const KLightSession *GetParent() const { return m_parent; }
 

libraries/libmesosphere/include/mesosphere/kern_k_light_condition_variable.hpp

@@ -26,7 +26,9 @@ namespace ams::kern {
         private:
             KThread::WaiterList m_wait_list;
         public:
-            constexpr ALWAYS_INLINE KLightConditionVariable() : m_wait_list() { /* ... */ }
+            constexpr explicit ALWAYS_INLINE KLightConditionVariable(util::ConstantInitializeTag) : m_wait_list() { /* ... */ }
+
+            explicit ALWAYS_INLINE KLightConditionVariable() { /* ... */ }
         public:
             void Wait(KLightLock *lock, s64 timeout = -1ll, bool allow_terminating_thread = true);
             void Broadcast();

libraries/libmesosphere/include/mesosphere/kern_k_light_lock.hpp

@@ -23,30 +23,25 @@ namespace ams::kern {
 
     class KLightLock {
         private:
-            std::atomic<uintptr_t> m_tag;
+            util::Atomic<uintptr_t> m_tag;
         public:
-            constexpr KLightLock() : m_tag(0) { /* ... */ }
+            constexpr ALWAYS_INLINE KLightLock() : m_tag(0) { /* ... */ }
 
             void Lock() {
                 MESOSPHERE_ASSERT_THIS();
 
                 const uintptr_t cur_thread = reinterpret_cast<uintptr_t>(GetCurrentThreadPointer());
-                const uintptr_t cur_thread_tag = (cur_thread | 1);
 
                 while (true) {
-                    uintptr_t old_tag = m_tag.load(std::memory_order_relaxed);
+                    uintptr_t old_tag = m_tag.Load<std::memory_order_relaxed>();
 
-                    while (!m_tag.compare_exchange_weak(old_tag, (old_tag == 0) ? cur_thread : old_tag | 1, std::memory_order_acquire)) {
-                        if ((old_tag | 1) == cur_thread_tag) {
-                            return;
-                        }
+                    while (!m_tag.CompareExchangeWeak<std::memory_order_acquire>(old_tag, (old_tag == 0) ? cur_thread : (old_tag | 1))) {
+                        /* ... */
                     }
 
-                    if ((old_tag == 0) || ((old_tag | 1) == cur_thread_tag)) {
+                    if (old_tag == 0 || this->LockSlowPath(old_tag | 1, cur_thread)) {
                         break;
                     }
-
-                    this->LockSlowPath(old_tag | 1, cur_thread);
                 }
             }
 
@@ -54,19 +49,18 @@ namespace ams::kern {
                 MESOSPHERE_ASSERT_THIS();
 
                 const uintptr_t cur_thread = reinterpret_cast<uintptr_t>(GetCurrentThreadPointer());
+
                 uintptr_t expected = cur_thread;
-                do {
-                    if (expected != cur_thread) {
-                        return this->UnlockSlowPath(cur_thread);
-                    }
-                } while (!m_tag.compare_exchange_weak(expected, 0, std::memory_order_release));
+                if (!m_tag.CompareExchangeStrong<std::memory_order_release>(expected, 0)) {
+                    this->UnlockSlowPath(cur_thread);
+                }
             }
 
-            void LockSlowPath(uintptr_t owner, uintptr_t cur_thread);
+            bool LockSlowPath(uintptr_t owner, uintptr_t cur_thread);
             void UnlockSlowPath(uintptr_t cur_thread);
 
-            ALWAYS_INLINE bool IsLocked() const { return m_tag.load() != 0; }
-            ALWAYS_INLINE bool IsLockedByCurrentThread() const { return (m_tag.load() | 0x1ul) == (reinterpret_cast<uintptr_t>(GetCurrentThreadPointer()) | 0x1ul); }
+            ALWAYS_INLINE bool IsLocked() const { return m_tag.Load() != 0; }
+            ALWAYS_INLINE bool IsLockedByCurrentThread() const { return (m_tag.Load() | 0x1ul) == (reinterpret_cast<uintptr_t>(GetCurrentThreadPointer()) | 0x1ul); }
     };
 
     using KScopedLightLock = KScopedLock<KLightLock>;

libraries/libmesosphere/include/mesosphere/kern_k_light_server_session.hpp

@@ -32,7 +32,7 @@ namespace ams::kern {
             u64 m_server_thread_id;
             KThread *m_server_thread;
         public:
-            constexpr KLightServerSession() : m_parent(), m_request_list(), m_current_request(), m_server_thread_id(), m_server_thread() { /* ... */ }
+            explicit KLightServerSession() : m_current_request(nullptr), m_server_thread_id(std::numeric_limits<u64>::max()), m_server_thread() { /* ... */ }
 
             void Initialize(KLightSession *parent) {
                 /* Set member variables. */
@@ -40,7 +40,6 @@ namespace ams::kern {
             }
 
             virtual void Destroy() override;
-            static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
 
             constexpr const KLightSession *GetParent() const { return m_parent; }
 

libraries/libmesosphere/include/mesosphere/kern_k_light_session.hpp

@@ -46,17 +46,13 @@ namespace ams::kern {
             KProcess *m_process;
             bool m_initialized;
         public:
-            constexpr KLightSession()
-                : m_server(), m_client(), m_state(State::Invalid), m_port(), m_name(), m_process(), m_initialized()
-            {
-                /* ... */
-            }
+            explicit KLightSession() : m_state(State::Invalid), m_process(), m_initialized() { /* ... */ }
 
             void Initialize(KClientPort *client_port, uintptr_t name);
-            virtual void Finalize() override;
+            void Finalize();
 
-            virtual bool IsInitialized() const override { return m_initialized; }
-            virtual uintptr_t GetPostDestroyArgument() const override { return reinterpret_cast<uintptr_t>(m_process); }
+            bool IsInitialized() const { return m_initialized; }
+            uintptr_t GetPostDestroyArgument() const { return reinterpret_cast<uintptr_t>(m_process); }
 
             static void PostDestroy(uintptr_t arg);
 

libraries/libmesosphere/include/mesosphere/kern_k_memory_block.hpp

@@ -349,28 +349,30 @@ namespace ams::kern {
                 };
             }
         public:
-            constexpr KMemoryBlock()
-                : m_device_disable_merge_left_count(), m_device_disable_merge_right_count(), m_address(), m_num_pages(), m_memory_state(KMemoryState_None), m_ipc_lock_count(), m_device_use_count(), m_ipc_disable_merge_count(), m_permission(), m_original_permission(), m_attribute(), m_disable_merge_attribute()
-            {
-                /* ... */
-            }
+            explicit KMemoryBlock() { /* ... */ }
 
-            constexpr KMemoryBlock(KProcessAddress addr, size_t np, KMemoryState ms, KMemoryPermission p, KMemoryAttribute attr)
-                : m_device_disable_merge_left_count(), m_device_disable_merge_right_count(), m_address(addr), m_num_pages(np), m_memory_state(ms), m_ipc_lock_count(0), m_device_use_count(0), m_ipc_disable_merge_count(), m_permission(p), m_original_permission(KMemoryPermission_None), m_attribute(attr), m_disable_merge_attribute()
+            constexpr KMemoryBlock(util::ConstantInitializeTag, KProcessAddress addr, size_t np, KMemoryState ms, KMemoryPermission p, KMemoryAttribute attr)
+                : util::IntrusiveRedBlackTreeBaseNode<KMemoryBlock>(util::ConstantInitialize), m_device_disable_merge_left_count(),
+                  m_device_disable_merge_right_count(), m_address(addr), m_num_pages(np), m_memory_state(ms), m_ipc_lock_count(0),
+                  m_device_use_count(0), m_ipc_disable_merge_count(), m_permission(p), m_original_permission(KMemoryPermission_None),
+                  m_attribute(attr), m_disable_merge_attribute()
             {
                 /* ... */
             }
 
             constexpr void Initialize(KProcessAddress addr, size_t np, KMemoryState ms, KMemoryPermission p, KMemoryAttribute attr) {
                 MESOSPHERE_ASSERT_THIS();
-                m_address             = addr;
-                m_num_pages           = np;
-                m_memory_state        = ms;
-                m_ipc_lock_count      = 0;
-                m_device_use_count    = 0;
-                m_permission          = p;
-                m_original_permission = KMemoryPermission_None;
-                m_attribute           = attr;
+                m_device_disable_merge_left_count  = 0;
+                m_device_disable_merge_right_count = 0;
+                m_address                          = addr;
+                m_num_pages                        = np;
+                m_memory_state                     = ms;
+                m_ipc_lock_count                   = 0;
+                m_device_use_count                 = 0;
+                m_permission                       = p;
+                m_original_permission              = KMemoryPermission_None;
+                m_attribute                        = attr;
+                m_disable_merge_attribute          = KMemoryBlockDisableMergeAttribute_None;
             }
 
             constexpr bool HasProperties(KMemoryState s, KMemoryPermission p, KMemoryAttribute a) const {

libraries/libmesosphere/include/mesosphere/kern_k_memory_block_manager.hpp

@@ -88,7 +88,9 @@ namespace ams::kern {
         private:
             void CoalesceForUpdate(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages);
         public:
-            constexpr KMemoryBlockManager() : m_memory_block_tree(), m_start_address(), m_end_address() { /* ... */ }
+            constexpr explicit KMemoryBlockManager(util::ConstantInitializeTag) : m_memory_block_tree(), m_start_address(Null<KProcessAddress>), m_end_address(Null<KProcessAddress>) { /* ... */ }
+
+            explicit KMemoryBlockManager() { /* ... */ }
 
             iterator end() { return m_memory_block_tree.end(); }
             const_iterator end() const { return m_memory_block_tree.end(); }
@@ -105,7 +107,7 @@ namespace ams::kern {
             void UpdateIfMatch(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages, KMemoryState test_state, KMemoryPermission test_perm, KMemoryAttribute test_attr, KMemoryState state, KMemoryPermission perm, KMemoryAttribute attr);
 
             iterator FindIterator(KProcessAddress address) const {
-                return m_memory_block_tree.find(KMemoryBlock(address, 1, KMemoryState_Free, KMemoryPermission_None, KMemoryAttribute_None));
+                return m_memory_block_tree.find(KMemoryBlock(util::ConstantInitialize, address, 1, KMemoryState_Free, KMemoryPermission_None, KMemoryAttribute_None));
             }
 
             const KMemoryBlock *FindBlock(KProcessAddress address) const {

libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp

@@ -20,6 +20,8 @@
 
 #if defined(ATMOSPHERE_BOARD_NINTENDO_NX)
     #include <mesosphere/board/nintendo/nx/kern_k_memory_layout.hpp>
+#elif defined(ATMOSPHERE_BOARD_QEMU_VIRT)
+    #include <mesosphere/board/qemu/virt/kern_k_memory_layout.hpp>
 #else
     #error "Unknown board for KMemoryLayout"
 #endif
@@ -54,12 +56,12 @@ namespace ams::kern {
 
     class KMemoryLayout {
         private:
-            static /* constinit */ inline uintptr_t s_linear_phys_to_virt_diff;
-            static /* constinit */ inline uintptr_t s_linear_virt_to_phys_diff;
-            static /* constinit */ inline KMemoryRegionTree s_virtual_tree;
-            static /* constinit */ inline KMemoryRegionTree s_physical_tree;
-            static /* constinit */ inline KMemoryRegionTree s_virtual_linear_tree;
-            static /* constinit */ inline KMemoryRegionTree s_physical_linear_tree;
+            static constinit inline uintptr_t s_linear_phys_to_virt_diff;
+            static constinit inline uintptr_t s_linear_virt_to_phys_diff;
+            static constinit inline KMemoryRegionTree s_virtual_tree;
+            static constinit inline KMemoryRegionTree s_physical_tree;
+            static constinit inline KMemoryRegionTree s_virtual_linear_tree;
+            static constinit inline KMemoryRegionTree s_physical_linear_tree;
         private:
             template<typename AddressType> requires IsKTypedAddress<AddressType>
             static ALWAYS_INLINE bool IsTypedAddress(const KMemoryRegion *&region, AddressType address, KMemoryRegionTree &tree, KMemoryRegionType type) {
@@ -210,13 +212,17 @@ namespace ams::kern {
             static NOINLINE auto GetKernelPageTableHeapRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramKernelPtHeap); }
             static NOINLINE auto GetKernelInitPageTableRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramKernelInitPt); }
 
-            static NOINLINE auto GetKernelPoolManagementRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramPoolManagement); }
             static NOINLINE auto GetKernelPoolPartitionRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramPoolPartition); }
+            static NOINLINE auto GetKernelPoolManagementRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramPoolManagement); }
             static NOINLINE auto GetKernelSystemPoolRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramSystemPool); }
             static NOINLINE auto GetKernelSystemNonSecurePoolRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramSystemNonSecurePool); }
             static NOINLINE auto GetKernelAppletPoolRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramAppletPool); }
             static NOINLINE auto GetKernelApplicationPoolRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramApplicationPool); }
 
+            static NOINLINE bool HasKernelSystemNonSecurePoolRegion() { return GetPhysicalMemoryRegionTree().FindFirstDerived(KMemoryRegionType_DramSystemNonSecurePool) != nullptr; }
+            static NOINLINE bool HasKernelAppletPoolRegion() { return GetPhysicalMemoryRegionTree().FindFirstDerived(KMemoryRegionType_DramAppletPool) != nullptr; }
+            static NOINLINE bool HasKernelApplicationPoolRegion() { return GetPhysicalMemoryRegionTree().FindFirstDerived(KMemoryRegionType_DramApplicationPool) != nullptr; }
+
             static NOINLINE auto GetKernelTraceBufferRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_KernelTraceBuffer); }
     };
 

libraries/libmesosphere/include/mesosphere/kern_k_memory_manager.hpp

@@ -68,7 +68,7 @@ namespace ams::kern {
                     Impl *m_next;
                     Impl *m_prev;
                 public:
-                    Impl() : m_heap(), m_page_reference_counts(), m_management_region(), m_pool(), m_next(), m_prev() { /* ... */ }
+                    Impl() : m_heap(), m_page_reference_counts(), m_management_region(Null<KVirtualAddress>), m_pool(), m_next(), m_prev() { /* ... */ }
 
                     size_t Initialize(KPhysicalAddress address, size_t size, KVirtualAddress management, KVirtualAddress management_end, Pool p);
 

libraries/libmesosphere/include/mesosphere/kern_k_memory_region.hpp

@@ -43,13 +43,14 @@ namespace ams::kern {
                 }
             }
         public:
-            constexpr ALWAYS_INLINE KMemoryRegion() : m_address(0), m_pair_address(0), m_last_address(0), m_attributes(0), m_type_id(0) { /* ... */ }
-            constexpr ALWAYS_INLINE KMemoryRegion(uintptr_t a, size_t la, uintptr_t p, u32 r, u32 t) :
+            constexpr ALWAYS_INLINE KMemoryRegion() : util::IntrusiveRedBlackTreeBaseNode<KMemoryRegion>(util::ConstantInitialize), m_address(0), m_pair_address(0), m_last_address(0), m_attributes(0), m_type_id(0) { /* ... */ }
+
+            ALWAYS_INLINE KMemoryRegion(uintptr_t a, size_t la, uintptr_t p, u32 r, u32 t) :
                 m_address(a), m_pair_address(p), m_last_address(la), m_attributes(r), m_type_id(t)
             {
                 /* ... */
             }
-            constexpr ALWAYS_INLINE KMemoryRegion(uintptr_t a, size_t la, u32 r, u32 t) : KMemoryRegion(a, la, std::numeric_limits<uintptr_t>::max(), r, t) { /* ... */ }
+            ALWAYS_INLINE KMemoryRegion(uintptr_t a, size_t la, u32 r, u32 t) : KMemoryRegion(a, la, std::numeric_limits<uintptr_t>::max(), r, t) { /* ... */ }
         private:
             constexpr ALWAYS_INLINE void Reset(uintptr_t a, uintptr_t la, uintptr_t p, u32 r, u32 t) {
                 m_address      = a;

libraries/libmesosphere/include/mesosphere/kern_k_object_name.hpp

@@ -29,8 +29,6 @@ namespace ams::kern {
         private:
             char m_name[NameLengthMax];
             KAutoObject *m_object;
-        public:
-            constexpr KObjectName() : m_name(), m_object() { /* ... */ }
         public:
             static Result NewFromName(KAutoObject *obj, const char *name);
             static Result Delete(KAutoObject *obj, const char *name);

libraries/libmesosphere/include/mesosphere/kern_k_page_bitmap.hpp

@@ -43,44 +43,25 @@ namespace ams::kern {
                         return rnd_bit;
                     }
                 public:
-                    RandomBitGenerator() : m_rng(), m_entropy(), m_bits_available() {
+                    RandomBitGenerator() : m_entropy(), m_bits_available() {
                         m_rng.Initialize(static_cast<u32>(KSystemControl::GenerateRandomU64()));
                     }
 
                     size_t SelectRandomBit(u64 bitmap) {
-                        u64 selected     = 0;
+                        u64 selected = 0;
 
-                        u64 cur_num_bits = BITSIZEOF(bitmap) / 2;
-                        u64 cur_mask     = (1ull << cur_num_bits) - 1;
+                        for (size_t cur_num_bits = BITSIZEOF(bitmap) / 2; cur_num_bits != 0; cur_num_bits /= 2) {
+                            const u64 high = (bitmap >> cur_num_bits);
+                            const u64 low  = (bitmap & (~(UINT64_C(0xFFFFFFFFFFFFFFFF) << cur_num_bits)));
 
-                        while (cur_num_bits) {
-                            const u64 low  = (bitmap >> 0) & cur_mask;
-                            const u64 high = (bitmap >> cur_num_bits) & cur_mask;
-
-                            bool choose_low;
-                            if (high == 0) {
-                                /* If only low val is set, choose low. */
-                                choose_low = true;
-                            } else if (low == 0) {
-                                /* If only high val is set, choose high. */
-                                choose_low = false;
-                            } else {
-                                /* If both are set, choose random. */
-                                choose_low = this->GenerateRandomBit();
-                            }
-
-                            /* If we chose low, proceed with low. */
-                            if (choose_low) {
-                                bitmap    = low;
-                                selected += 0;
-                            } else {
+                            /* Choose high if we have high and (don't have low or select high randomly). */
+                            if (high && (low == 0 || this->GenerateRandomBit())) {
                                 bitmap    = high;
                                 selected += cur_num_bits;
+                            } else {
+                                bitmap    = low;
+                                selected += 0;
                             }
-
-                            /* Proceed. */
-                            cur_num_bits /= 2;
-                            cur_mask >>= cur_num_bits;
                         }
 
                         return selected;

libraries/libmesosphere/include/mesosphere/kern_k_page_group.hpp

@@ -28,11 +28,11 @@ namespace ams::kern {
         private:
             friend class KPageGroup;
         private:
-            KBlockInfo *m_next{};
-            u32 m_page_index{};
-            u32 m_num_pages{};
+            KBlockInfo *m_next;
+            u32 m_page_index;
+            u32 m_num_pages;
         public:
-            constexpr KBlockInfo() = default;
+            KBlockInfo() : m_next(nullptr) { /* ... */ }
 
             constexpr ALWAYS_INLINE void Initialize(KPhysicalAddress addr, size_t np) {
                 MESOSPHERE_ASSERT(util::IsAligned(GetInteger(addr), PageSize));

libraries/libmesosphere/include/mesosphere/kern_k_page_heap.hpp

@@ -59,7 +59,7 @@ namespace ams::kern {
                     size_t m_block_shift;
                     size_t m_next_block_shift;
                 public:
-                    Block() : m_bitmap(), m_heap_address(), m_end_offset(), m_block_shift(), m_next_block_shift() { /* ... */ }
+                    Block() : m_bitmap(), m_heap_address(Null<KPhysicalAddress>), m_end_offset(), m_block_shift(), m_next_block_shift() { /* ... */ }
 
                     constexpr size_t GetShift() const { return m_block_shift; }
                     constexpr size_t GetNextShift() const { return m_next_block_shift; }
@@ -134,7 +134,7 @@ namespace ams::kern {
 
             void FreeBlock(KPhysicalAddress block, s32 index);
         public:
-            KPageHeap() : m_heap_address(), m_heap_size(), m_initial_used_size(), m_num_blocks(), m_blocks() { /* ... */ }
+            KPageHeap() : m_heap_address(Null<KPhysicalAddress>), m_heap_size(), m_initial_used_size(), m_num_blocks(), m_blocks() { /* ... */ }
 
             constexpr KPhysicalAddress GetAddress() const { return m_heap_address; }
             constexpr size_t GetSize() const { return m_heap_size; }

libraries/libmesosphere/include/mesosphere/kern_k_page_table_base.hpp

@@ -144,45 +144,60 @@ namespace ams::kern {
                     PageLinkedList *GetPageList() { return std::addressof(m_ll); }
             };
         private:
-            KProcessAddress m_address_space_start{};
-            KProcessAddress m_address_space_end{};
-            KProcessAddress m_heap_region_start{};
-            KProcessAddress m_heap_region_end{};
-            KProcessAddress m_current_heap_end{};
-            KProcessAddress m_alias_region_start{};
-            KProcessAddress m_alias_region_end{};
-            KProcessAddress m_stack_region_start{};
-            KProcessAddress m_stack_region_end{};
-            KProcessAddress m_kernel_map_region_start{};
-            KProcessAddress m_kernel_map_region_end{};
-            KProcessAddress m_alias_code_region_start{};
-            KProcessAddress m_alias_code_region_end{};
-            KProcessAddress m_code_region_start{};
-            KProcessAddress m_code_region_end{};
-            size_t m_max_heap_size{};
-            size_t m_mapped_physical_memory_size{};
-            size_t m_mapped_unsafe_physical_memory{};
-            size_t m_mapped_ipc_server_memory{};
-            mutable KLightLock m_general_lock{};
-            mutable KLightLock m_map_physical_memory_lock{};
-            KLightLock m_device_map_lock{};
-            KPageTableImpl m_impl{};
-            KMemoryBlockManager m_memory_block_manager{};
-            u32 m_allocate_option{};
-            u32 m_address_space_width{};
-            bool m_is_kernel{};
-            bool m_enable_aslr{};
-            bool m_enable_device_address_space_merge{};
-            KMemoryBlockSlabManager *m_memory_block_slab_manager{};
-            KBlockInfoManager *m_block_info_manager{};
-            KResourceLimit *m_resource_limit{};
-            const KMemoryRegion *m_cached_physical_linear_region{};
-            const KMemoryRegion *m_cached_physical_heap_region{};
-            MemoryFillValue m_heap_fill_value{};
-            MemoryFillValue m_ipc_fill_value{};
-            MemoryFillValue m_stack_fill_value{};
+            KProcessAddress m_address_space_start;
+            KProcessAddress m_address_space_end;
+            KProcessAddress m_heap_region_start;
+            KProcessAddress m_heap_region_end;
+            KProcessAddress m_current_heap_end;
+            KProcessAddress m_alias_region_start;
+            KProcessAddress m_alias_region_end;
+            KProcessAddress m_stack_region_start;
+            KProcessAddress m_stack_region_end;
+            KProcessAddress m_kernel_map_region_start;
+            KProcessAddress m_kernel_map_region_end;
+            KProcessAddress m_alias_code_region_start;
+            KProcessAddress m_alias_code_region_end;
+            KProcessAddress m_code_region_start;
+            KProcessAddress m_code_region_end;
+            size_t m_max_heap_size;
+            size_t m_mapped_physical_memory_size;
+            size_t m_mapped_unsafe_physical_memory;
+            size_t m_mapped_ipc_server_memory;
+            mutable KLightLock m_general_lock;
+            mutable KLightLock m_map_physical_memory_lock;
+            KLightLock m_device_map_lock;
+            KPageTableImpl m_impl;
+            KMemoryBlockManager m_memory_block_manager;
+            u32 m_allocate_option;
+            u32 m_address_space_width;
+            bool m_is_kernel;
+            bool m_enable_aslr;
+            bool m_enable_device_address_space_merge;
+            KMemoryBlockSlabManager *m_memory_block_slab_manager;
+            KBlockInfoManager *m_block_info_manager;
+            KResourceLimit *m_resource_limit;
+            const KMemoryRegion *m_cached_physical_linear_region;
+            const KMemoryRegion *m_cached_physical_heap_region;
+            MemoryFillValue m_heap_fill_value;
+            MemoryFillValue m_ipc_fill_value;
+            MemoryFillValue m_stack_fill_value;
         public:
-            constexpr KPageTableBase() { /* ... */ }
+            constexpr explicit KPageTableBase(util::ConstantInitializeTag)
+                : m_address_space_start(Null<KProcessAddress>), m_address_space_end(Null<KProcessAddress>), m_heap_region_start(Null<KProcessAddress>),
+                  m_heap_region_end(Null<KProcessAddress>), m_current_heap_end(Null<KProcessAddress>), m_alias_region_start(Null<KProcessAddress>),
+                  m_alias_region_end(Null<KProcessAddress>), m_stack_region_start(Null<KProcessAddress>), m_stack_region_end(Null<KProcessAddress>),
+                  m_kernel_map_region_start(Null<KProcessAddress>), m_kernel_map_region_end(Null<KProcessAddress>), m_alias_code_region_start(Null<KProcessAddress>),
+                  m_alias_code_region_end(Null<KProcessAddress>), m_code_region_start(Null<KProcessAddress>), m_code_region_end(Null<KProcessAddress>),
+                  m_max_heap_size(), m_mapped_physical_memory_size(), m_mapped_unsafe_physical_memory(), m_mapped_ipc_server_memory(), m_general_lock(),
+                  m_map_physical_memory_lock(), m_device_map_lock(), m_impl(util::ConstantInitialize), m_memory_block_manager(util::ConstantInitialize),
+                  m_allocate_option(), m_address_space_width(), m_is_kernel(), m_enable_aslr(), m_enable_device_address_space_merge(),
+                  m_memory_block_slab_manager(), m_block_info_manager(), m_resource_limit(), m_cached_physical_linear_region(), m_cached_physical_heap_region(),
+                  m_heap_fill_value(), m_ipc_fill_value(), m_stack_fill_value()
+            {
+                /* ... */
+            }
+
+            explicit KPageTableBase() { /* ... */ }
 
             NOINLINE Result InitializeForKernel(bool is_64_bit, void *table, KVirtualAddress start, KVirtualAddress end);
             NOINLINE Result InitializeForProcess(ams::svc::CreateProcessFlag as_type, bool enable_aslr, bool enable_device_address_space_merge, bool from_back, KMemoryManager::Pool pool, void *table, KProcessAddress start, KProcessAddress end, KProcessAddress code_address, size_t code_size, KMemoryBlockSlabManager *mem_block_slab_manager, KBlockInfoManager *block_info_manager, KResourceLimit *resource_limit);

libraries/libmesosphere/include/mesosphere/kern_k_page_table_manager.hpp

@@ -27,9 +27,10 @@ namespace ams::kern {
         private:
             using BaseHeap = KDynamicResourceManager<impl::PageTablePage, true>;
         private:
-            KPageTableSlabHeap *m_pt_heap{};
+            KPageTableSlabHeap *m_pt_heap;
         public:
-            constexpr KPageTableManager() = default;
+            constexpr explicit KPageTableManager(util::ConstantInitializeTag) : m_pt_heap() { /* ... */ }
+            explicit KPageTableManager() { /* ... */ }
 
             ALWAYS_INLINE void Initialize(KDynamicPageManager *page_allocator, KPageTableSlabHeap *pt_heap) {
                 m_pt_heap = pt_heap;

libraries/libmesosphere/include/mesosphere/kern_k_port.hpp

@@ -41,11 +41,13 @@ namespace ams::kern {
             State m_state;
             bool m_is_light;
         public:
-            constexpr KPort() : m_server(), m_client(), m_name(), m_state(State::Invalid), m_is_light() { /* ... */ }
+            explicit KPort() : m_state(State::Invalid), m_is_light() { /* ... */ }
 
             static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
 
             void Initialize(s32 max_sessions, bool is_light, uintptr_t name);
+            void Finalize() { /* ... */ }
+
             void OnClientClosed();
             void OnServerClosed();
 

libraries/libmesosphere/include/mesosphere/kern_k_priority_queue.hpp

@@ -235,7 +235,7 @@ namespace ams::kern {
             KPriorityQueueImpl m_suggested_queue;
         private:
             constexpr ALWAYS_INLINE void ClearAffinityBit(u64 &affinity, s32 core) {
-                affinity &= ~(u64(1ul) << core);
+                affinity &= ~(UINT64_C(1) << core);
             }
 
             constexpr ALWAYS_INLINE s32 GetNextCore(u64 &affinity) {

libraries/libmesosphere/include/mesosphere/kern_k_process.hpp

@@ -34,7 +34,7 @@
 
 namespace ams::kern {
 
-    class KProcess final : public KAutoObjectWithSlabHeapAndContainer<KProcess, KSynchronizationObject>, public KWorkerTask {
+    class KProcess final : public KAutoObjectWithSlabHeapAndContainer<KProcess, KWorkerTask> {
         MESOSPHERE_AUTOOBJECT_TRAITS(KProcess, KSynchronizationObject);
         public:
             enum State {
@@ -57,73 +57,73 @@ namespace ams::kern {
             using TLPTree = util::IntrusiveRedBlackTreeBaseTraits<KThreadLocalPage>::TreeType<KThreadLocalPage>;
             using TLPIterator = TLPTree::iterator;
         private:
-            KProcessPageTable           m_page_table{};
-            std::atomic<size_t>         m_used_kernel_memory_size{};
-            TLPTree                     m_fully_used_tlp_tree{};
-            TLPTree                     m_partially_used_tlp_tree{};
-            s32                         m_ideal_core_id{};
-            void                       *m_attached_object{};
-            KResourceLimit             *m_resource_limit{};
-            KVirtualAddress             m_system_resource_address{};
-            size_t                      m_system_resource_num_pages{};
-            size_t                      m_memory_release_hint{};
-            State                       m_state{};
-            KLightLock                  m_state_lock{};
-            KLightLock                  m_list_lock{};
-            KConditionVariable          m_cond_var{};
-            KAddressArbiter             m_address_arbiter{};
-            u64                         m_entropy[4]{};
-            bool                        m_is_signaled{};
-            bool                        m_is_initialized{};
-            bool                        m_is_application{};
-            char                        m_name[13]{};
-            std::atomic<u16>            m_num_running_threads{};
-            u32                         m_flags{};
-            KMemoryManager::Pool        m_memory_pool{};
-            s64                         m_schedule_count{};
-            KCapabilities               m_capabilities{};
-            ams::svc::ProgramId         m_program_id{};
-            u64                         m_process_id{};
-            s64                         m_creation_time{};
-            KProcessAddress             m_code_address{};
-            size_t                      m_code_size{};
-            size_t                      m_main_thread_stack_size{};
-            size_t                      m_max_process_memory{};
-            u32                         m_version{};
-            KHandleTable                m_handle_table{};
-            KProcessAddress             m_plr_address{};
-            void                       *m_plr_heap_address{};
-            KThread                    *m_exception_thread{};
-            ThreadList                  m_thread_list{};
-            SharedMemoryInfoList        m_shared_memory_list{};
-            IoRegionList                m_io_region_list{};
-            bool                        m_is_suspended{};
-            bool                        m_is_immortal{};
-            bool                        m_is_jit_debug{};
-            bool                        m_is_handle_table_initialized{};
-            ams::svc::DebugEvent        m_jit_debug_event_type{};
-            ams::svc::DebugException    m_jit_debug_exception_type{};
-            uintptr_t                   m_jit_debug_params[4]{};
-            u64                         m_jit_debug_thread_id{};
-            KWaitObject                 m_wait_object{};
-            KThread                    *m_running_threads[cpu::NumCores]{};
-            u64                         m_running_thread_idle_counts[cpu::NumCores]{};
-            KThread                    *m_pinned_threads[cpu::NumCores]{};
-            std::atomic<s64>            m_cpu_time{};
-            std::atomic<s64>            m_num_process_switches{};
-            std::atomic<s64>            m_num_thread_switches{};
-            std::atomic<s64>            m_num_fpu_switches{};
-            std::atomic<s64>            m_num_supervisor_calls{};
-            std::atomic<s64>            m_num_ipc_messages{};
-            std::atomic<s64>            m_num_ipc_replies{};
-            std::atomic<s64>            m_num_ipc_receives{};
-            KDynamicPageManager         m_dynamic_page_manager{};
-            KMemoryBlockSlabManager     m_memory_block_slab_manager{};
-            KBlockInfoManager           m_block_info_manager{};
-            KPageTableManager           m_page_table_manager{};
-            KMemoryBlockSlabHeap        m_memory_block_heap{};
-            KBlockInfoSlabHeap          m_block_info_heap{};
-            KPageTableSlabHeap          m_page_table_heap{};
+            KProcessPageTable           m_page_table;
+            util::Atomic<size_t>        m_used_kernel_memory_size;
+            TLPTree                     m_fully_used_tlp_tree;
+            TLPTree                     m_partially_used_tlp_tree;
+            s32                         m_ideal_core_id;
+            void                       *m_attached_object;
+            KResourceLimit             *m_resource_limit;
+            KVirtualAddress             m_system_resource_address;
+            size_t                      m_system_resource_num_pages;
+            size_t                      m_memory_release_hint;
+            State                       m_state;
+            KLightLock                  m_state_lock;
+            KLightLock                  m_list_lock;
+            KConditionVariable          m_cond_var;
+            KAddressArbiter             m_address_arbiter;
+            u64                         m_entropy[4];
+            bool                        m_is_signaled;
+            bool                        m_is_initialized;
+            bool                        m_is_application;
+            char                        m_name[13];
+            util::Atomic<u16>           m_num_running_threads;
+            u32                         m_flags;
+            KMemoryManager::Pool        m_memory_pool;
+            s64                         m_schedule_count;
+            KCapabilities               m_capabilities;
+            ams::svc::ProgramId         m_program_id;
+            u64                         m_process_id;
+            s64                         m_creation_time;
+            KProcessAddress             m_code_address;
+            size_t                      m_code_size;
+            size_t                      m_main_thread_stack_size;
+            size_t                      m_max_process_memory;
+            u32                         m_version;
+            KHandleTable                m_handle_table;
+            KProcessAddress             m_plr_address;
+            void                       *m_plr_heap_address;
+            KThread                    *m_exception_thread;
+            ThreadList                  m_thread_list;
+            SharedMemoryInfoList        m_shared_memory_list;
+            IoRegionList                m_io_region_list;
+            bool                        m_is_suspended;
+            bool                        m_is_immortal;
+            bool                        m_is_jit_debug;
+            bool                        m_is_handle_table_initialized;
+            ams::svc::DebugEvent        m_jit_debug_event_type;
+            ams::svc::DebugException    m_jit_debug_exception_type;
+            uintptr_t                   m_jit_debug_params[4];
+            u64                         m_jit_debug_thread_id;
+            KWaitObject                 m_wait_object;
+            KThread                    *m_running_threads[cpu::NumCores];
+            u64                         m_running_thread_idle_counts[cpu::NumCores];
+            KThread                    *m_pinned_threads[cpu::NumCores];
+            util::Atomic<s64>           m_cpu_time;
+            util::Atomic<s64>           m_num_process_switches;
+            util::Atomic<s64>           m_num_thread_switches;
+            util::Atomic<s64>           m_num_fpu_switches;
+            util::Atomic<s64>           m_num_supervisor_calls;
+            util::Atomic<s64>           m_num_ipc_messages;
+            util::Atomic<s64>           m_num_ipc_replies;
+            util::Atomic<s64>           m_num_ipc_receives;
+            KDynamicPageManager         m_dynamic_page_manager;
+            KMemoryBlockSlabManager     m_memory_block_slab_manager;
+            KBlockInfoManager           m_block_info_manager;
+            KPageTableManager           m_page_table_manager;
+            KMemoryBlockSlabHeap        m_memory_block_heap;
+            KBlockInfoSlabHeap          m_block_info_heap;
+            KPageTableSlabHeap          m_page_table_heap;
         private:
             Result Initialize(const ams::svc::CreateProcessParameter &params);
 
@@ -145,7 +145,7 @@ namespace ams::kern {
                 m_pinned_threads[core_id] = nullptr;
             }
         public:
-            KProcess() { /* ... */ }
+            explicit KProcess() : m_is_initialized(false) { /* ... */ }
 
             Result Initialize(const ams::svc::CreateProcessParameter &params, const KPageGroup &pg, const u32 *caps, s32 num_caps, KResourceLimit *res_limit, KMemoryManager::Pool pool, bool immortal);
             Result Initialize(const ams::svc::CreateProcessParameter &params, svc::KUserPointer<const u32 *> caps, s32 num_caps, KResourceLimit *res_limit, KMemoryManager::Pool pool);
@@ -284,8 +284,12 @@ namespace ams::kern {
 
             constexpr KProcessAddress GetProcessLocalRegionAddress() const { return m_plr_address; }
 
+            constexpr void *GetProcessLocalRegionHeapAddress() const { return m_plr_heap_address; }
+
+            KThread *GetExceptionThread() const { return m_exception_thread; }
+
             void AddCpuTime(s64 diff) { m_cpu_time += diff; }
-            s64 GetCpuTime() { return m_cpu_time; }
+            s64 GetCpuTime() { return m_cpu_time.Load(); }
 
             constexpr s64 GetScheduledCount() const { return m_schedule_count; }
             void IncrementScheduledCount() { ++m_schedule_count; }
@@ -380,13 +384,14 @@ namespace ams::kern {
             }
         public:
             /* Overridden parent functions. */
-            virtual bool IsInitialized() const override { return m_is_initialized; }
+            bool IsInitialized() const { return m_is_initialized; }
 
             static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
 
-            virtual void Finalize() override;
+            void Finalize();
 
-            virtual u64 GetId() const override final { return this->GetProcessId(); }
+            ALWAYS_INLINE u64 GetIdImpl() const { return this->GetProcessId(); }
+            ALWAYS_INLINE u64 GetId() const { return this->GetIdImpl(); }
 
             virtual bool IsSignaled() const override {
                 MESOSPHERE_ASSERT_THIS();
@@ -394,7 +399,7 @@ namespace ams::kern {
                 return m_is_signaled;
             }
 
-            virtual void DoWorkerTask() override;
+            void DoWorkerTaskImpl();
         private:
             void ChangeState(State new_state) {
                 if (m_state != new_state) {

libraries/libmesosphere/include/mesosphere/kern_k_readable_event.hpp

@@ -27,18 +27,31 @@ namespace ams::kern {
             bool m_is_signaled;
             KEvent *m_parent;
         public:
-            constexpr explicit KReadableEvent() : KSynchronizationObject(), m_is_signaled(), m_parent() { MESOSPHERE_ASSERT_THIS(); }
+            constexpr explicit KReadableEvent(util::ConstantInitializeTag) : KSynchronizationObject(util::ConstantInitialize), m_is_signaled(), m_parent() { MESOSPHERE_ASSERT_THIS(); }
+
+            explicit KReadableEvent() { /* ... */ }
 
             void Initialize(KEvent *parent);
 
             constexpr KEvent *GetParent() const { return m_parent; }
 
             Result Signal();
-            Result Clear();
+            Result Reset();
+
+            Result Clear() {
+                MESOSPHERE_ASSERT_THIS();
+
+                /* Try to perform a reset, succeeding unconditionally. */
+                this->Reset();
+
+                return ResultSuccess();
+            }
 
             virtual bool IsSignaled() const override;
             virtual void Destroy() override;
-            virtual Result Reset();
+
+            /* NOTE: This is a virtual function in Nintendo's kernel. */
+            /* virtual Result Reset(); */
     };
 
 }

libraries/libmesosphere/include/mesosphere/kern_k_resource_limit.hpp

@@ -33,12 +33,20 @@ namespace ams::kern {
             s32 m_waiter_count;
             KLightConditionVariable m_cond_var;
         public:
-            constexpr ALWAYS_INLINE KResourceLimit() : m_limit_values(), m_current_values(), m_current_hints(), m_peak_values(), m_lock(), m_waiter_count(), m_cond_var() { /* ... */ }
+            constexpr explicit ALWAYS_INLINE KResourceLimit(util::ConstantInitializeTag)
+                : KAutoObjectWithSlabHeapAndContainer<KResourceLimit, KAutoObjectWithList>(util::ConstantInitialize),
+                  m_limit_values(), m_current_values(), m_current_hints(), m_peak_values(), m_lock(), m_waiter_count(),
+                  m_cond_var(util::ConstantInitialize)
+            {
+                /* ... */
+            }
+
+            explicit ALWAYS_INLINE KResourceLimit() { /* ... */ }
 
             static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
 
             void Initialize();
-            virtual void Finalize() override;
+            void Finalize();
 
             s64 GetLimitValue(ams::svc::LimitableResource which) const;
             s64 GetCurrentValue(ams::svc::LimitableResource which) const;

libraries/libmesosphere/include/mesosphere/kern_k_scheduler.hpp

@@ -39,14 +39,16 @@ namespace ams::kern {
             static_assert(ams::svc::HighestThreadPriority <= HighestCoreMigrationAllowedPriority);
 
             struct SchedulingState {
-                std::atomic<u8> needs_scheduling;
-                bool interrupt_task_runnable;
-                bool should_count_idle;
-                u64  idle_count;
-                KThread *highest_priority_thread;
-                void *idle_thread_stack;
-                KThread *prev_thread;
-                KInterruptTaskManager *interrupt_task_manager;
+                util::Atomic<bool> needs_scheduling{false};
+                bool interrupt_task_runnable{false};
+                bool should_count_idle{false};
+                u64  idle_count{0};
+                KThread *highest_priority_thread{nullptr};
+                void *idle_thread_stack{nullptr};
+                KThread *prev_thread{nullptr};
+                KInterruptTaskManager *interrupt_task_manager{nullptr};
+
+                constexpr SchedulingState() = default;
             };
         private:
             friend class KScopedSchedulerLock;
@@ -58,10 +60,9 @@ namespace ams::kern {
             s32 m_core_id;
             s64 m_last_context_switch_time;
             KThread *m_idle_thread;
-            std::atomic<KThread *> m_current_thread;
+            util::Atomic<KThread *> m_current_thread;
         public:
-            constexpr KScheduler()
-                : m_state(), m_is_active(false), m_core_id(0), m_last_context_switch_time(0), m_idle_thread(nullptr), m_current_thread(nullptr)
+            constexpr KScheduler() : m_state(), m_is_active(false), m_core_id(0), m_last_context_switch_time(0), m_idle_thread(nullptr), m_current_thread(nullptr)
             {
                 m_state.needs_scheduling        = true;
                 m_state.interrupt_task_runnable = false;
@@ -102,7 +103,7 @@ namespace ams::kern {
             }
 
             ALWAYS_INLINE KThread *GetSchedulerCurrentThread() const {
-                return m_current_thread;
+                return m_current_thread.Load();
             }
 
             ALWAYS_INLINE s64 GetLastContextSwitchTime() const {
@@ -182,7 +183,7 @@ namespace ams::kern {
 
                 GetCurrentThread().EnableDispatch();
 
-                if (m_state.needs_scheduling.load()) {
+                if (m_state.needs_scheduling.Load()) {
                     /* Disable interrupts, and then check again if rescheduling is needed. */
                     KScopedInterruptDisable intr_disable;
 
@@ -192,7 +193,7 @@ namespace ams::kern {
 
             ALWAYS_INLINE void RescheduleCurrentCoreImpl() {
                 /* Check that scheduling is needed. */
-                if (AMS_LIKELY(m_state.needs_scheduling.load())) {
+                if (AMS_LIKELY(m_state.needs_scheduling.Load())) {
                     GetCurrentThread().DisableDispatch();
                     this->Schedule();
                     GetCurrentThread().EnableDispatch();
@@ -211,12 +212,12 @@ namespace ams::kern {
     };
 
     consteval bool KScheduler::ValidateAssemblyOffsets() {
-        static_assert(__builtin_offsetof(KScheduler, m_state.needs_scheduling)        == KSCHEDULER_NEEDS_SCHEDULING);
-        static_assert(__builtin_offsetof(KScheduler, m_state.interrupt_task_runnable) == KSCHEDULER_INTERRUPT_TASK_RUNNABLE);
-        static_assert(__builtin_offsetof(KScheduler, m_state.highest_priority_thread) == KSCHEDULER_HIGHEST_PRIORITY_THREAD);
-        static_assert(__builtin_offsetof(KScheduler, m_state.idle_thread_stack)       == KSCHEDULER_IDLE_THREAD_STACK);
-        static_assert(__builtin_offsetof(KScheduler, m_state.prev_thread)             == KSCHEDULER_PREVIOUS_THREAD);
-        static_assert(__builtin_offsetof(KScheduler, m_state.interrupt_task_manager)  == KSCHEDULER_INTERRUPT_TASK_MANAGER);
+        static_assert(AMS_OFFSETOF(KScheduler, m_state.needs_scheduling)        == KSCHEDULER_NEEDS_SCHEDULING);
+        static_assert(AMS_OFFSETOF(KScheduler, m_state.interrupt_task_runnable) == KSCHEDULER_INTERRUPT_TASK_RUNNABLE);
+        static_assert(AMS_OFFSETOF(KScheduler, m_state.highest_priority_thread) == KSCHEDULER_HIGHEST_PRIORITY_THREAD);
+        static_assert(AMS_OFFSETOF(KScheduler, m_state.idle_thread_stack)       == KSCHEDULER_IDLE_THREAD_STACK);
+        static_assert(AMS_OFFSETOF(KScheduler, m_state.prev_thread)             == KSCHEDULER_PREVIOUS_THREAD);
+        static_assert(AMS_OFFSETOF(KScheduler, m_state.interrupt_task_manager)  == KSCHEDULER_INTERRUPT_TASK_MANAGER);
 
         return true;
     }

libraries/libmesosphere/include/mesosphere/kern_k_server_port.hpp

@@ -34,7 +34,8 @@ namespace ams::kern {
             LightSessionList m_light_session_list;
             KPort *m_parent;
         public:
-            constexpr KServerPort() : m_session_list(), m_light_session_list(), m_parent() { /* ... */ }
+            constexpr explicit KServerPort(util::ConstantInitializeTag) : KSynchronizationObject(util::ConstantInitialize), m_session_list(), m_light_session_list(), m_parent() { /* ... */ }
+            explicit KServerPort() { /* ... */ }
 
             void Initialize(KPort *parent);
             void EnqueueSession(KServerSession *session);

libraries/libmesosphere/include/mesosphere/kern_k_server_session.hpp

@@ -33,7 +33,8 @@ namespace ams::kern {
             KSessionRequest *m_current_request;
             KLightLock m_lock;
         public:
-            constexpr KServerSession() : m_parent(), m_request_list(), m_current_request(), m_lock() { /* ... */ }
+            constexpr explicit KServerSession(util::ConstantInitializeTag) : KSynchronizationObject(util::ConstantInitialize), m_parent(), m_request_list(), m_current_request(), m_lock() { /* ... */ }
+            explicit KServerSession() : m_current_request(nullptr), m_lock() { /* ... */ }
 
             virtual void Destroy() override;
 

libraries/libmesosphere/include/mesosphere/kern_k_session.hpp

@@ -35,33 +35,37 @@ namespace ams::kern {
                 ServerClosed = 3,
             };
         private:
+            util::Atomic<std::underlying_type<State>::type> m_atomic_state;
+            bool m_initialized;
             KServerSession m_server;
             KClientSession m_client;
-            std::atomic<std::underlying_type<State>::type> m_atomic_state;
             KClientPort *m_port;
             uintptr_t m_name;
             KProcess *m_process;
-            bool m_initialized;
         private:
             ALWAYS_INLINE void SetState(State state) {
                 m_atomic_state = static_cast<u8>(state);
             }
 
             ALWAYS_INLINE State GetState() const {
-                return static_cast<State>(m_atomic_state.load());
+                return static_cast<State>(m_atomic_state.Load());
             }
         public:
-            constexpr KSession()
-                : m_server(), m_client(), m_atomic_state(static_cast<std::underlying_type<State>::type>(State::Invalid)), m_port(), m_name(), m_process(), m_initialized()
+            constexpr explicit KSession(util::ConstantInitializeTag)
+                : KAutoObjectWithSlabHeapAndContainer<KSession, KAutoObjectWithList, true>(util::ConstantInitialize),
+                  m_atomic_state(static_cast<std::underlying_type<State>::type>(State::Invalid)), m_initialized(),
+                  m_server(util::ConstantInitialize), m_client(util::ConstantInitialize),  m_port(), m_name(), m_process()
             {
                 /* ... */
             }
 
-            void Initialize(KClientPort *client_port, uintptr_t name);
-            virtual void Finalize() override;
+            explicit KSession() : m_atomic_state(util::ToUnderlying(State::Invalid)), m_initialized(false), m_process(nullptr) { /* ... */ }
 
-            virtual bool IsInitialized() const override { return m_initialized; }
-            virtual uintptr_t GetPostDestroyArgument() const override { return reinterpret_cast<uintptr_t>(m_process); }
+            void Initialize(KClientPort *client_port, uintptr_t name);
+            void Finalize();
+
+            bool IsInitialized() const { return m_initialized; }
+            uintptr_t GetPostDestroyArgument() const { return reinterpret_cast<uintptr_t>(m_process); }
 
             static void PostDestroy(uintptr_t arg);
 

libraries/libmesosphere/include/mesosphere/kern_k_session_request.hpp

@@ -33,14 +33,14 @@ namespace ams::kern {
 
                     class Mapping {
                         private:
-                            KProcessAddress m_client_address;
-                            KProcessAddress m_server_address;
+                            uintptr_t m_client_address;
+                            uintptr_t m_server_address;
                             size_t m_size;
                             KMemoryState m_state;
                         public:
                             constexpr void Set(KProcessAddress c, KProcessAddress s, size_t sz, KMemoryState st) {
-                                m_client_address = c;
-                                m_server_address = s;
+                                m_client_address = GetInteger(c);
+                                m_server_address = GetInteger(s);
                                 m_size           = sz;
                                 m_state          = st;
                             }
@@ -57,7 +57,9 @@ namespace ams::kern {
                     u8 m_num_recv;
                     u8 m_num_exch;
                 public:
-                    constexpr explicit SessionMappings() : m_static_mappings(), m_mappings(), m_num_send(), m_num_recv(), m_num_exch() { /* ... */ }
+                    constexpr explicit SessionMappings(util::ConstantInitializeTag) : m_static_mappings(), m_mappings(), m_num_send(), m_num_recv(), m_num_exch() { /* ... */ }
+
+                    explicit SessionMappings() : m_mappings(nullptr), m_num_send(), m_num_recv(), m_num_exch() { /* ... */ }
 
                     void Initialize() { /* ... */ }
                     void Finalize();
@@ -119,8 +121,6 @@ namespace ams::kern {
                             return m_mappings[index - NumStaticMappings];
                         }
                     }
-
-
             };
         private:
             SessionMappings m_mappings;
@@ -130,20 +130,22 @@ namespace ams::kern {
             uintptr_t m_address;
             size_t m_size;
         public:
-            constexpr KSessionRequest() : m_mappings(), m_thread(), m_server(), m_event(), m_address(), m_size() { /* ... */ }
+            constexpr explicit KSessionRequest(util::ConstantInitializeTag) : KAutoObject(util::ConstantInitialize), m_mappings(util::ConstantInitialize), m_thread(), m_server(), m_event(), m_address(), m_size() { /* ... */ }
+
+            explicit KSessionRequest() : m_thread(nullptr), m_server(nullptr), m_event(nullptr) { /* ... */ }
 
             static KSessionRequest *Create() {
                 KSessionRequest *req = KSessionRequest::Allocate();
-                if (req != nullptr) {
-                    KAutoObject::Create(req);
+                if (AMS_LIKELY(req != nullptr)) {
+                    KAutoObject::Create<KSessionRequest>(req);
                 }
                 return req;
             }
 
             static KSessionRequest *CreateFromUnusedSlabMemory() {
                 KSessionRequest *req = KSessionRequest::AllocateFromUnusedSlabMemory();
-                if (req != nullptr) {
-                    KAutoObject::Create(req);
+                if (AMS_LIKELY(req != nullptr)) {
+                    KAutoObject::Create<KSessionRequest>(req);
                 }
                 return req;
             }
@@ -167,20 +169,6 @@ namespace ams::kern {
                 }
             }
 
-            virtual void Finalize() override {
-                m_mappings.Finalize();
-
-                if (m_thread) {
-                    m_thread->Close();
-                }
-                if (m_event) {
-                    m_event->Close();
-                }
-                if (m_server) {
-                    m_server->Close();
-                }
-            }
-
             static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
 
             constexpr ALWAYS_INLINE KThread *GetThread() const { return m_thread; }
@@ -227,6 +215,21 @@ namespace ams::kern {
             constexpr ALWAYS_INLINE KProcessAddress GetExchangeServerAddress(size_t i) const { return m_mappings.GetExchangeServerAddress(i);  }
             constexpr ALWAYS_INLINE size_t          GetExchangeSize(size_t i)          const { return m_mappings.GetExchangeSize(i);           }
             constexpr ALWAYS_INLINE KMemoryState    GetExchangeMemoryState(size_t i)   const { return m_mappings.GetExchangeMemoryState(i);    }
+        private:
+            /* NOTE: This is public and virtual in Nintendo's kernel. */
+            void Finalize() {
+                m_mappings.Finalize();
+
+                if (m_thread) {
+                    m_thread->Close();
+                }
+                if (m_event) {
+                    m_event->Close();
+                }
+                if (m_server) {
+                    m_server->Close();
+                }
+            }
     };
 
 }

libraries/libmesosphere/include/mesosphere/kern_k_shared_memory.hpp

@@ -42,9 +42,9 @@ namespace ams::kern {
             }
 
             Result Initialize(KProcess *owner, size_t size, ams::svc::MemoryPermission own_perm, ams::svc::MemoryPermission rem_perm);
-            virtual void Finalize() override;
+            void Finalize();
 
-            virtual bool IsInitialized() const override { return m_is_initialized; }
+            bool IsInitialized() const { return m_is_initialized; }
             static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
 
             Result Map(KProcessPageTable *table, KProcessAddress address, size_t size, KProcess *process, ams::svc::MemoryPermission map_perm);

libraries/libmesosphere/include/mesosphere/kern_k_shared_memory_info.hpp

@@ -26,7 +26,7 @@ namespace ams::kern {
             KSharedMemory *m_shared_memory;
             size_t m_reference_count;
         public:
-            constexpr KSharedMemoryInfo() : m_shared_memory(), m_reference_count() { /* ... */ }
+            explicit KSharedMemoryInfo() { /* ... */ }
             ~KSharedMemoryInfo() { /* ... */ }
 
             constexpr void Initialize(KSharedMemory *m) {

libraries/libmesosphere/include/mesosphere/kern_k_slab_heap.hpp

@@ -81,8 +81,7 @@ namespace ams::kern {
             uintptr_t m_end{};
         private:
             ALWAYS_INLINE void UpdatePeakImpl(uintptr_t obj) {
-                static_assert(std::atomic_ref<uintptr_t>::is_always_lock_free);
-                std::atomic_ref<uintptr_t> peak_ref(m_peak);
+                const util::AtomicRef<uintptr_t> peak_ref(m_peak);
 
                 const uintptr_t alloc_peak = obj + this->GetObjectSize();
                 uintptr_t cur_peak = m_peak;
@@ -90,7 +89,7 @@ namespace ams::kern {
                     if (alloc_peak <= cur_peak) {
                         break;
                     }
-                } while (!peak_ref.compare_exchange_strong(cur_peak, alloc_peak));
+                } while (!peak_ref.CompareExchangeStrong(cur_peak, alloc_peak));
             }
         public:
             constexpr KSlabHeapBase() = default;
@@ -112,8 +111,8 @@ namespace ams::kern {
                 /* Set our tracking variables. */
                 const size_t num_obj = (memory_size / obj_size);
                 m_start = reinterpret_cast<uintptr_t>(memory);
-                m_end  = m_start + num_obj * obj_size;
-                m_peak = m_start;
+                m_end   = m_start + num_obj * obj_size;
+                m_peak  = m_start;
 
                 /* Free the objects. */
                 u8 *cur = reinterpret_cast<u8 *>(m_end);

libraries/libmesosphere/include/mesosphere/kern_k_synchronization_object.hpp

@@ -32,18 +32,20 @@ namespace ams::kern {
             ThreadListNode *m_thread_list_head;
             ThreadListNode *m_thread_list_tail;
         protected:
-            constexpr ALWAYS_INLINE explicit KSynchronizationObject() : KAutoObjectWithList(), m_thread_list_head(), m_thread_list_tail() { MESOSPHERE_ASSERT_THIS(); }
+            constexpr ALWAYS_INLINE explicit KSynchronizationObject(util::ConstantInitializeTag) : KAutoObjectWithList(util::ConstantInitialize), m_thread_list_head(), m_thread_list_tail() { MESOSPHERE_ASSERT_THIS(); }
+            ALWAYS_INLINE explicit KSynchronizationObject() : m_thread_list_head(), m_thread_list_tail() { MESOSPHERE_ASSERT_THIS(); }
 
-            virtual void OnFinalizeSynchronizationObject() { MESOSPHERE_ASSERT_THIS(); }
+            /* NOTE: This is a virtual function which is overridden only by KDebugBase in Nintendo's kernel. */
+            /* virtual void OnFinalizeSynchronizationObject() { MESOSPHERE_ASSERT_THIS(); } */
 
             void NotifyAvailable(Result result);
-            void NotifyAvailable() {
+            ALWAYS_INLINE void NotifyAvailable() {
                 return this->NotifyAvailable(ResultSuccess());
             }
         public:
             static Result Wait(s32 *out_index, KSynchronizationObject **objects, const s32 num_objects, s64 timeout);
         public:
-            virtual void Finalize() override;
+            void Finalize();
             virtual bool IsSignaled() const { AMS_INFINITE_LOOP(); }
 
             void DumpWaiters();

libraries/libmesosphere/include/mesosphere/kern_k_system_control_base.hpp

@@ -0,0 +1,110 @@
+/*
+ * Copyright (c) Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_k_spin_lock.hpp>
+
+namespace ams::kern {
+
+    struct InitialProcessBinaryLayout;
+
+}
+
+namespace ams::kern {
+
+    class KSystemControlBase {
+        protected:
+            /* Nintendo uses std::mt19937_t for randomness. */
+            /* To save space (and because mt19337_t isn't secure anyway), */
+            /* We will use TinyMT. */
+            static constinit inline bool         s_initialized_random_generator;
+            static constinit inline util::TinyMT s_random_generator{util::ConstantInitialize};
+            static constinit inline KSpinLock    s_random_lock;
+        public:
+            class Init {
+                public:
+                    /* Initialization. */
+                    static size_t GetRealMemorySize();
+                    static size_t GetIntendedMemorySize();
+                    static KPhysicalAddress GetKernelPhysicalBaseAddress(KPhysicalAddress base_address);
+                    static void GetInitialProcessBinaryLayout(InitialProcessBinaryLayout *out);
+                    static bool ShouldIncreaseThreadResourceLimit();
+                    static void CpuOn(u64 core_id, uintptr_t entrypoint, uintptr_t arg);
+                    static size_t GetApplicationPoolSize();
+                    static size_t GetAppletPoolSize();
+                    static size_t GetMinimumNonSecureSystemPoolSize();
+                    static u8 GetDebugLogUartPort();
+
+                    /* Randomness. */
+                    static void GenerateRandom(u64 *dst, size_t count);
+                    static u64  GenerateRandomRange(u64 min, u64 max);
+            };
+        public:
+            /* Initialization. */
+            static NOINLINE void InitializePhase1(bool skip_target_system = false);
+            static NOINLINE void InitializePhase2();
+            static NOINLINE u32 GetCreateProcessMemoryPool();
+
+            /* Randomness. */
+            static void GenerateRandom(u64 *dst, size_t count);
+            static u64  GenerateRandomRange(u64 min, u64 max);
+            static u64  GenerateRandomU64();
+
+            /* Register access Access. */
+            static Result ReadWriteRegister(u32 *out, ams::svc::PhysicalAddress address, u32 mask, u32 value);
+            static void ReadWriteRegisterPrivileged(u32 *out, ams::svc::PhysicalAddress address, u32 mask, u32 value);
+
+            static u32 ReadRegisterPrivileged(ams::svc::PhysicalAddress address);
+            static void WriteRegisterPrivileged(ams::svc::PhysicalAddress address, u32 value);
+
+            /* Power management. */
+            static void SleepSystem();
+            static NORETURN void StopSystem(void *arg = nullptr);
+
+            /* User access. */
+            #if defined(ATMOSPHERE_ARCH_ARM64)
+            static void CallSecureMonitorFromUser(ams::svc::lp64::SecureMonitorArguments *args);
+            #endif
+
+            /* Secure Memory. */
+            static size_t CalculateRequiredSecureMemorySize(size_t size, u32 pool);
+            static Result AllocateSecureMemory(KVirtualAddress *out, size_t size, u32 pool);
+            static void FreeSecureMemory(KVirtualAddress address, size_t size, u32 pool);
+        protected:
+            template<typename F>
+            static ALWAYS_INLINE u64 GenerateUniformRange(u64 min, u64 max, F f) {
+                /* Handle the case where the difference is too large to represent. */
+                if (max == std::numeric_limits<u64>::max() && min == std::numeric_limits<u64>::min()) {
+                    return f();
+                }
+
+                /* Iterate until we get a value in range. */
+                const u64 range_size    = ((max + 1) - min);
+                const u64 effective_max = (std::numeric_limits<u64>::max() / range_size) * range_size;
+                while (true) {
+                    if (const u64 rnd = f(); rnd < effective_max) {
+                        return min + (rnd % range_size);
+                    }
+                }
+            }
+
+            /* User access. */
+            #if defined(ATMOSPHERE_ARCH_ARM64)
+            static void CallSecureMonitorFromUserImpl(ams::svc::lp64::SecureMonitorArguments *args);
+            #endif
+    };
+
+}

libraries/libmesosphere/include/mesosphere/kern_k_target_system.hpp

@@ -21,6 +21,7 @@ namespace ams::kern {
 
     class KTargetSystem {
         private:
+            friend class KSystemControlBase;
             friend class KSystemControl;
         private:
             static inline constinit bool s_is_debug_mode;

libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp

@@ -32,7 +32,7 @@ namespace ams::kern {
 
     using KThreadFunction = void (*)(uintptr_t);
 
-    class KThread final : public KAutoObjectWithSlabHeapAndContainer<KThread, KSynchronizationObject>, public util::IntrusiveListBaseNode<KThread>, public KTimerTask, public KWorkerTask {
+    class KThread final : public KAutoObjectWithSlabHeapAndContainer<KThread, KWorkerTask>, public util::IntrusiveListBaseNode<KThread>, public KTimerTask {
         MESOSPHERE_AUTOOBJECT_TRAITS(KThread, KSynchronizationObject);
         private:
             friend class KProcess;
@@ -89,7 +89,7 @@ namespace ams::kern {
                 KThreadContext *context;
                 KThread *cur_thread;
                 s16 disable_count;
-                std::atomic<u8> dpc_flags;
+                util::Atomic<u8> dpc_flags;
                 u8 current_svc_id;
                 bool is_calling_svc;
                 bool is_in_exception_handler;
@@ -101,18 +101,18 @@ namespace ams::kern {
             static_assert(alignof(StackParameters) == 0x10);
             static_assert(sizeof(StackParameters) == THREAD_STACK_PARAMETERS_SIZE);
 
-            static_assert(__builtin_offsetof(StackParameters, svc_access_flags)        == THREAD_STACK_PARAMETERS_SVC_PERMISSION);
-            static_assert(__builtin_offsetof(StackParameters, context)                 == THREAD_STACK_PARAMETERS_CONTEXT);
-            static_assert(__builtin_offsetof(StackParameters, cur_thread)              == THREAD_STACK_PARAMETERS_CUR_THREAD);
-            static_assert(__builtin_offsetof(StackParameters, disable_count)           == THREAD_STACK_PARAMETERS_DISABLE_COUNT);
-            static_assert(__builtin_offsetof(StackParameters, dpc_flags)               == THREAD_STACK_PARAMETERS_DPC_FLAGS);
-            static_assert(__builtin_offsetof(StackParameters, current_svc_id)          == THREAD_STACK_PARAMETERS_CURRENT_SVC_ID);
-            static_assert(__builtin_offsetof(StackParameters, is_calling_svc)          == THREAD_STACK_PARAMETERS_IS_CALLING_SVC);
-            static_assert(__builtin_offsetof(StackParameters, is_in_exception_handler) == THREAD_STACK_PARAMETERS_IS_IN_EXCEPTION_HANDLER);
-            static_assert(__builtin_offsetof(StackParameters, is_pinned)               == THREAD_STACK_PARAMETERS_IS_PINNED);
+            static_assert(AMS_OFFSETOF(StackParameters, svc_access_flags)        == THREAD_STACK_PARAMETERS_SVC_PERMISSION);
+            static_assert(AMS_OFFSETOF(StackParameters, context)                 == THREAD_STACK_PARAMETERS_CONTEXT);
+            static_assert(AMS_OFFSETOF(StackParameters, cur_thread)              == THREAD_STACK_PARAMETERS_CUR_THREAD);
+            static_assert(AMS_OFFSETOF(StackParameters, disable_count)           == THREAD_STACK_PARAMETERS_DISABLE_COUNT);
+            static_assert(AMS_OFFSETOF(StackParameters, dpc_flags)               == THREAD_STACK_PARAMETERS_DPC_FLAGS);
+            static_assert(AMS_OFFSETOF(StackParameters, current_svc_id)          == THREAD_STACK_PARAMETERS_CURRENT_SVC_ID);
+            static_assert(AMS_OFFSETOF(StackParameters, is_calling_svc)          == THREAD_STACK_PARAMETERS_IS_CALLING_SVC);
+            static_assert(AMS_OFFSETOF(StackParameters, is_in_exception_handler) == THREAD_STACK_PARAMETERS_IS_IN_EXCEPTION_HANDLER);
+            static_assert(AMS_OFFSETOF(StackParameters, is_pinned)               == THREAD_STACK_PARAMETERS_IS_PINNED);
 
             #if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP)
-            static_assert(__builtin_offsetof(StackParameters, is_single_step)          == THREAD_STACK_PARAMETERS_IS_SINGLE_STEP);
+            static_assert(AMS_OFFSETOF(StackParameters, is_single_step)          == THREAD_STACK_PARAMETERS_IS_SINGLE_STEP);
             #endif
 
             struct QueueEntry {
@@ -120,8 +120,6 @@ namespace ams::kern {
                     KThread *m_prev;
                     KThread *m_next;
                 public:
-                    constexpr QueueEntry() : m_prev(nullptr), m_next(nullptr) { /* ... */ }
-
                     constexpr void Initialize() {
                         m_prev = nullptr;
                         m_next = nullptr;
@@ -140,7 +138,9 @@ namespace ams::kern {
                 KSynchronizationObject *m_sync_objects[ams::svc::ArgumentHandleCountMax];
                 ams::svc::Handle        m_handles[ams::svc::ArgumentHandleCountMax * (sizeof(KSynchronizationObject *) / sizeof(ams::svc::Handle))];
 
-                constexpr SyncObjectBuffer() : m_sync_objects() { /* ... */ }
+                constexpr explicit SyncObjectBuffer(util::ConstantInitializeTag) : m_sync_objects() { /* ... */ }
+
+                explicit SyncObjectBuffer() { /* ... */ }
             };
             static_assert(sizeof(SyncObjectBuffer::m_sync_objects) == sizeof(SyncObjectBuffer::m_handles));
 
@@ -177,67 +177,79 @@ namespace ams::kern {
             static_assert(ams::util::HasRedBlackKeyType<ConditionVariableComparator>);
             static_assert(std::same_as<ams::util::RedBlackKeyType<ConditionVariableComparator, void>, ConditionVariableComparator::RedBlackKeyType>);
         private:
-            static inline std::atomic<u64> s_next_thread_id = 0;
-        private:
-            alignas(16) KThreadContext      m_thread_context{};
-            util::IntrusiveListNode         m_process_list_node{};
-            util::IntrusiveRedBlackTreeNode m_condvar_arbiter_tree_node{};
-            s32                             m_priority{};
+            util::IntrusiveListNode         m_process_list_node;
+            util::IntrusiveRedBlackTreeNode m_condvar_arbiter_tree_node;
+            s32                             m_priority;
 
             using ConditionVariableThreadTreeTraits = util::IntrusiveRedBlackTreeMemberTraitsDeferredAssert<&KThread::m_condvar_arbiter_tree_node>;
             using ConditionVariableThreadTree       = ConditionVariableThreadTreeTraits::TreeType<ConditionVariableComparator>;
 
-            ConditionVariableThreadTree    *m_condvar_tree{};
-            uintptr_t                       m_condvar_key{};
-            u64                             m_virtual_affinity_mask{};
-            KAffinityMask                   m_physical_affinity_mask{};
-            u64                             m_thread_id{};
-            std::atomic<s64>                m_cpu_time{};
-            KProcessAddress                 m_address_key{};
-            KProcess                       *m_parent{};
-            void                           *m_kernel_stack_top{};
-            u32                            *m_light_ipc_data{};
-            KProcessAddress                 m_tls_address{};
-            void                           *m_tls_heap_address{};
-            KLightLock                      m_activity_pause_lock{};
-            SyncObjectBuffer                m_sync_object_buffer{};
-            s64                             m_schedule_count{};
-            s64                             m_last_scheduled_tick{};
-            QueueEntry                      m_per_core_priority_queue_entry[cpu::NumCores]{};
-            KLightLock                     *m_waiting_lock{};
-            KThreadQueue                   *m_wait_queue{};
-            WaiterList                      m_waiter_list{};
-            WaiterList                      m_pinned_waiter_list{};
-            KThread                        *m_lock_owner{};
-            uintptr_t                       m_debug_params[3]{};
-            KAutoObject                    *m_closed_object{};
-            u32                             m_address_key_value{};
-            u32                             m_suspend_request_flags{};
-            u32                             m_suspend_allowed_flags{};
-            s32                             m_synced_index{};
+            ConditionVariableThreadTree    *m_condvar_tree;
+            uintptr_t                       m_condvar_key;
+            alignas(16) KThreadContext      m_thread_context;
+            u64                             m_virtual_affinity_mask;
+            KAffinityMask                   m_physical_affinity_mask;
+            u64                             m_thread_id;
+            util::Atomic<s64>               m_cpu_time;
+            KProcessAddress                 m_address_key;
+            KProcess                       *m_parent;
+            void                           *m_kernel_stack_top;
+            u32                            *m_light_ipc_data;
+            KProcessAddress                 m_tls_address;
+            void                           *m_tls_heap_address;
+            KLightLock                      m_activity_pause_lock;
+            SyncObjectBuffer                m_sync_object_buffer;
+            s64                             m_schedule_count;
+            s64                             m_last_scheduled_tick;
+            QueueEntry                      m_per_core_priority_queue_entry[cpu::NumCores];
+            KThreadQueue                   *m_wait_queue;
+            WaiterList                      m_waiter_list;
+            WaiterList                      m_pinned_waiter_list;
+            KThread                        *m_lock_owner;
+            uintptr_t                       m_debug_params[3];
+            KAutoObject                    *m_closed_object;
+            u32                             m_address_key_value;
+            u32                             m_suspend_request_flags;
+            u32                             m_suspend_allowed_flags;
+            s32                             m_synced_index;
             Result                          m_wait_result;
             Result                          m_debug_exception_result;
-            s32                             m_base_priority{};
-            s32                             m_base_priority_on_unpin{};
-            s32                             m_physical_ideal_core_id{};
-            s32                             m_virtual_ideal_core_id{};
-            s32                             m_num_kernel_waiters{};
-            s32                             m_current_core_id{};
-            s32                             m_core_id{};
-            KAffinityMask                   m_original_physical_affinity_mask{};
-            s32                             m_original_physical_ideal_core_id{};
-            s32                             m_num_core_migration_disables{};
-            ThreadState                     m_thread_state{};
-            std::atomic<u8>                 m_termination_requested{};
-            bool                            m_wait_cancelled{};
-            bool                            m_cancellable{};
-            bool                            m_signaled{};
-            bool                            m_initialized{};
-            bool                            m_debug_attached{};
-            s8                              m_priority_inheritance_count{};
-            bool                            m_resource_limit_release_hint{};
+            s32                             m_base_priority;
+            s32                             m_base_priority_on_unpin;
+            s32                             m_physical_ideal_core_id;
+            s32                             m_virtual_ideal_core_id;
+            s32                             m_num_kernel_waiters;
+            s32                             m_current_core_id;
+            s32                             m_core_id;
+            KAffinityMask                   m_original_physical_affinity_mask;
+            s32                             m_original_physical_ideal_core_id;
+            s32                             m_num_core_migration_disables;
+            ThreadState                     m_thread_state;
+            util::Atomic<bool>              m_termination_requested;
+            bool                            m_wait_cancelled;
+            bool                            m_cancellable;
+            bool                            m_signaled;
+            bool                            m_initialized;
+            bool                            m_debug_attached;
+            s8                              m_priority_inheritance_count;
+            bool                            m_resource_limit_release_hint;
         public:
-            constexpr KThread() : m_wait_result(svc::ResultNoSynchronizationObject()), m_debug_exception_result(ResultSuccess()) { /* ... */ }
+            constexpr explicit KThread(util::ConstantInitializeTag)
+                : KAutoObjectWithSlabHeapAndContainer<KThread, KWorkerTask>(util::ConstantInitialize), KTimerTask(util::ConstantInitialize),
+                  m_process_list_node{}, m_condvar_arbiter_tree_node{util::ConstantInitialize}, m_priority{-1}, m_condvar_tree{}, m_condvar_key{},
+                  m_thread_context{util::ConstantInitialize}, m_virtual_affinity_mask{}, m_physical_affinity_mask{}, m_thread_id{}, m_cpu_time{0}, m_address_key{Null<KProcessAddress>}, m_parent{},
+                  m_kernel_stack_top{}, m_light_ipc_data{}, m_tls_address{Null<KProcessAddress>}, m_tls_heap_address{}, m_activity_pause_lock{}, m_sync_object_buffer{util::ConstantInitialize},
+                  m_schedule_count{}, m_last_scheduled_tick{}, m_per_core_priority_queue_entry{}, m_wait_queue{}, m_waiter_list{}, m_pinned_waiter_list{},
+                  m_lock_owner{}, m_debug_params{}, m_closed_object{}, m_address_key_value{}, m_suspend_request_flags{}, m_suspend_allowed_flags{}, m_synced_index{},
+                  m_wait_result{svc::ResultNoSynchronizationObject()}, m_debug_exception_result{ResultSuccess()}, m_base_priority{}, m_base_priority_on_unpin{},
+                  m_physical_ideal_core_id{}, m_virtual_ideal_core_id{}, m_num_kernel_waiters{}, m_current_core_id{}, m_core_id{}, m_original_physical_affinity_mask{},
+                  m_original_physical_ideal_core_id{}, m_num_core_migration_disables{}, m_thread_state{}, m_termination_requested{false}, m_wait_cancelled{},
+                  m_cancellable{}, m_signaled{}, m_initialized{}, m_debug_attached{}, m_priority_inheritance_count{}, m_resource_limit_release_hint{}
+            {
+                /* ... */
+            }
+
+            explicit KThread() : m_priority(-1), m_condvar_tree(nullptr), m_condvar_key(0), m_parent(nullptr), m_initialized(false) { /* ... */ }
 
             Result Initialize(KThreadFunction func, uintptr_t arg, void *kern_stack_top, KProcessAddress user_stack_top, s32 prio, s32 virt_core, KProcess *owner, ThreadType type);
         private:
@@ -349,15 +361,15 @@ namespace ams::kern {
             #endif
 
             ALWAYS_INLINE void RegisterDpc(DpcFlag flag) {
-                this->GetStackParameters().dpc_flags.fetch_or(flag);
+                this->GetStackParameters().dpc_flags |= flag;
             }
 
             ALWAYS_INLINE void ClearDpc(DpcFlag flag) {
-                this->GetStackParameters().dpc_flags.fetch_and(~flag);
+                this->GetStackParameters().dpc_flags &= ~flag;
             }
 
             ALWAYS_INLINE u8 GetDpc() const {
-                return this->GetStackParameters().dpc_flags.load();
+                return this->GetStackParameters().dpc_flags.Load();
             }
 
             ALWAYS_INLINE bool HasDpc() const {
@@ -517,7 +529,7 @@ namespace ams::kern {
                 m_closed_object = object;
 
                 /* Schedule destruction DPC. */
-                if ((this->GetStackParameters().dpc_flags.load(std::memory_order_relaxed) & DpcFlag_PerformDestruction) == 0) {
+                if ((this->GetStackParameters().dpc_flags.Load<std::memory_order_relaxed>() & DpcFlag_PerformDestruction) == 0) {
                     this->RegisterDpc(DpcFlag_PerformDestruction);
                 }
             }
@@ -550,7 +562,7 @@ namespace ams::kern {
                 MESOSPHERE_UNUSED(core_id);
             }
 
-            s64 GetCpuTime() const { return m_cpu_time.load(); }
+            s64 GetCpuTime() const { return m_cpu_time.Load(); }
 
             s64 GetCpuTime(s32 core_id) const {
                 MESOSPHERE_ABORT_UNLESS(0 <= core_id && core_id < static_cast<s32>(cpu::NumCores));
@@ -592,23 +604,25 @@ namespace ams::kern {
             ALWAYS_INLINE void *GetKernelStackTop() const { return m_kernel_stack_top; }
 
             ALWAYS_INLINE bool IsTerminationRequested() const {
-                return m_termination_requested.load() || this->GetRawState() == ThreadState_Terminated;
+                return m_termination_requested.Load() || this->GetRawState() == ThreadState_Terminated;
             }
 
             size_t GetKernelStackUsage() const;
         public:
             /* Overridden parent functions. */
-            virtual u64 GetId() const override final { return this->GetThreadId(); }
+            ALWAYS_INLINE u64 GetIdImpl() const { return this->GetThreadId(); }
+            ALWAYS_INLINE u64 GetId() const { return this->GetIdImpl(); }
 
-            virtual bool IsInitialized() const override { return m_initialized; }
-            virtual uintptr_t GetPostDestroyArgument() const override { return reinterpret_cast<uintptr_t>(m_parent) | (m_resource_limit_release_hint ? 1 : 0); }
+            bool IsInitialized() const { return m_initialized; }
+            uintptr_t GetPostDestroyArgument() const { return reinterpret_cast<uintptr_t>(m_parent) | (m_resource_limit_release_hint ? 1 : 0); }
 
             static void PostDestroy(uintptr_t arg);
 
-            virtual void Finalize() override;
+            void Finalize();
+
             virtual bool IsSignaled() const override;
-            virtual void OnTimer() override;
-            virtual void DoWorkerTask() override;
+            void OnTimer();
+            void DoWorkerTaskImpl();
         public:
             static constexpr bool IsConditionVariableThreadTreeValid() {
                 return ConditionVariableThreadTreeTraits::IsValid();
@@ -651,14 +665,8 @@ namespace ams::kern {
         return GetCurrentThread().GetCurrentCore();
     }
 
-    ALWAYS_INLINE void KAutoObject::ScheduleDestruction() {
-        MESOSPHERE_ASSERT_THIS();
-
-        /* Set our object to destroy. */
-        m_next_closed_object = GetCurrentThread().GetClosedObject();
-
-        /* Set ourselves as the thread's next object to destroy. */
-        GetCurrentThread().SetClosedObject(this);
+    ALWAYS_INLINE void KTimerTask::OnTimer() {
+        static_cast<KThread *>(this)->OnTimer();
     }
 
 }

libraries/libmesosphere/include/mesosphere/kern_k_thread_local_page.hpp

@@ -33,19 +33,25 @@ namespace ams::kern {
             KProcess *m_owner;
             bool m_is_region_free[RegionsPerPage];
         public:
-            constexpr explicit KThreadLocalPage(KProcessAddress addr) : m_virt_addr(addr), m_owner(nullptr), m_is_region_free() {
-                for (size_t i = 0; i < RegionsPerPage; i++) {
+            explicit KThreadLocalPage(KProcessAddress addr) : m_virt_addr(addr), m_owner(nullptr) {
+                for (size_t i = 0; i < util::size(m_is_region_free); i++) {
                     m_is_region_free[i] = true;
                 }
             }
 
-            constexpr explicit KThreadLocalPage() : KThreadLocalPage(Null<KProcessAddress>) { /* ... */ }
+            explicit KThreadLocalPage() : KThreadLocalPage(Null<KProcessAddress>) { /* ... */ }
 
             constexpr ALWAYS_INLINE KProcessAddress GetAddress() const { return m_virt_addr; }
+        public:
+            using RedBlackKeyType = KProcessAddress;
 
-            static constexpr ALWAYS_INLINE int Compare(const KThreadLocalPage &lhs, const KThreadLocalPage &rhs) {
-                const KProcessAddress lval = lhs.GetAddress();
-                const KProcessAddress rval = rhs.GetAddress();
+            static constexpr ALWAYS_INLINE RedBlackKeyType GetRedBlackKey(const RedBlackKeyType  &v) { return v; }
+            static constexpr ALWAYS_INLINE RedBlackKeyType GetRedBlackKey(const KThreadLocalPage &v) { return v.GetAddress(); }
+
+            template<typename T> requires (std::same_as<T, KThreadLocalPage> || std::same_as<T, RedBlackKeyType>)
+            static constexpr ALWAYS_INLINE int Compare(const T &lhs, const KThreadLocalPage &rhs) {
+                const KProcessAddress lval = GetRedBlackKey(lhs);
+                const KProcessAddress rval = GetRedBlackKey(rhs);
 
                 if (lval < rval) {
                     return -1;
@@ -107,8 +113,8 @@ namespace ams::kern {
 
     /* Miscellaneous sanity checking. */
     static_assert(ams::svc::ThreadLocalRegionSize == THREAD_LOCAL_REGION_SIZE);
-    static_assert(__builtin_offsetof(ams::svc::ThreadLocalRegion, message_buffer) == THREAD_LOCAL_REGION_MESSAGE_BUFFER);
-    static_assert(__builtin_offsetof(ams::svc::ThreadLocalRegion, disable_count)  == THREAD_LOCAL_REGION_DISABLE_COUNT);
-    static_assert(__builtin_offsetof(ams::svc::ThreadLocalRegion, interrupt_flag) == THREAD_LOCAL_REGION_INTERRUPT_FLAG);
+    static_assert(AMS_OFFSETOF(ams::svc::ThreadLocalRegion, message_buffer) == THREAD_LOCAL_REGION_MESSAGE_BUFFER);
+    static_assert(AMS_OFFSETOF(ams::svc::ThreadLocalRegion, disable_count)  == THREAD_LOCAL_REGION_DISABLE_COUNT);
+    static_assert(AMS_OFFSETOF(ams::svc::ThreadLocalRegion, interrupt_flag) == THREAD_LOCAL_REGION_INTERRUPT_FLAG);
 
 }

libraries/libmesosphere/include/mesosphere/kern_k_timer_task.hpp

@@ -30,7 +30,8 @@ namespace ams::kern {
                 }
             }
         public:
-            constexpr ALWAYS_INLINE KTimerTask() : m_time(0) { /* ... */ }
+            constexpr explicit ALWAYS_INLINE KTimerTask(util::ConstantInitializeTag) : util::IntrusiveRedBlackTreeBaseNode<KTimerTask>(util::ConstantInitialize), m_time(0) { /* ... */ }
+            explicit ALWAYS_INLINE KTimerTask() : m_time(0) { /* ... */ }
 
             constexpr ALWAYS_INLINE void SetTime(s64 t) {
                 m_time = t;
@@ -40,8 +41,9 @@ namespace ams::kern {
                 return m_time;
             }
 
-            virtual void OnTimer() = 0;
-
+            /* NOTE: This is virtual in Nintendo's kernel. Prior to 13.0.0, KWaitObject was also a TimerTask; this is no longer the case. */
+            /* Since this is now KThread exclusive, we have devirtualized (see inline declaration for this inside kern_kthread.hpp). */
+            void OnTimer();
     };
 
 }

libraries/libmesosphere/include/mesosphere/kern_k_transfer_memory.hpp

@@ -36,10 +36,10 @@ namespace ams::kern {
             }
 
             Result Initialize(KProcessAddress addr, size_t size, ams::svc::MemoryPermission own_perm);
-            virtual void Finalize() override;
+            void Finalize();
 
-            virtual bool IsInitialized() const override { return m_is_initialized; }
-            virtual uintptr_t GetPostDestroyArgument() const override { return reinterpret_cast<uintptr_t>(m_owner); }
+            bool IsInitialized() const { return m_is_initialized; }
+            uintptr_t GetPostDestroyArgument() const { return reinterpret_cast<uintptr_t>(m_owner); }
             static void PostDestroy(uintptr_t arg);
 
             Result Map(KProcessAddress address, size_t size, ams::svc::MemoryPermission map_perm);

libraries/libmesosphere/include/mesosphere/kern_k_typed_address.hpp

@@ -26,7 +26,7 @@ namespace ams::kern {
             uintptr_t m_address;
         public:
             /* Constructors. */
-            constexpr ALWAYS_INLINE KTypedAddress() : m_address(0) { /* ... */ }
+            ALWAYS_INLINE KTypedAddress() { /* ... */ }
             constexpr ALWAYS_INLINE KTypedAddress(uintptr_t a) : m_address(a) { /* ... */ }
             template<typename U>
             constexpr ALWAYS_INLINE explicit KTypedAddress(U *ptr) : m_address(reinterpret_cast<uintptr_t>(ptr)) { /* ... */ }
@@ -146,13 +146,13 @@ namespace ams::kern {
     }
 
     template<typename T, typename U>
-    constexpr ALWAYS_INLINE T *GetPointer(KTypedAddress<true, U> address) {
-        return CONST_FOLD(reinterpret_cast<T *>(address.GetValue()));
+    ALWAYS_INLINE T *GetPointer(KTypedAddress<true, U> address) {
+        return reinterpret_cast<T *>(address.GetValue());
     }
 
     template<typename T>
-    constexpr ALWAYS_INLINE void *GetVoidPointer(KTypedAddress<true, T> address) {
-        return CONST_FOLD(reinterpret_cast<void *>(address.GetValue()));
+    ALWAYS_INLINE void *GetVoidPointer(KTypedAddress<true, T> address) {
+        return reinterpret_cast<void *>(address.GetValue());
     }
 
 #else
@@ -170,12 +170,12 @@ namespace ams::kern {
 
     template<typename T>
     constexpr ALWAYS_INLINE T *GetPointer(uintptr_t address) {
-        return CONST_FOLD(reinterpret_cast<T *>(address));
+        return reinterpret_cast<T *>(address);
     }
 
     template<typename T>
     constexpr ALWAYS_INLINE void *GetVoidPointer(uintptr_t address) {
-        return CONST_FOLD(reinterpret_cast<void *>(address));
+        return reinterpret_cast<void *>(address);
     }
 
 #endif

libraries/libmesosphere/include/mesosphere/kern_k_worker_task.hpp

@@ -18,16 +18,23 @@
 
 namespace ams::kern {
 
-    class KWorkerTask {
+    /* NOTE: We inherit KWorkerTask from KSynchronizationObject in order to devirtualize DoWorkerTask, which is exclusive to KThread/KProcess. */
+    /* This should be reverted, if Nintendo adds new types of worker tasks in the future. */
+
+    /* Nintendo has class KWorkerTask { ... } with identical interface but DoWorkerTask() virtual. */
+
+    class KWorkerTask : public KSynchronizationObject {
         private:
             KWorkerTask *m_next_task;
         public:
-            constexpr ALWAYS_INLINE KWorkerTask() : m_next_task(nullptr) { /* ... */ }
+            constexpr ALWAYS_INLINE explicit KWorkerTask(util::ConstantInitializeTag) : KSynchronizationObject(util::ConstantInitialize), m_next_task(nullptr) { /* ... */  }
+
+            ALWAYS_INLINE explicit KWorkerTask() : m_next_task(nullptr) { /* ... */  }
 
             constexpr ALWAYS_INLINE KWorkerTask *GetNextTask() const { return m_next_task; }
             constexpr ALWAYS_INLINE void SetNextTask(KWorkerTask *task) { m_next_task = task; }
 
-            virtual void DoWorkerTask() = 0;
+            void DoWorkerTask();
     };
 
 }

libraries/libmesosphere/include/mesosphere/kern_select_cpu.hpp

@@ -39,6 +39,16 @@
 
     }
 
+#elif defined(ATMOSPHERE_BOARD_QEMU_VIRT)
+
+    #include <mesosphere/board/qemu/virt/kern_cpu_map.hpp>
+
+    namespace ams::kern::cpu {
+
+        using namespace ams::kern::board::qemu::virt::impl::cpu;
+
+    }
+
 #else
     #error "Unknown board for CPU Map"
 #endif

libraries/libmesosphere/include/mesosphere/kern_select_system_control.hpp

@@ -15,6 +15,7 @@
  */
 #pragma once
 #include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_k_system_control_base.hpp>
 
 #ifdef ATMOSPHERE_BOARD_NINTENDO_NX
     #include <mesosphere/board/nintendo/nx/kern_k_system_control.hpp>
@@ -23,6 +24,28 @@
         using ams::kern::board::nintendo::nx::KSystemControl;
     }
 
+#elif defined(ATMOSPHERE_BOARD_QEMU_VIRT)
+    #include <mesosphere/board/qemu/virt/kern_k_system_control.hpp>
+
+    namespace ams::kern {
+        using ams::kern::board::qemu::virt::KSystemControl;
+    }
+
 #else
     #error "Unknown board for KSystemControl"
 #endif
+
+namespace ams::kern {
+
+    ALWAYS_INLINE u32 KSystemControlBase::ReadRegisterPrivileged(ams::svc::PhysicalAddress address) {
+        u32 v;
+        KSystemControl::ReadWriteRegisterPrivileged(std::addressof(v), address, 0x00000000u, 0);
+        return v;
+    }
+
+    ALWAYS_INLINE void KSystemControlBase::WriteRegisterPrivileged(ams::svc::PhysicalAddress address, u32 value) {
+        u32 v;
+        KSystemControl::ReadWriteRegisterPrivileged(std::addressof(v), address, 0xFFFFFFFFu, value);
+    }
+
+}

libraries/libmesosphere/include/mesosphere/kern_slab_helpers.hpp

@@ -64,12 +64,11 @@ namespace ams::kern {
             static size_t GetNumRemaining() { return s_slab_heap.GetNumRemaining(); }
     };
 
-    template<typename Derived, typename Base, bool SupportDynamicExpansion = false>
+    template<typename Derived, typename Base, bool SupportDynamicExpansion = false> requires std::derived_from<Base, KAutoObjectWithList>
     class KAutoObjectWithSlabHeapAndContainer : public Base {
-        static_assert(std::is_base_of<KAutoObjectWithList, Base>::value);
         private:
             static constinit inline KSlabHeap<Derived, SupportDynamicExpansion> s_slab_heap;
-            static constinit inline KAutoObjectWithListContainer s_container;
+            static constinit inline KAutoObjectWithListContainer<Derived> s_container;
         private:
             static ALWAYS_INLINE Derived *Allocate() {
                 return s_slab_heap.Allocate();
@@ -79,30 +78,48 @@ namespace ams::kern {
                 s_slab_heap.Free(obj);
             }
         public:
-            class ListAccessor : public KAutoObjectWithListContainer::ListAccessor {
+            class ListAccessor : public KAutoObjectWithListContainer<Derived>::ListAccessor {
                 public:
-                    ALWAYS_INLINE ListAccessor() : KAutoObjectWithListContainer::ListAccessor(s_container) { /* ... */ }
+                    ALWAYS_INLINE ListAccessor() : KAutoObjectWithListContainer<Derived>::ListAccessor(s_container) { /* ... */ }
                     ALWAYS_INLINE ~ListAccessor() { /* ... */ }
             };
-        public:
-            constexpr KAutoObjectWithSlabHeapAndContainer() = default;
-
-            virtual void Destroy() override {
-                const bool is_initialized = this->IsInitialized();
-                uintptr_t arg = 0;
-                if (is_initialized) {
-                    s_container.Unregister(this);
-                    arg = this->GetPostDestroyArgument();
-                    this->Finalize();
-                }
-                Free(static_cast<Derived *>(this));
-                if (is_initialized) {
-                    Derived::PostDestroy(arg);
+        private:
+            static ALWAYS_INLINE bool IsInitialized(const Derived *obj) {
+                if constexpr (requires { { obj->IsInitialized() } -> std::same_as<bool>; }) {
+                    return obj->IsInitialized();
+                } else {
+                    return true;
                 }
             }
 
-            virtual bool IsInitialized() const { return true; }
-            virtual uintptr_t GetPostDestroyArgument() const { return 0; }
+            static ALWAYS_INLINE uintptr_t GetPostDestroyArgument(const Derived *obj) {
+                if constexpr (requires { { obj->GetPostDestroyArgument() } -> std::same_as<uintptr_t>; }) {
+                    return obj->GetPostDestroyArgument();
+                } else {
+                    return 0;
+                }
+            }
+        public:
+            constexpr explicit KAutoObjectWithSlabHeapAndContainer(util::ConstantInitializeTag) : Base(util::ConstantInitialize) { /* ... */ }
+
+            explicit KAutoObjectWithSlabHeapAndContainer() { /* ... */ }
+
+            /* NOTE: IsInitialized() and GetPostDestroyArgument() are virtual functions declared in this class, */
+            /* in Nintendo's kernel. We fully devirtualize them, as Destroy() is the only user of them. */
+            /* We also devirtualize KAutoObject::Finalize(), which is only used by this function in Nintendo's kernel. */
+            virtual void Destroy() override final {
+                Derived * const derived = static_cast<Derived *>(this);
+
+                if (IsInitialized(derived)) {
+                    s_container.Unregister(derived);
+                    const uintptr_t arg = GetPostDestroyArgument(derived);
+                    derived->Finalize();
+                    Free(derived);
+                    Derived::PostDestroy(arg);
+                } else {
+                    Free(derived);
+                }
+            }
 
             size_t GetSlabIndex() const {
                 return s_slab_heap.GetObjectIndex(static_cast<const Derived *>(this));
@@ -116,7 +133,7 @@ namespace ams::kern {
             static Derived *Create() {
                 Derived *obj = Allocate();
                 if (AMS_LIKELY(obj != nullptr)) {
-                    KAutoObject::Create(obj);
+                    KAutoObject::Create<Derived>(obj);
                 }
                 return obj;
             }
@@ -128,7 +145,7 @@ namespace ams::kern {
                 Derived * const obj = GetPointer<Derived>(AllocateUnusedSlabMemory(sizeof(Derived), alignof(Derived)));
                 if (AMS_LIKELY(obj != nullptr)) {
                     std::construct_at(obj);
-                    KAutoObject::Create(obj);
+                    KAutoObject::Create<Derived>(obj);
                 }
                 return obj;
             }

libraries/libmesosphere/source/arch/arm64/kern_cpu.cpp

@@ -96,7 +96,7 @@ namespace ams::kern::arch::arm64::cpu {
                 KLightLock m_lock;
                 KLightLock m_cv_lock;
                 KLightConditionVariable m_cv;
-                std::atomic<u64> m_target_cores;
+                util::Atomic<u64> m_target_cores;
                 volatile Operation m_operation;
             private:
                 static void ThreadFunction(uintptr_t _this) {
@@ -104,12 +104,12 @@ namespace ams::kern::arch::arm64::cpu {
                 }
 
                 void ThreadFunctionImpl() {
-                    const s32 core_id = GetCurrentCoreId();
+                    const u64 core_mask = (1ul << GetCurrentCoreId());
                     while (true) {
                         /* Wait for a request to come in. */
                         {
                             KScopedLightLock lk(m_cv_lock);
-                            while ((m_target_cores.load() & (1ul << core_id)) == 0) {
+                            while ((m_target_cores.Load() & core_mask) == 0) {
                                 m_cv.Wait(std::addressof(m_cv_lock));
                             }
                         }
@@ -120,7 +120,9 @@ namespace ams::kern::arch::arm64::cpu {
                         /* Broadcast, if there's nothing pending. */
                         {
                             KScopedLightLock lk(m_cv_lock);
-                            if (m_target_cores.load() == 0) {
+
+                            m_target_cores &= ~core_mask;
+                            if (m_target_cores.Load() == 0) {
                                 m_cv.Broadcast();
                             }
                         }
@@ -129,7 +131,7 @@ namespace ams::kern::arch::arm64::cpu {
 
                 void ProcessOperation();
             public:
-                constexpr KCacheHelperInterruptHandler() : KInterruptHandler(), m_lock(), m_cv_lock(), m_cv(), m_target_cores(), m_operation(Operation::Idle) { /* ... */ }
+                constexpr KCacheHelperInterruptHandler() : KInterruptHandler(), m_lock(), m_cv_lock(), m_cv(util::ConstantInitialize), m_target_cores(0), m_operation(Operation::Idle) { /* ... */ }
 
                 void Initialize(s32 core_id) {
                     /* Reserve a thread from the system limit. */
@@ -150,6 +152,7 @@ namespace ams::kern::arch::arm64::cpu {
                 virtual KInterruptTask *OnInterrupt(s32 interrupt_id) override {
                     MESOSPHERE_UNUSED(interrupt_id);
                     this->ProcessOperation();
+                    m_target_cores &= ~(1ul << GetCurrentCoreId());
                     return nullptr;
                 }
 
@@ -163,7 +166,7 @@ namespace ams::kern::arch::arm64::cpu {
                     if ((op == Operation::InstructionMemoryBarrier) || (Kernel::GetState() == Kernel::State::Initializing)) {
                         /* Check that there's no on-going operation. */
                         MESOSPHERE_ABORT_UNLESS(m_operation == Operation::Idle);
-                        MESOSPHERE_ABORT_UNLESS(m_target_cores.load() == 0);
+                        MESOSPHERE_ABORT_UNLESS(m_target_cores.Load() == 0);
 
                         /* Set operation. */
                         m_operation = op;
@@ -171,13 +174,13 @@ namespace ams::kern::arch::arm64::cpu {
                         /* For certain operations, we want to send an interrupt. */
                         m_target_cores = other_cores_mask;
 
-                        const u64 target_mask = m_target_cores.load();
+                        const u64 target_mask = m_target_cores.Load();
 
                         DataSynchronizationBarrier();
                         Kernel::GetInterruptManager().SendInterProcessorInterrupt(KInterruptName_CacheOperation, target_mask);
 
                         this->ProcessOperation();
-                        while (m_target_cores.load() != 0) {
+                        while (m_target_cores.Load() != 0) {
                             cpu::Yield();
                         }
 
@@ -189,7 +192,7 @@ namespace ams::kern::arch::arm64::cpu {
 
                         /* Check that there's no on-going operation. */
                         MESOSPHERE_ABORT_UNLESS(m_operation == Operation::Idle);
-                        MESOSPHERE_ABORT_UNLESS(m_target_cores.load() == 0);
+                        MESOSPHERE_ABORT_UNLESS(m_target_cores.Load() == 0);
 
                         /* Set operation. */
                         m_operation = op;
@@ -199,7 +202,7 @@ namespace ams::kern::arch::arm64::cpu {
 
                         /* Use the condvar. */
                         m_cv.Broadcast();
-                        while (m_target_cores.load() != 0) {
+                        while (m_target_cores.Load() != 0) {
                             m_cv.Wait(std::addressof(m_cv_lock));
                         }
 
@@ -286,8 +289,6 @@ namespace ams::kern::arch::arm64::cpu {
                     FlushDataCacheBySetWay(0);
                     break;
             }
-
-            m_target_cores &= ~(1ul << GetCurrentCoreId());
         }
 
         ALWAYS_INLINE void SetEventLocally() {
@@ -383,11 +384,13 @@ namespace ams::kern::arch::arm64::cpu {
             /* Store cache from L1 up to (level of coherence - 1). */
             for (int level = 0; level < levels_of_coherency - 1; ++level) {
                 PerformCacheOperationBySetWayImpl<true>(level, StoreDataCacheLineBySetWayImpl);
+                DataSynchronizationBarrier();
             }
 
             /* Flush cache from (level of coherence - 1) down to L0. */
             for (int level = levels_of_coherency; level > 0; --level) {
                 PerformCacheOperationBySetWayImpl<true>(level - 1, FlushDataCacheLineBySetWayImpl);
+                DataSynchronizationBarrier();
             }
         }
 

libraries/libmesosphere/source/arch/arm64/kern_exception_handlers.cpp

@@ -126,7 +126,7 @@ namespace ams::kern::arch::arm64 {
                     const bool is_aarch64 = (context->psr & 0x10) == 0;
                     if (is_aarch64) {
                         /* 64-bit. */
-                        ams::svc::aarch64::ExceptionInfo *info = std::addressof(GetPointer<ams::svc::aarch64::ProcessLocalRegion>(cur_process.GetProcessLocalRegionAddress())->exception_info);
+                        ams::svc::aarch64::ExceptionInfo *info = std::addressof(static_cast<ams::svc::aarch64::ProcessLocalRegion *>(cur_process.GetProcessLocalRegionHeapAddress())->exception_info);
 
                         for (size_t i = 0; i < util::size(info->r); ++i) {
                             info->r[i] = context->x[i];
@@ -141,7 +141,7 @@ namespace ams::kern::arch::arm64 {
                         info->far    = far;
                     } else {
                         /* 32-bit. */
-                        ams::svc::aarch32::ExceptionInfo *info = std::addressof(GetPointer<ams::svc::aarch32::ProcessLocalRegion>(cur_process.GetProcessLocalRegionAddress())->exception_info);
+                        ams::svc::aarch32::ExceptionInfo *info = std::addressof(static_cast<ams::svc::aarch32::ProcessLocalRegion *>(cur_process.GetProcessLocalRegionHeapAddress())->exception_info);
 
                         for (size_t i = 0; i < util::size(info->r); ++i) {
                             info->r[i] = context->x[i];
@@ -201,17 +201,17 @@ namespace ams::kern::arch::arm64 {
                     context->pc   = GetInteger(cur_process.GetEntryPoint());
                     context->x[0] = type;
                     if (is_aarch64) {
-                        context->x[1] = GetInteger(cur_process.GetProcessLocalRegionAddress() + __builtin_offsetof(ams::svc::aarch64::ProcessLocalRegion, exception_info));
+                        context->x[1]   = GetInteger(cur_process.GetProcessLocalRegionAddress() + AMS_OFFSETOF(ams::svc::aarch64::ProcessLocalRegion, exception_info));
 
-                        auto *plr    = GetPointer<ams::svc::aarch64::ProcessLocalRegion>(cur_process.GetProcessLocalRegionAddress());
-                        context->sp  = util::AlignDown(reinterpret_cast<uintptr_t>(plr->data) + sizeof(plr->data), 0x10);
-                        context->psr = 0;
+                        const auto *plr = GetPointer<ams::svc::aarch64::ProcessLocalRegion>(cur_process.GetProcessLocalRegionAddress());
+                        context->sp     = util::AlignDown(reinterpret_cast<uintptr_t>(plr->data) + sizeof(plr->data), 0x10);
+                        context->psr    = 0;
                     } else {
-                        context->x[1] = GetInteger(cur_process.GetProcessLocalRegionAddress() + __builtin_offsetof(ams::svc::aarch32::ProcessLocalRegion, exception_info));
+                        context->x[1]   = GetInteger(cur_process.GetProcessLocalRegionAddress() + AMS_OFFSETOF(ams::svc::aarch32::ProcessLocalRegion, exception_info));
 
-                        auto *plr      = GetPointer<ams::svc::aarch32::ProcessLocalRegion>(cur_process.GetProcessLocalRegionAddress());
-                        context->x[13] = util::AlignDown(reinterpret_cast<uintptr_t>(plr->data) + sizeof(plr->data), 0x10);
-                        context->psr   = 0x10;
+                        const auto *plr = GetPointer<ams::svc::aarch32::ProcessLocalRegion>(cur_process.GetProcessLocalRegionAddress());
+                        context->x[13]  = util::AlignDown(reinterpret_cast<uintptr_t>(plr->data) + sizeof(plr->data), 0x08);
+                        context->psr    = 0x10;
                     }
 
                     /* Set exception SVC permissions. */
@@ -380,10 +380,10 @@ namespace ams::kern::arch::arm64 {
         const bool is_aarch64 = (e_ctx->psr & 0x10) == 0;
         if (is_aarch64) {
             /* We're 64-bit. */
-            info.info64 = GetPointer<ams::svc::aarch64::ProcessLocalRegion>(cur_process.GetProcessLocalRegionAddress())->exception_info;
+            info.info64 = static_cast<const ams::svc::aarch64::ProcessLocalRegion *>(cur_process.GetProcessLocalRegionHeapAddress())->exception_info;
         } else {
             /* We're 32-bit. */
-            info.info32 = GetPointer<ams::svc::aarch32::ProcessLocalRegion>(cur_process.GetProcessLocalRegionAddress())->exception_info;
+            info.info32 = static_cast<const ams::svc::aarch32::ProcessLocalRegion *>(cur_process.GetProcessLocalRegionHeapAddress())->exception_info;
         }
 
         /* Try to leave the user exception. */

libraries/libmesosphere/source/arch/arm64/kern_k_debug.cpp

@@ -283,9 +283,6 @@ namespace ams::kern::arch::arm64 {
             /* Check that the name is a valid instruction breakpoint. */
             R_UNLESS((name - ams::svc::HardwareBreakPointRegisterName_I0) <= num_bp, svc::ResultNotSupported());
 
-            /* We may be getting the process, so prepare a scoped reference holder. */
-            KScopedAutoObject<KProcess> process;
-
             /* Configure flags/value. */
             if ((flags & 1) != 0) {
                 /* We're enabling the breakpoint. Check that the flags are allowable. */

libraries/libmesosphere/source/arch/arm64/kern_k_page_table.cpp

@@ -233,7 +233,7 @@ namespace ams::kern::arch::arm64 {
 
                 /* Begin the traversal. */
                 TraversalContext context;
-                TraversalEntry   cur_entry  = {};
+                TraversalEntry   cur_entry  = { .phys_addr = Null<KPhysicalAddress>, .block_size = 0, .sw_reserved_bits = 0 };
                 bool             cur_valid  = false;
                 TraversalEntry   next_entry;
                 bool             next_valid;

libraries/libmesosphere/source/board/nintendo/nx/kern_k_device_page_table.cpp

@@ -345,8 +345,8 @@ namespace ams::kern::board::nintendo::nx {
         /* Globals. */
         constinit KLightLock g_lock;
         constinit u8 g_reserved_asid;
-        constinit KPhysicalAddress   g_memory_controller_address;
-        constinit KPhysicalAddress   g_reserved_table_phys_addr;
+        constinit KPhysicalAddress   g_memory_controller_address{Null<KPhysicalAddress>};
+        constinit KPhysicalAddress   g_reserved_table_phys_addr{Null<KPhysicalAddress>};
         constinit KDeviceAsidManager g_asid_manager;
         constinit u32 g_saved_page_tables[AsidCount];
         constinit u32 g_saved_asid_registers[ams::svc::DeviceName_Count];
@@ -1130,7 +1130,7 @@ namespace ams::kern::board::nintendo::nx {
             size_t cur_size;
             {
                 /* Get the current contiguous range. */
-                KPageTableBase::MemoryRange contig_range = {};
+                KPageTableBase::MemoryRange contig_range = { .address = Null<KPhysicalAddress>, .size = 0 };
                 R_TRY(page_table->OpenMemoryRangeForMapDeviceAddressSpace(std::addressof(contig_range), process_address + mapped_size, size - mapped_size, ConvertToKMemoryPermission(device_perm), is_aligned));
 
                 /* Ensure we close the range when we're done. */
@@ -1288,7 +1288,7 @@ namespace ams::kern::board::nintendo::nx {
         MESOSPHERE_ASSERT(((device_address + size - 1) & ~DeviceVirtualAddressMask) == 0);
 
         /* We need to traverse the ranges that make up our mapping, to make sure they're all good. Start by getting a contiguous range. */
-        KPageTableBase::MemoryRange contig_range = {};
+        KPageTableBase::MemoryRange contig_range = { .address = Null<KPhysicalAddress>, .size = 0 };
         if (R_FAILED(page_table->OpenMemoryRangeForUnmapDeviceAddressSpace(std::addressof(contig_range), process_address, size))) {
             return false;
         }

libraries/libmesosphere/source/board/nintendo/nx/kern_k_sleep_manager.cpp

@@ -66,14 +66,14 @@ namespace ams::kern::board::nintendo::nx {
         constinit u64 g_sleep_target_cores;
         constinit KLightLock g_request_lock;
         constinit KLightLock g_cv_lock;
-        constinit KLightConditionVariable g_cv;
+        constinit KLightConditionVariable g_cv{util::ConstantInitialize};
         constinit KPhysicalAddress g_sleep_buffer_phys_addrs[cpu::NumCores];
         alignas(1_KB) constinit u64 g_sleep_buffers[cpu::NumCores][1_KB / sizeof(u64)];
         constinit SavedSystemRegisters g_sleep_system_registers[cpu::NumCores] = {};
 
         void PowerOnCpu(int core_id, KPhysicalAddress entry_phys_addr, u64 context_id) {
             /* Request the secure monitor power on the core. */
-            smc::CpuOn(cpu::MultiprocessorAffinityRegisterAccessor().GetCpuOnArgument() | core_id, GetInteger(entry_phys_addr), context_id);
+            ::ams::kern::arch::arm64::smc::CpuOn<smc::SmcId_Supervisor, true>(cpu::MultiprocessorAffinityRegisterAccessor().GetCpuOnArgument() | core_id, GetInteger(entry_phys_addr), context_id);
         }
 
         void WaitOtherCpuPowerOff() {