OmniNX/Horizon-OC
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revert "hoc-clk: add live vdd2, live boost clock and basic pwm dimming"

Browse Source 
This reverts commit 15b7df8ef1.
This commit is contained in:
souldbminersmwc
2025-11-09 16:14:52 -05:00
parent 22ec140738
commit 21a3f953d7
3804 changed files with 435 additions and 570162 deletions
Download Patch File Download Diff File Expand all files Collapse all files

164
Source/Atmosphere-MTC-Unlock/mesosphere/kernel/source/arch/arm64/exception_vectors.s
View File

@@ -1,164 +0,0 @@
/*
* Copyright (c) Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
/* Some macros taken from https://github.com/ARM-software/arm-trusted-firmware/blob/master/include/common/aarch64/asm_macros.S */
/*
* Copyright (c) 2013-2017, ARM Limited and Contributors. All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
/*
* Declare the exception vector table, enforcing it is aligned on a
* 2KB boundary, as required by the ARMv8 architecture.
* Use zero bytes as the fill value to be stored in the padding bytes
* so that it inserts illegal AArch64 instructions. This increases
* security, robustness and potentially facilitates debugging.
*/
.macro vector_base label, section_name=.vectors
.section \section_name, "ax"
.align 11, 0
\label:
.endm
/*
* Create an entry in the exception vector table, enforcing it is
* aligned on a 128-byte boundary, as required by the ARMv8 architecture.
* Use zero bytes as the fill value to be stored in the padding bytes
* so that it inserts illegal AArch64 instructions. This increases
* security, robustness and potentially facilitates debugging.
*/
.macro vector_entry label, section_name=.vectors
.cfi_sections .debug_frame
.section \section_name, "ax"
.align 7, 0
.type \label, %function
.func \label
.cfi_startproc
\label:
.endm
/*
* This macro verifies that the given vector doesnt exceed the
* architectural limit of 32 instructions. This is meant to be placed
* immediately after the last instruction in the vector. It takes the
* vector entry as the parameter
*/
.macro check_vector_size since
.endfunc
.cfi_endproc
.if (. - \since) > (32 * 4)
.error "Vector exceeds 32 instructions"
.endif
.endm
/* Actual Vectors for Kernel. */
.global _ZN3ams4kern16ExceptionVectorsEv
vector_base _ZN3ams4kern16ExceptionVectorsEv
/* Current EL, SP0 */
vector_entry synch_sp0
/* Just infinite loop. */
clrex
nop
b synch_sp0
check_vector_size synch_sp0
vector_entry irq_sp0
clrex
nop
b irq_sp0
check_vector_size irq_sp0
vector_entry fiq_sp0
clrex
nop
b fiq_sp0
check_vector_size fiq_sp0
vector_entry serror_sp0
clrex
nop
b serror_sp0
check_vector_size serror_sp0
/* Current EL, SPx */
vector_entry synch_spx
clrex
b _ZN3ams4kern4arch5arm6430EL1SynchronousExceptionHandlerEv
check_vector_size synch_spx
vector_entry irq_spx
b _ZN3ams4kern4arch5arm6422EL1IrqExceptionHandlerEv
check_vector_size irq_spx
vector_entry fiq_spx
clrex
nop
b fiq_spx
check_vector_size fiq_spx
vector_entry serror_spx
clrex
nop
b _ZN3ams4kern4arch5arm6421EL1SystemErrorHandlerEv
check_vector_size serror_spx
/* Lower EL, A64 */
vector_entry synch_a64
clrex
b _ZN3ams4kern4arch5arm6430EL0SynchronousExceptionHandlerEv
check_vector_size synch_a64
vector_entry irq_a64
clrex
b _ZN3ams4kern4arch5arm6425EL0A64IrqExceptionHandlerEv
check_vector_size irq_a64
vector_entry fiq_a64
clrex
nop
b fiq_a64
check_vector_size fiq_a64
vector_entry serror_a64
clrex
nop
b _ZN3ams4kern4arch5arm6421EL0SystemErrorHandlerEv
check_vector_size serror_a64
/* Lower EL, A32 */
vector_entry synch_a32
clrex
b _ZN3ams4kern4arch5arm6430EL0SynchronousExceptionHandlerEv
check_vector_size synch_a32
vector_entry irq_a32
clrex
b _ZN3ams4kern4arch5arm6425EL0A32IrqExceptionHandlerEv
check_vector_size irq_a32
vector_entry fiq_a32
clrex
nop
b fiq_a32
check_vector_size fiq_a32
vector_entry serror_a32
clrex
nop
b _ZN3ams4kern4arch5arm6421EL0SystemErrorHandlerEv
check_vector_size serror_a32

772
Source/Atmosphere-MTC-Unlock/mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp
View File

@@ -1,772 +0,0 @@
/*
* Copyright (c) Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <mesosphere.hpp>
extern "C" void __rodata_start();
extern "C" void __rodata_end();
extern "C" void __bin_start__();
extern "C" void __bin_end__();
namespace ams::kern {
void ExceptionVectors();
}
namespace ams::kern::init {
/* Prototypes for functions declared in ASM that we need to reference. */
void StartOtherCore(const ams::kern::init::KInitArguments *init_args);
void IdentityMappedFunctionAreaBegin();
void IdentityMappedFunctionAreaEnd();
size_t GetMiscUnknownDebugRegionSize();
size_t GetSecureUnknownRegionSize();
void InitializeDebugRegisters();
void InitializeExceptionVectors();
namespace {
/* Global Allocator. */
constinit KInitialPageAllocator g_initial_page_allocator;
constinit KInitArguments g_init_arguments[cpu::NumCores];
/* Globals for passing data between InitializeCorePhase1 and InitializeCorePhase2. */
constinit InitialProcessBinaryLayoutWithSize g_phase2_initial_process_binary_meta{};
constinit KPhysicalAddress g_phase2_resource_end_phys_addr = Null<KPhysicalAddress>;
constinit u64 g_phase2_linear_region_phys_to_virt_diff = 0;
/* Page table attributes. */
constexpr PageTableEntry KernelTextAttribute(PageTableEntry::Permission_KernelRX, PageTableEntry::PageAttribute_NormalMemory, PageTableEntry::Shareable_InnerShareable, PageTableEntry::MappingFlag_Mapped);
constexpr PageTableEntry KernelRoDataAttribute(PageTableEntry::Permission_KernelR, PageTableEntry::PageAttribute_NormalMemory, PageTableEntry::Shareable_InnerShareable, PageTableEntry::MappingFlag_Mapped);
constexpr PageTableEntry KernelRwDataAttribute(PageTableEntry::Permission_KernelRW, PageTableEntry::PageAttribute_NormalMemory, PageTableEntry::Shareable_InnerShareable, PageTableEntry::MappingFlag_Mapped);
constexpr PageTableEntry KernelMmioAttribute(PageTableEntry::Permission_KernelRW, PageTableEntry::PageAttribute_Device_nGnRE, PageTableEntry::Shareable_OuterShareable, PageTableEntry::MappingFlag_Mapped);
constexpr PageTableEntry KernelRwDataUncachedAttribute(PageTableEntry::Permission_KernelRW, PageTableEntry::PageAttribute_NormalMemoryNotCacheable, PageTableEntry::Shareable_InnerShareable, PageTableEntry::MappingFlag_Mapped);
void TurnOnAllCores() {
cpu::MultiprocessorAffinityRegisterAccessor mpidr;
const auto arg = mpidr.GetCpuOnArgument();
const auto current_core = mpidr.GetAff0();
for (s32 i = 0; i < static_cast<s32>(cpu::NumCores); i++) {
if (static_cast<s32>(current_core) != i) {
KSystemControl::Init::TurnOnCpu(arg | i, g_init_arguments + i);
}
}
}
void InvokeMain(u64 core_id) {
/* Clear cpacr_el1. */
cpu::SetCpacrEl1(0);
cpu::InstructionMemoryBarrier();
/* Initialize registers. */
InitializeDebugRegisters();
InitializeExceptionVectors();
/* Set exception stack. */
cpu::SetCntvCvalEl0(GetInteger(KMemoryLayout::GetExceptionStackTopAddress(static_cast<s32>(core_id))) - sizeof(KThread::StackParameters));
/* Call main. */
HorizonKernelMain(static_cast<s32>(core_id));
}
void SetupInitialArguments() {
/* Determine whether we're running on a cortex-a53 or a-57. */
cpu::MainIdRegisterAccessor midr_el1;
const auto implementer = midr_el1.GetImplementer();
const auto primary_part = midr_el1.GetPrimaryPartNumber();
const bool needs_cpu_ctlr = (implementer == cpu::MainIdRegisterAccessor::Implementer::ArmLimited) && (primary_part == cpu::MainIdRegisterAccessor::PrimaryPartNumber::CortexA57 || primary_part == cpu::MainIdRegisterAccessor::PrimaryPartNumber::CortexA53);
/* Get parameters for initial arguments. */
const u64 cpuactlr = needs_cpu_ctlr ? cpu::GetCpuActlrEl1() : 0;
const u64 cpuectlr = needs_cpu_ctlr ? cpu::GetCpuEctlrEl1() : 0;
for (s32 i = 0; i < static_cast<s32>(cpu::NumCores); ++i) {
/* Get the arguments. */
KInitArguments *init_args = g_init_arguments + i;
/* Set the arguments. */
init_args->cpuactlr = cpuactlr;
init_args->cpuectlr = cpuectlr;
init_args->sp = GetInteger(KMemoryLayout::GetMainStackTopAddress(i)) - sizeof(KThread::StackParameters);
init_args->entrypoint = reinterpret_cast<uintptr_t>(::ams::kern::init::InvokeMain);
init_args->argument = static_cast<u64>(i);
}
}
KVirtualAddress GetRandomAlignedRegionWithGuard(size_t size, size_t alignment, KInitialPageTable &pt, KMemoryRegionTree &tree, u32 type_id, size_t guard_size) {
/* Check that the size is valid. */
MESOSPHERE_INIT_ABORT_UNLESS(size > 0);
/* We want to find the total extents of the type id. */
const auto extents = tree.GetDerivedRegionExtents(type_id);
/* Ensure that our alignment is correct. */
MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(extents.GetAddress(), alignment));
const uintptr_t first_address = extents.GetAddress();
const uintptr_t last_address = extents.GetLastAddress();
const uintptr_t first_index = first_address / alignment;
const uintptr_t last_index = last_address / alignment;
while (true) {
const uintptr_t candidate_start = KSystemControl::Init::GenerateRandomRange(first_index, last_index) * alignment;
const uintptr_t candidate_end = candidate_start + size + guard_size;
/* Ensure that the candidate doesn't overflow with the size/guard. */
if (!(candidate_start < candidate_end) || !(candidate_start >= guard_size)) {
continue;
}
const uintptr_t candidate_last = candidate_end - 1;
/* Ensure that the candidate fits within the region. */
if (candidate_last > last_address) {
continue;
}
/* Ensure that the candidate range is free. */
if (!pt.IsFree(candidate_start, size)) {
continue;
}
/* Locate the candidate's guard start, and ensure the whole range fits/has the correct type id. */
if (const auto &candidate_region = *tree.Find(candidate_start - guard_size); !(candidate_last <= candidate_region.GetLastAddress() && candidate_region.GetType() == type_id)) {
continue;
}
return candidate_start;
}
}
KVirtualAddress GetRandomAlignedRegion(size_t size, size_t alignment, KInitialPageTable &pt, KMemoryRegionTree &tree, u32 type_id) {
return GetRandomAlignedRegionWithGuard(size, alignment, pt, tree, type_id, 0);
}
void MapStackForCore(KInitialPageTable &page_table, KMemoryRegionType type, u32 core_id) {
constexpr size_t StackSize = PageSize;
constexpr size_t StackAlign = PageSize;
const KVirtualAddress stack_start_virt = GetRandomAlignedRegionWithGuard(StackSize, StackAlign, page_table, KMemoryLayout::GetVirtualMemoryRegionTree(), KMemoryRegionType_KernelMisc, PageSize);
const KPhysicalAddress stack_start_phys = g_initial_page_allocator.Allocate(PageSize);
MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetVirtualMemoryRegionTree().Insert(GetInteger(stack_start_virt), StackSize, type, core_id));
page_table.Map(stack_start_virt, StackSize, stack_start_phys, KernelRwDataAttribute, g_initial_page_allocator, 0);
}
class KInitialPageAllocatorForFinalizeIdentityMapping final {
private:
struct FreeListEntry {
FreeListEntry *next;
};
private:
FreeListEntry *m_free_list_head;
u64 m_phys_to_virt_offset;
public:
template<kern::arch::arm64::init::IsInitialPageAllocator PageAllocator>
KInitialPageAllocatorForFinalizeIdentityMapping(PageAllocator &allocator, u64 phys_to_virt) : m_free_list_head(nullptr), m_phys_to_virt_offset(phys_to_virt) {
/* Allocate and free two pages. */
for (size_t i = 0; i < 2; ++i) {
KPhysicalAddress page = allocator.Allocate(PageSize);
MESOSPHERE_INIT_ABORT_UNLESS(page != Null<KPhysicalAddress>);
/* Free the pages. */
this->Free(page, PageSize);
}
}
public:
KPhysicalAddress Allocate(size_t size) {
/* Check that the size is correct. */
MESOSPHERE_INIT_ABORT_UNLESS(size == PageSize);
/* Check that we have a free page. */
FreeListEntry *head = m_free_list_head;
MESOSPHERE_INIT_ABORT_UNLESS(head != nullptr);
/* Update the free list. */
m_free_list_head = head->next;
/* Return the page. */
return KPhysicalAddress(reinterpret_cast<uintptr_t>(head) - m_phys_to_virt_offset);
}
void Free(KPhysicalAddress phys_addr, size_t size) {
/* Check that the size is correct. */
MESOSPHERE_INIT_ABORT_UNLESS(size == PageSize);
/* Convert to a free list entry. */
FreeListEntry *fl = reinterpret_cast<FreeListEntry *>(GetInteger(phys_addr) + m_phys_to_virt_offset);
/* Insert into free list. */
fl->next = m_free_list_head;
m_free_list_head = fl;
}
};
static_assert(kern::arch::arm64::init::IsInitialPageAllocator<KInitialPageAllocatorForFinalizeIdentityMapping>);
void SetupAllTtbr0Entries(KInitialPageTable &init_pt, KInitialPageAllocator &allocator) {
/* Validate that the ttbr0 array is in rodata. */
const uintptr_t rodata_start = reinterpret_cast<uintptr_t>(__rodata_start);
const uintptr_t rodata_end = reinterpret_cast<uintptr_t>(__rodata_end);
MESOSPHERE_INIT_ABORT_UNLESS(rodata_start < rodata_end);
MESOSPHERE_INIT_ABORT_UNLESS(rodata_start <= reinterpret_cast<uintptr_t>(std::addressof(KPageTable::GetTtbr0Entry(0))));
MESOSPHERE_INIT_ABORT_UNLESS(reinterpret_cast<uintptr_t>(std::addressof(KPageTable::GetTtbr0Entry(KPageTable::NumTtbr0Entries))) < rodata_end);
/* Allocate pages for all ttbr0 entries. */
for (size_t i = 0; i < KPageTable::NumTtbr0Entries; ++i) {
/* Allocate a page. */
KPhysicalAddress page = allocator.Allocate(PageSize);
MESOSPHERE_INIT_ABORT_UNLESS(page != Null<KPhysicalAddress>);
/* Check that the page is allowed to be a ttbr0 entry. */
MESOSPHERE_INIT_ABORT_UNLESS((GetInteger(page) & UINT64_C(0xFFFF000000000001)) == 0);
/* Get the physical address of the ttbr0 entry. */
const auto ttbr0_phys_ptr = init_pt.GetPhysicalAddress(KVirtualAddress(std::addressof(KPageTable::GetTtbr0Entry(i))));
/* Set the entry to the newly allocated page. */
*reinterpret_cast<volatile u64 *>(GetInteger(ttbr0_phys_ptr)) = (static_cast<u64>(i) << 48) | GetInteger(page);
}
}
void FinalizeIdentityMapping(KInitialPageTable &init_pt, KInitialPageAllocator &allocator, u64 phys_to_virt_offset) {
/* Create an allocator for identity mapping finalization. */
KInitialPageAllocatorForFinalizeIdentityMapping finalize_allocator(allocator, phys_to_virt_offset);
/* Get the physical address of crt0. */
const KPhysicalAddress start_phys_addr = init_pt.GetPhysicalAddress(reinterpret_cast<uintptr_t>(::ams::kern::init::IdentityMappedFunctionAreaBegin));
/* Unmap the entire identity mapping. */
init_pt.UnmapTtbr0Entries(phys_to_virt_offset);
/* Re-map only the first page of code. */
const size_t size = util::AlignUp<size_t>(reinterpret_cast<uintptr_t>(::ams::kern::init::IdentityMappedFunctionAreaEnd) - reinterpret_cast<uintptr_t>(::ams::kern::init::IdentityMappedFunctionAreaBegin), PageSize);
init_pt.Map(KVirtualAddress(GetInteger(start_phys_addr)), size, start_phys_addr, KernelTextAttribute, finalize_allocator, phys_to_virt_offset);
}
}
void InitializeCorePhase1(uintptr_t misc_unk_debug_phys_addr, void **initial_state) {
/* Ensure our first argument is page aligned. */
MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(misc_unk_debug_phys_addr, PageSize));
/* Decode the initial state. */
const auto initial_page_allocator_state = *static_cast<KInitialPageAllocator::State *>(initial_state[0]);
g_phase2_initial_process_binary_meta = *static_cast<InitialProcessBinaryLayoutWithSize *>(initial_state[1]);
/* Restore the page allocator state setup by kernel loader. */
g_initial_page_allocator.InitializeFromState(std::addressof(initial_page_allocator_state));
/* Ensure that the T1SZ is correct (and what we expect). */
MESOSPHERE_INIT_ABORT_UNLESS((cpu::TranslationControlRegisterAccessor().GetT1Size() / arch::arm64::L1BlockSize) == arch::arm64::MaxPageTableEntries);
/* Create page table object for use during initialization. */
KInitialPageTable init_pt;
/* Initialize the slab allocator counts. */
InitializeSlabResourceCounts();
/* Insert the root region for the virtual memory tree, from which all other regions will derive. */
KMemoryLayout::GetVirtualMemoryRegionTree().InsertDirectly(KernelVirtualAddressSpaceBase, KernelVirtualAddressSpaceBase + KernelVirtualAddressSpaceSize - 1);
/* Insert the root region for the physical memory tree, from which all other regions will derive. */
KMemoryLayout::GetPhysicalMemoryRegionTree().InsertDirectly(KernelPhysicalAddressSpaceBase, KernelPhysicalAddressSpaceBase + KernelPhysicalAddressSpaceSize - 1);
/* Save start and end for ease of use. */
const uintptr_t code_start_virt_addr = reinterpret_cast<uintptr_t>(__bin_start__);
const uintptr_t code_end_virt_addr = reinterpret_cast<uintptr_t>(__bin_end__);
/* Setup the containing kernel region. */
constexpr size_t KernelRegionSize = 1_GB;
constexpr size_t KernelRegionAlign = 1_GB;
const KVirtualAddress kernel_region_start = util::AlignDown(code_start_virt_addr, KernelRegionAlign);
size_t kernel_region_size = KernelRegionSize;
if (!(kernel_region_start + KernelRegionSize - 1 <= KernelVirtualAddressSpaceLast)) {
kernel_region_size = KernelVirtualAddressSpaceEnd - GetInteger(kernel_region_start);
}
MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetVirtualMemoryRegionTree().Insert(GetInteger(kernel_region_start), kernel_region_size, KMemoryRegionType_Kernel));
/* Setup the code region. */
constexpr size_t CodeRegionAlign = PageSize;
const KVirtualAddress code_region_start = util::AlignDown(code_start_virt_addr, CodeRegionAlign);
const KVirtualAddress code_region_end = util::AlignUp(code_end_virt_addr, CodeRegionAlign);
const size_t code_region_size = GetInteger(code_region_end) - GetInteger(code_region_start);
MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetVirtualMemoryRegionTree().Insert(GetInteger(code_region_start), code_region_size, KMemoryRegionType_KernelCode));
/* Setup board-specific device physical regions. */
SetupDevicePhysicalMemoryRegions();
/* Determine the amount of space needed for the misc region. */
size_t misc_region_needed_size;
{
/* Each core has a one page stack for all three stack types (Main, Idle, Exception). */
misc_region_needed_size = cpu::NumCores * (3 * (PageSize + PageSize));
/* Account for each auto-map device. */
for (const auto &region : KMemoryLayout::GetPhysicalMemoryRegionTree()) {
if (region.HasTypeAttribute(KMemoryRegionAttr_ShouldKernelMap)) {
/* Check that the region is valid. */
MESOSPHERE_INIT_ABORT_UNLESS(region.GetEndAddress() != 0);
/* Account for the region. */
const auto aligned_start = util::AlignDown(region.GetAddress(), PageSize);
const auto aligned_end = util::AlignUp(region.GetLastAddress(), PageSize);
const size_t cur_region_size = aligned_end - aligned_start;
misc_region_needed_size += cur_region_size;
/* Account for alignment requirements. */
const size_t min_align = std::min<size_t>(util::GetAlignment(cur_region_size), util::GetAlignment(aligned_start));
misc_region_needed_size += min_align >= KernelAslrAlignment ? KernelAslrAlignment : PageSize;
}
}
/* Account for the unknown debug region. */
misc_region_needed_size += GetMiscUnknownDebugRegionSize();
/* Multiply the needed size by three, to account for the need for guard space. */
misc_region_needed_size *= 3;
}
/* Decide on the actual size for the misc region. */
constexpr size_t MiscRegionAlign = KernelAslrAlignment;
constexpr size_t MiscRegionMinimumSize = 32_MB;
const size_t misc_region_size = util::AlignUp(std::max(misc_region_needed_size, MiscRegionMinimumSize), MiscRegionAlign);
MESOSPHERE_INIT_ABORT_UNLESS(misc_region_size > 0);
/* Setup the misc region. */
const KVirtualAddress misc_region_start = GetRandomAlignedRegion(misc_region_size, MiscRegionAlign, init_pt, KMemoryLayout::GetVirtualMemoryRegionTree(), KMemoryRegionType_Kernel);
MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetVirtualMemoryRegionTree().Insert(GetInteger(misc_region_start), misc_region_size, KMemoryRegionType_KernelMisc));
/* Determine if we'll use extra thread resources. */
const bool use_extra_resources = KSystemControl::Init::ShouldIncreaseThreadResourceLimit();
/* Setup the stack region. */
const size_t stack_region_size = use_extra_resources ? 24_MB : 14_MB;
constexpr size_t StackRegionAlign = KernelAslrAlignment;
const KVirtualAddress stack_region_start = GetRandomAlignedRegion(stack_region_size, StackRegionAlign, init_pt, KMemoryLayout::GetVirtualMemoryRegionTree(), KMemoryRegionType_Kernel);
MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetVirtualMemoryRegionTree().Insert(GetInteger(stack_region_start), stack_region_size, KMemoryRegionType_KernelStack));
/* Determine the size of the resource region. */
const size_t resource_region_size = KMemoryLayout::GetResourceRegionSizeForInit(use_extra_resources);
/* Determine the size of the slab region. */
const size_t slab_region_size = util::AlignUp(CalculateTotalSlabHeapSize(), PageSize);
MESOSPHERE_INIT_ABORT_UNLESS(slab_region_size <= resource_region_size);
/* Setup the slab region. */
const KPhysicalAddress code_start_phys_addr = g_phase2_initial_process_binary_meta.layout.kern_address;
const KPhysicalAddress code_end_phys_addr = code_start_phys_addr + code_region_size;
const KPhysicalAddress slab_start_phys_addr = code_end_phys_addr;
const KPhysicalAddress slab_end_phys_addr = slab_start_phys_addr + slab_region_size;
constexpr size_t SlabRegionAlign = KernelAslrAlignment;
const size_t slab_region_needed_size = util::AlignUp(GetInteger(code_end_phys_addr) + slab_region_size, SlabRegionAlign) - util::AlignDown(GetInteger(code_end_phys_addr), SlabRegionAlign);
const KVirtualAddress slab_region_start = GetRandomAlignedRegion(slab_region_needed_size, SlabRegionAlign, init_pt, KMemoryLayout::GetVirtualMemoryRegionTree(), KMemoryRegionType_Kernel) + (GetInteger(code_end_phys_addr) % SlabRegionAlign);
MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetVirtualMemoryRegionTree().Insert(GetInteger(slab_region_start), slab_region_size, KMemoryRegionType_KernelSlab));
/* Setup the temp region. */
constexpr size_t TempRegionSize = 128_MB;
constexpr size_t TempRegionAlign = KernelAslrAlignment;
const KVirtualAddress temp_region_start = GetRandomAlignedRegion(TempRegionSize, TempRegionAlign, init_pt, KMemoryLayout::GetVirtualMemoryRegionTree(), KMemoryRegionType_Kernel);
MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetVirtualMemoryRegionTree().Insert(GetInteger(temp_region_start), TempRegionSize, KMemoryRegionType_KernelTemp));
/* Automatically map in devices that have auto-map attributes, from largest region to smallest region. */
{
/* We want to map the regions from largest to smallest. */
KMemoryRegion *largest;
do {
/* Begin with no knowledge of the largest region. */
largest = nullptr;
for (auto &region : KMemoryLayout::GetPhysicalMemoryRegionTree()) {
/* We only care about kernel regions. */
if (!region.IsDerivedFrom(KMemoryRegionType_Kernel)) {
continue;
}
/* Check whether we should map the region. */
if (!region.HasTypeAttribute(KMemoryRegionAttr_ShouldKernelMap)) {
continue;
}
/* If this region has already been mapped, no need to consider it. */
if (region.HasTypeAttribute(KMemoryRegionAttr_DidKernelMap)) {
continue;
}
/* Check that the region is valid. */
MESOSPHERE_INIT_ABORT_UNLESS(region.GetEndAddress() != 0);
/* Update the largest region. */
if (largest == nullptr || largest->GetSize() < region.GetSize()) {
largest = std::addressof(region);
}
}
/* If we found a region, map it. */
if (largest != nullptr) {
/* Set the attribute to note we've mapped this region. */
largest->SetTypeAttribute(KMemoryRegionAttr_DidKernelMap);
/* Create a virtual pair region and insert it into the tree. */
const KPhysicalAddress map_phys_addr = util::AlignDown(largest->GetAddress(), PageSize);
const size_t map_size = util::AlignUp(largest->GetEndAddress(), PageSize) - GetInteger(map_phys_addr);
const size_t min_align = std::min<size_t>(util::GetAlignment(map_size), util::GetAlignment(GetInteger(map_phys_addr)));
const size_t map_align = min_align >= KernelAslrAlignment ? KernelAslrAlignment : PageSize;
const KVirtualAddress map_virt_addr = GetRandomAlignedRegionWithGuard(map_size, map_align, init_pt, KMemoryLayout::GetVirtualMemoryRegionTree(), KMemoryRegionType_KernelMisc, PageSize);
MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetVirtualMemoryRegionTree().Insert(GetInteger(map_virt_addr), map_size, KMemoryRegionType_KernelMiscMappedDevice));
largest->SetPairAddress(GetInteger(map_virt_addr) + largest->GetAddress() - GetInteger(map_phys_addr));
/* Map the page in to our page table. */
init_pt.Map(map_virt_addr, map_size, map_phys_addr, KernelMmioAttribute, g_initial_page_allocator, 0);
}
} while (largest != nullptr);
}
/* Setup the basic DRAM regions. */
SetupDramPhysicalMemoryRegions();
/* Automatically map in reserved physical memory that has auto-map attributes. */
{
/* We want to map the regions from largest to smallest. */
KMemoryRegion *largest;
do {
/* Begin with no knowledge of the largest region. */
largest = nullptr;
for (auto &region : KMemoryLayout::GetPhysicalMemoryRegionTree()) {
/* We only care about reserved memory. */
if (!region.IsDerivedFrom(KMemoryRegionType_DramReservedBase)) {
continue;
}
/* Check whether we should map the region. */
if (!region.HasTypeAttribute(KMemoryRegionAttr_ShouldKernelMap)) {
continue;
}
/* If this region has already been mapped, no need to consider it. */
if (region.HasTypeAttribute(KMemoryRegionAttr_DidKernelMap)) {
continue;
}
/* Check that the region is valid. */
MESOSPHERE_INIT_ABORT_UNLESS(region.GetEndAddress() != 0);
/* Update the largest region. */
if (largest == nullptr || largest->GetSize() < region.GetSize()) {
largest = std::addressof(region);
}
}
/* If we found a region, map it. */
if (largest != nullptr) {
/* Set the attribute to note we've mapped this region. */
largest->SetTypeAttribute(KMemoryRegionAttr_DidKernelMap);
/* Create a virtual pair region and insert it into the tree. */
const KPhysicalAddress map_phys_addr = util::AlignDown(largest->GetAddress(), PageSize);
const size_t map_size = util::AlignUp(largest->GetEndAddress(), PageSize) - GetInteger(map_phys_addr);
const size_t min_align = std::min<size_t>(util::GetAlignment(map_size), util::GetAlignment(GetInteger(map_phys_addr)));
const size_t map_align = min_align >= KernelAslrAlignment ? KernelAslrAlignment : PageSize;
const KVirtualAddress map_virt_addr = GetRandomAlignedRegionWithGuard(map_size, map_align, init_pt, KMemoryLayout::GetVirtualMemoryRegionTree(), KMemoryRegionType_KernelMisc, PageSize);
MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetVirtualMemoryRegionTree().Insert(GetInteger(map_virt_addr), map_size, KMemoryRegionType_KernelMiscUnknownDebug));
largest->SetPairAddress(GetInteger(map_virt_addr) + largest->GetAddress() - GetInteger(map_phys_addr));
/* Map the page in to our page table. */
const auto attribute = largest->HasTypeAttribute(KMemoryRegionAttr_Uncached) ? KernelRwDataUncachedAttribute : KernelRwDataAttribute;
init_pt.Map(map_virt_addr, map_size, map_phys_addr, attribute, g_initial_page_allocator, 0);
}
} while (largest != nullptr);
}
/* Insert a physical region for the kernel code region. */
MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetPhysicalMemoryRegionTree().Insert(GetInteger(code_start_phys_addr), code_region_size, KMemoryRegionType_DramKernelCode));
/* Insert a physical region for the kernel slab region. */
MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetPhysicalMemoryRegionTree().Insert(GetInteger(slab_start_phys_addr), slab_region_size, KMemoryRegionType_DramKernelSlab));
/* Map the slab region. */
init_pt.Map(slab_region_start, slab_region_size, slab_start_phys_addr, KernelRwDataAttribute, g_initial_page_allocator, 0);
/* Physically randomize the slab region. */
/* NOTE: Nintendo does this only on 10.0.0+ */
init_pt.PhysicallyRandomize(slab_region_start, slab_region_size, false);
/* Insert a physical region for the secure applet memory. */
const auto secure_applet_end_phys_addr = slab_end_phys_addr + KSystemControl::SecureAppletMemorySize;
if constexpr (KSystemControl::SecureAppletMemorySize > 0) {
MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetPhysicalMemoryRegionTree().Insert(GetInteger(slab_end_phys_addr), KSystemControl::SecureAppletMemorySize, KMemoryRegionType_DramKernelSecureAppletMemory));
}
/* Insert a physical region for the unknown debug2 region. */
const size_t secure_unknown_size = GetSecureUnknownRegionSize();
const auto secure_unknown_end_phys_addr = secure_applet_end_phys_addr + secure_unknown_size;
if (secure_unknown_size > 0) {
MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetPhysicalMemoryRegionTree().Insert(GetInteger(secure_applet_end_phys_addr), secure_unknown_size, KMemoryRegionType_DramKernelSecureUnknown));
}
/* Determine size available for kernel page table heaps. */
const KPhysicalAddress resource_end_phys_addr = slab_start_phys_addr + resource_region_size;
g_phase2_resource_end_phys_addr = resource_end_phys_addr;
const size_t page_table_heap_size = GetInteger(resource_end_phys_addr) - GetInteger(secure_unknown_end_phys_addr);
/* Insert a physical region for the kernel page table heap region */
MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetPhysicalMemoryRegionTree().Insert(GetInteger(secure_unknown_end_phys_addr), page_table_heap_size, KMemoryRegionType_DramKernelPtHeap));
/* All DRAM regions that we haven't tagged by this point will be mapped under the linear mapping. Tag them. */
for (auto &region : KMemoryLayout::GetPhysicalMemoryRegionTree()) {
if (region.GetType() == KMemoryRegionType_Dram) {
/* Check that the region is valid. */
MESOSPHERE_INIT_ABORT_UNLESS(region.GetEndAddress() != 0);
/* Set the linear map attribute. */
region.SetTypeAttribute(KMemoryRegionAttr_LinearMapped);
}
}
/* Get the linear region extents. */
const auto linear_extents = KMemoryLayout::GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionAttr_LinearMapped);
MESOSPHERE_INIT_ABORT_UNLESS(linear_extents.GetEndAddress() != 0);
/* Setup the linear mapping region. */
constexpr size_t LinearRegionAlign = 1_GB;
const KPhysicalAddress aligned_linear_phys_start = util::AlignDown(linear_extents.GetAddress(), LinearRegionAlign);
const size_t linear_region_size = util::AlignUp(linear_extents.GetEndAddress(), LinearRegionAlign) - GetInteger(aligned_linear_phys_start);
const KVirtualAddress linear_region_start = GetRandomAlignedRegionWithGuard(linear_region_size, LinearRegionAlign, init_pt, KMemoryLayout::GetVirtualMemoryRegionTree(), KMemoryRegionType_None, LinearRegionAlign);
const uintptr_t linear_region_phys_to_virt_diff = GetInteger(linear_region_start) - GetInteger(aligned_linear_phys_start);
/* Map and create regions for all the linearly-mapped data. */
{
uintptr_t cur_phys_addr = 0;
uintptr_t cur_size = 0;
for (auto &region : KMemoryLayout::GetPhysicalMemoryRegionTree()) {
if (!region.HasTypeAttribute(KMemoryRegionAttr_LinearMapped)) {
continue;
}
MESOSPHERE_INIT_ABORT_UNLESS(region.GetEndAddress() != 0);
if (cur_size == 0) {
cur_phys_addr = region.GetAddress();
cur_size = region.GetSize();
} else if (cur_phys_addr + cur_size == region.GetAddress()) {
cur_size += region.GetSize();
} else {
const uintptr_t cur_virt_addr = cur_phys_addr + linear_region_phys_to_virt_diff;
init_pt.Map(cur_virt_addr, cur_size, cur_phys_addr, KernelRwDataAttribute, g_initial_page_allocator, 0);
cur_phys_addr = region.GetAddress();
cur_size = region.GetSize();
}
const uintptr_t region_virt_addr = region.GetAddress() + linear_region_phys_to_virt_diff;
if (!region.HasTypeAttribute(KMemoryRegionAttr_ShouldKernelMap)) {
region.SetPairAddress(region_virt_addr);
}
MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetVirtualMemoryRegionTree().Insert(region_virt_addr, region.GetSize(), GetTypeForVirtualLinearMapping(region.GetType())));
KMemoryRegion *virt_region = KMemoryLayout::GetVirtualMemoryRegionTree().FindModifiable(region_virt_addr);
MESOSPHERE_INIT_ABORT_UNLESS(virt_region != nullptr);
virt_region->SetPairAddress(region.GetAddress());
}
/* Map the last block, which we may have skipped. */
if (cur_size != 0) {
const uintptr_t cur_virt_addr = cur_phys_addr + linear_region_phys_to_virt_diff;
init_pt.Map(cur_virt_addr, cur_size, cur_phys_addr, KernelRwDataAttribute, g_initial_page_allocator, 0);
}
}
/* NOTE: Unknown function is called here which is ifdef'd out on retail kernel. */
/* The unknown function is immediately before the function which gets an unknown debug region size, inside this translation unit. */
/* It's likely that this is some kind of initializer for this unknown debug region. */
/* Create regions for and map all core-specific stacks. */
for (size_t i = 0; i < cpu::NumCores; i++) {
MapStackForCore(init_pt, KMemoryRegionType_KernelMiscMainStack, i);
MapStackForCore(init_pt, KMemoryRegionType_KernelMiscIdleStack, i);
MapStackForCore(init_pt, KMemoryRegionType_KernelMiscExceptionStack, i);
}
/* Setup the initial arguments. */
SetupInitialArguments();
/* Set linear difference for Phase2. */
g_phase2_linear_region_phys_to_virt_diff = linear_region_phys_to_virt_diff;
}
void InitializeCorePhase2() {
/* Create page table object for use during remaining initialization. */
KInitialPageTable init_pt;
/* Setup all ttbr0 pages. */
SetupAllTtbr0Entries(init_pt, g_initial_page_allocator);
/* Unmap the identity mapping. */
FinalizeIdentityMapping(init_pt, g_initial_page_allocator, g_phase2_linear_region_phys_to_virt_diff);
/* Finalize the page allocator, we're done allocating at this point. */
KInitialPageAllocator::State final_init_page_table_state;
g_initial_page_allocator.GetFinalState(std::addressof(final_init_page_table_state));
const KPhysicalAddress final_init_page_table_end_address = final_init_page_table_state.end_address;
const size_t init_page_table_region_size = GetInteger(final_init_page_table_end_address) - GetInteger(g_phase2_resource_end_phys_addr);
/* Insert regions for the initial page table region. */
MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetPhysicalMemoryRegionTree().Insert(GetInteger(g_phase2_resource_end_phys_addr), init_page_table_region_size, KMemoryRegionType_DramKernelInitPt));
MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetVirtualMemoryRegionTree().Insert(GetInteger(g_phase2_resource_end_phys_addr) + g_phase2_linear_region_phys_to_virt_diff, init_page_table_region_size, KMemoryRegionType_VirtualDramKernelInitPt));
/* Insert a physical region for the kernel trace buffer */
if constexpr (IsKTraceEnabled) {
const KPhysicalAddress ktrace_buffer_phys_addr = GetInteger(g_phase2_resource_end_phys_addr) + init_page_table_region_size;
MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetPhysicalMemoryRegionTree().Insert(GetInteger(ktrace_buffer_phys_addr), KTraceBufferSize, KMemoryRegionType_KernelTraceBuffer));
MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetPhysicalMemoryRegionTree().Insert(GetInteger(ktrace_buffer_phys_addr) + g_phase2_linear_region_phys_to_virt_diff, KTraceBufferSize, GetTypeForVirtualLinearMapping(KMemoryRegionType_KernelTraceBuffer)));
}
/* All linear-mapped DRAM regions that we haven't tagged by this point will be allocated to some pool partition. Tag them. */
for (auto &region : KMemoryLayout::GetPhysicalMemoryRegionTree()) {
constexpr auto UntaggedLinearDram = util::FromUnderlying<KMemoryRegionType>(util::ToUnderlying<KMemoryRegionType>(KMemoryRegionType_Dram) | util::ToUnderlying(KMemoryRegionAttr_LinearMapped));
if (region.GetType() == UntaggedLinearDram) {
region.SetType(KMemoryRegionType_DramPoolPartition);
}
}
/* Set the linear memory offsets, to enable conversion between physical and virtual addresses. */
KMemoryLayout::InitializeLinearMemoryAddresses(g_phase2_linear_region_phys_to_virt_diff);
/* Set the initial process binary physical address. */
/* NOTE: Nintendo does this after pool partition setup, but it's a requirement that we do it before */
/* to retain compatibility with < 5.0.0. */
const KPhysicalAddress ini_address = g_phase2_initial_process_binary_meta.layout.address;
const size_t ini_size = g_phase2_initial_process_binary_meta.size;
MESOSPHERE_INIT_ABORT_UNLESS(ini_address != Null<KPhysicalAddress>);
SetInitialProcessBinaryPhysicalAddress(ini_address, ini_size);
/* Setup all other memory regions needed to arrange the pool partitions. */
SetupPoolPartitionMemoryRegions();
/* Validate the initial process binary address. */
{
const KMemoryRegion *ini_region = KMemoryLayout::Find(ini_address);
/* Check that the region is non-kernel dram. */
MESOSPHERE_INIT_ABORT_UNLESS(ini_region->IsDerivedFrom(KMemoryRegionType_DramUserPool));
/* Check that the region contains the ini. */
MESOSPHERE_INIT_ABORT_UNLESS(ini_region->GetAddress() <= GetInteger(ini_address));
MESOSPHERE_INIT_ABORT_UNLESS(GetInteger(ini_address) + ini_size <= ini_region->GetEndAddress());
MESOSPHERE_INIT_ABORT_UNLESS(ini_region->GetEndAddress() != 0);
}
/* Cache all linear regions in their own trees for faster access, later. */
KMemoryLayout::InitializeLinearMemoryRegionTrees();
/* Turn on all other cores. */
TurnOnAllCores();
}
KInitArguments *GetInitArguments(s32 core_id) {
return g_init_arguments + core_id;
}
void InitializeDebugRegisters() {
/* Determine how many watchpoints and breakpoints we have */
cpu::DebugFeatureRegisterAccessor aa64dfr0;
const auto num_watchpoints = aa64dfr0.GetNumWatchpoints();
const auto num_breakpoints = aa64dfr0.GetNumBreakpoints();
cpu::EnsureInstructionConsistencyFullSystem();
/* Clear the debug monitor register and the os lock access register. */
cpu::MonitorDebugSystemControlRegisterAccessor(0).Store();
cpu::EnsureInstructionConsistencyFullSystem();
cpu::OsLockAccessRegisterAccessor(0).Store();
cpu::EnsureInstructionConsistencyFullSystem();
/* Clear all debug watchpoints/breakpoints. */
#define FOR_I_IN_15_TO_1(HANDLER, ...) \
HANDLER(15, ## __VA_ARGS__) HANDLER(14, ## __VA_ARGS__) HANDLER(13, ## __VA_ARGS__) HANDLER(12, ## __VA_ARGS__) \
HANDLER(11, ## __VA_ARGS__) HANDLER(10, ## __VA_ARGS__) HANDLER(9, ## __VA_ARGS__) HANDLER(8, ## __VA_ARGS__) \
HANDLER(7, ## __VA_ARGS__) HANDLER(6, ## __VA_ARGS__) HANDLER(5, ## __VA_ARGS__) HANDLER(4, ## __VA_ARGS__) \
HANDLER(3, ## __VA_ARGS__) HANDLER(2, ## __VA_ARGS__) HANDLER(1, ## __VA_ARGS__)
#define MESOSPHERE_INITIALIZE_WATCHPOINT_CASE(ID, ...) \
case ID: \
cpu::SetDbgWcr##ID##El1(__VA_ARGS__); \
cpu::SetDbgWvr##ID##El1(__VA_ARGS__); \
[[fallthrough]];
#define MESOSPHERE_INITIALIZE_BREAKPOINT_CASE(ID, ...) \
case ID: \
cpu::SetDbgBcr##ID##El1(__VA_ARGS__); \
cpu::SetDbgBvr##ID##El1(__VA_ARGS__); \
[[fallthrough]];
switch (num_watchpoints) {
FOR_I_IN_15_TO_1(MESOSPHERE_INITIALIZE_WATCHPOINT_CASE, 0)
case 0:
cpu::SetDbgWcr0El1(0);
cpu::SetDbgWvr0El1(0);
[[fallthrough]];
default:
break;
}
switch (num_breakpoints) {
FOR_I_IN_15_TO_1(MESOSPHERE_INITIALIZE_BREAKPOINT_CASE, 0)
default:
break;
}
cpu::SetDbgBcr0El1(0);
cpu::SetDbgBvr0El1(0);
#undef MESOSPHERE_INITIALIZE_WATCHPOINT_CASE
#undef MESOSPHERE_INITIALIZE_BREAKPOINT_CASE
#undef FOR_I_IN_15_TO_1
cpu::EnsureInstructionConsistencyFullSystem();
/* Initialize the context id register to all 1s. */
cpu::ContextIdRegisterAccessor(0).SetProcId(std::numeric_limits<u32>::max()).Store();
cpu::EnsureInstructionConsistencyFullSystem();
/* Configure the debug monitor register. */
cpu::MonitorDebugSystemControlRegisterAccessor(0).SetMde(true).SetTdcc(true).Store();
cpu::EnsureInstructionConsistencyFullSystem();
}
void InitializeExceptionVectors() {
cpu::SetVbarEl1(reinterpret_cast<uintptr_t>(::ams::kern::ExceptionVectors));
cpu::SetTpidrEl1(0);
cpu::SetExceptionThreadStackTop(0);
cpu::EnsureInstructionConsistencyFullSystem();
}
size_t GetMiscUnknownDebugRegionSize() {
return 0;
}
size_t GetSecureUnknownRegionSize() {
return 0;
}
}

637
Source/Atmosphere-MTC-Unlock/mesosphere/kernel/source/arch/arm64/init/start.s
View File

@@ -1,637 +0,0 @@
/*
* Copyright (c) Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <mesosphere/kern_select_assembly_offsets.h>
/* For some reason GAS doesn't know about it, even with .cpu cortex-a57 */
#define cpuactlr_el1 s3_1_c15_c2_0
#define cpuectlr_el1 s3_1_c15_c2_1
#define LOAD_IMMEDIATE_32(reg, val) \
mov reg, #(((val) >> 0x00) & 0xFFFF); \
movk reg, #(((val) >> 0x10) & 0xFFFF), lsl#16
#define LOAD_IMMEDIATE_64(reg, val) \
mov reg, #(((val) >> 0x00) & 0xFFFF); \
movk reg, #(((val) >> 0x10) & 0xFFFF), lsl#16; \
movk reg, #(((val) >> 0x20) & 0xFFFF), lsl#32; \
movk reg, #(((val) >> 0x30) & 0xFFFF), lsl#48
#define LOAD_FROM_LABEL(reg, label) \
adr reg, label; \
ldr reg, [reg]
#define LOAD_RELATIVE_FROM_LABEL(reg, reg2, label) \
adr reg2, label; \
ldr reg, [reg2]; \
add reg, reg, reg2
#define INDIRECT_RELATIVE_CALL(reg, reg2, label) \
adr reg, label; \
add reg, reg, reg2; \
blr reg
#define SETUP_SYSTEM_REGISTER(_reg, _sr) \
LOAD_FROM_LABEL(_reg, __sysreg_constant_ ## _sr); \
msr _sr, _reg
.section .start, "ax", %progbits
.global _start
_start:
b _ZN3ams4kern4init10StartCore0Emm
__metadata_magic_number:
.ascii "MSS1" /* Magic, if executed as gadget "adds w13, w26, #0x4d4, lsl #12" */
__metadata_offset:
.word __metadata_begin - _start
#ifdef ATMOSPHERE_BOARD_NINTENDO_NX
.global _ZN3ams4kern17GetTargetFirmwareEv
.type _ZN3ams4kern17GetTargetFirmwareEv, %function
_ZN3ams4kern17GetTargetFirmwareEv:
adr x0, __metadata_target_firmware
ldr w0, [x0]
ret
#endif
.section .crt0.text.start, "ax", %progbits
/* ams::kern::init::IdentityMappedFunctionAreaBegin() */
.global _ZN3ams4kern4init31IdentityMappedFunctionAreaBeginEv
.type _ZN3ams4kern4init31IdentityMappedFunctionAreaBeginEv, %function
_ZN3ams4kern4init31IdentityMappedFunctionAreaBeginEv:
/* NOTE: This is not a real function, and only exists as a label for safety. */
/* ================ Functions after this line remain identity-mapped after initialization finishes. ================ */
/* ams::kern::init::StartCore0(uintptr_t, uintptr_t) */
.section .crt0.text._ZN3ams4kern4init10StartCore0Emm, "ax", %progbits
.global _ZN3ams4kern4init10StartCore0Emm
.type _ZN3ams4kern4init10StartCore0Emm, %function
_ZN3ams4kern4init10StartCore0Emm:
/* Mask all interrupts. */
msr daifset, #0xF
/* Save arguments for later use. */
mov x19, x0
mov x20, x1
/* Check our current EL. We want to be executing out of EL1. */
mrs x1, currentel
/* Check if we're EL1. */
cmp x1, #0x4
b.eq 2f
/* Check if we're EL2. */
cmp x1, #0x8
b.eq 1f
0: /* We're EL3. This is a panic condition. */
b 0b
1: /* We're EL2. */
#ifdef ATMOSPHERE_BOARD_NINTENDO_NX
/* On NX board, this is a panic condition. */
b 1b
#else
/* Otherwise, deprivilege to EL2. */
/* TODO: Does N still have this? We need it for qemu emulation/unit testing, we should come up with a better solution maybe. */
bl _ZN3ams4kern4init16JumpFromEL2ToEL1Ev
#endif
2: /* We're EL1. */
/* Flush the entire data cache and invalidate the entire TLB. */
bl _ZN3ams4kern4arch5arm643cpu32FlushEntireDataCacheWithoutStackEv
/* Invalidate the instruction cache, and ensure instruction consistency. */
ic ialluis
dsb sy
isb
/* Disable the MMU/Caches. */
bl _ZN3ams4kern4init19DisableMmuAndCachesEv
#ifdef ATMOSPHERE_BOARD_NINTENDO_NX
/* Get the target firmware from exosphere. */
LOAD_IMMEDIATE_32(w0, 0xC3000004)
mov w1, #65000
smc #1
cmp x0, #0
3:
b.ne 3b
/* Store the target firmware. */
adr x0, __metadata_target_firmware
str w1, [x0]
#endif
/* Get the unknown debug region. */
/* TODO: This is always zero in release kernels -- what is this? Is it the device tree buffer? */
mov x21, #0
nop
/* We want to invoke kernel loader. */
adr x0, _start
adr x1, __metadata_kernel_layout
LOAD_RELATIVE_FROM_LABEL(x2, x4, __metadata_ini_offset)
LOAD_RELATIVE_FROM_LABEL(x3, x4, __metadata_kernelldr_offset)
/* Invoke kernel loader. */
blr x3
/* Save the offset to virtual address from this page's physical address for our use. */
mov x24, x1
/* Clear the platform register (used for Kernel::GetCurrentThreadPointer()) */
mov x18, #0
/* At this point kernelldr has been invoked, and we are relocated at a random virtual address. */
/* Next thing to do is to set up our memory management and slabheaps -- all the other core initialization. */
/* Call ams::kern::init::InitializeCore(uintptr_t, void **) */
mov x1, x0 /* Kernelldr returns a state object for the kernel to re-use. */
mov x0, x21 /* Use the address we determined earlier. */
nop
INDIRECT_RELATIVE_CALL(x16, x24, _ZN3ams4kern4init20InitializeCorePhase1EmPPv)
/* Get the init arguments for core 0. */
mov x0, xzr
nop
INDIRECT_RELATIVE_CALL(x16, x24, _ZN3ams4kern4init16GetInitArgumentsEi)
/* Setup the stack pointer. */
ldr x2, [x0, #(INIT_ARGUMENTS_SP)]
mov sp, x2
/* Perform further initialization with the stack pointer set up, as required. */
/* This will include e.g. unmapping the identity mapping. */
nop
INDIRECT_RELATIVE_CALL(x16, x24, _ZN3ams4kern4init20InitializeCorePhase2Ev)
/* Get the init arguments for core 0. */
mov x0, xzr
nop
INDIRECT_RELATIVE_CALL(x16, x24, _ZN3ams4kern4init16GetInitArgumentsEi)
/* Retrieve entrypoint and argument. */
ldr x1, [x0, #(INIT_ARGUMENTS_ENTRYPOINT)]
ldr x0, [x0, #(INIT_ARGUMENTS_ARGUMENT)]
/* Set sctlr_el1 and ensure instruction consistency. */
SETUP_SYSTEM_REGISTER(x3, sctlr_el1)
dsb sy
isb
/* Invoke the entrypoint. */
blr x1
0: /* If we return here, something has gone wrong, so wait forever. */
b 0b
/* ams::kern::init::StartOtherCore(const ams::kern::init::KInitArguments *) */
.section .crt0.text._ZN3ams4kern4init14StartOtherCoreEPKNS1_14KInitArgumentsE, "ax", %progbits
.global _ZN3ams4kern4init14StartOtherCoreEPKNS1_14KInitArgumentsE
.type _ZN3ams4kern4init14StartOtherCoreEPKNS1_14KInitArgumentsE, %function
_ZN3ams4kern4init14StartOtherCoreEPKNS1_14KInitArgumentsE:
/* Preserve the KInitArguments pointer in a register. */
mov x20, x0
/* Check our current EL. We want to be executing out of EL1. */
mrs x1, currentel
/* Check if we're EL1. */
cmp x1, #0x4
b.eq 2f
/* Check if we're EL2. */
cmp x1, #0x8
b.eq 1f
0: /* We're EL3. This is a panic condition. */
b 0b
1: /* We're EL2. */
#ifdef ATMOSPHERE_BOARD_NINTENDO_NX
/* On NX board, this is a panic condition. */
b 1b
#else
/* Otherwise, deprivilege to EL2. */
/* TODO: Does N still have this? We need it for qemu emulation/unit testing, we should come up with a better solution maybe. */
bl _ZN3ams4kern4init16JumpFromEL2ToEL1Ev
#endif
2: /* We're EL1. */
/* Flush the entire data cache and invalidate the entire TLB. */
bl _ZN3ams4kern4arch5arm643cpu32FlushEntireDataCacheWithoutStackEv
/* Invalidate the instruction cache, and ensure instruction consistency. */
ic ialluis
dsb sy
isb
/* Disable the MMU/Caches. */
bl _ZN3ams4kern4init19DisableMmuAndCachesEv
/* Setup system registers using values from constants table. */
SETUP_SYSTEM_REGISTER(x1, ttbr0_el1)
SETUP_SYSTEM_REGISTER(x1, ttbr1_el1)
SETUP_SYSTEM_REGISTER(x1, tcr_el1)
SETUP_SYSTEM_REGISTER(x1, mair_el1)
/* Perform cpu-specific setup. */
mrs x1, midr_el1
ubfx x2, x1, #0x18, #0x8 /* Extract implementer bits. */
cmp x2, #0x41 /* Implementer::ArmLimited */
b.ne 4f
ubfx x2, x1, #0x4, #0xC /* Extract primary part number. */
cmp x2, #0xD07 /* PrimaryPartNumber::CortexA57 */
b.eq 3f
cmp x2, #0xD03 /* PrimaryPartNumber::CortexA53 */
b.eq 3f
b 4f
3: /* We're running on a Cortex-A53/Cortex-A57. */
/* NOTE: Nintendo compares these values instead of setting them, infinite looping on incorrect value. */
ldr x1, [x20, #(INIT_ARGUMENTS_CPUACTLR)]
msr cpuactlr_el1, x1
ldr x1, [x20, #(INIT_ARGUMENTS_CPUECTLR)]
msr cpuectlr_el1, x1
4:
/* Ensure instruction consistency. */
dsb sy
isb
/* Load remaining needed fields from the init args. */
ldr x2, [x20, #(INIT_ARGUMENTS_SP)]
ldr x1, [x20, #(INIT_ARGUMENTS_ENTRYPOINT)]
ldr x0, [x20, #(INIT_ARGUMENTS_ARGUMENT)]
/* Set sctlr_el1 and ensure instruction consistency. */
SETUP_SYSTEM_REGISTER(x3, sctlr_el1)
dsb sy
isb
/* Set the stack pointer. */
mov sp, x2
/* Clear the platform register (used for Kernel::GetCurrentThreadPointer()) */
mov x18, #0
/* Invoke the entrypoint. */
blr x1
0: /* If we return here, something has gone wrong, so wait forever. */
b 0b
/* Nintendo places the metadata after StartOthercore. */
.align 8
__metadata_begin:
__metadata_ini_offset:
.quad 0 /* INI1 base address. */
__metadata_kernelldr_offset:
.quad 0 /* Kernel Loader base address. */
__metadata_target_firmware:
.word 0xCCCCCCCC /* Target firmware. */
__metadata_kernel_layout:
.word _start - __metadata_kernel_layout /* rx_offset */
.word __rodata_start - __metadata_kernel_layout /* rx_end_offset */
.word __rodata_start - __metadata_kernel_layout /* ro_offset */
.word __data_start - __metadata_kernel_layout /* ro_end_offset */
.word __data_start - __metadata_kernel_layout /* rw_offset */
.word __bss_start__ - __metadata_kernel_layout /* rw_end_offset */
.word __bss_start__ - __metadata_kernel_layout /* bss_offset */
.word __bss_end__ - __metadata_kernel_layout /* bss_end_offset */
.word __end__ - __metadata_kernel_layout /* resource_offset */
.word _DYNAMIC - __metadata_kernel_layout /* dynamic_offset */
.word __init_array_start - __metadata_kernel_layout /* init_array_offset */
.word __init_array_end - __metadata_kernel_layout /* init_array_end_offset */
.word __sysreg_constant_begin - __metadata_kernel_layout /* sysreg_offset */
.if (. - __metadata_begin) != 0x48
.error "Incorrect Mesosphere Metadata"
.endif
/* TODO: Can we remove this while retaining QEMU support? */
#ifndef ATMOSPHERE_BOARD_NINTENDO_NX
/* ams::kern::init::JumpFromEL2ToEL1() */
.section .crt0.text._ZN3ams4kern4init16JumpFromEL2ToEL1Ev, "ax", %progbits
.global _ZN3ams4kern4init16JumpFromEL2ToEL1Ev
.type _ZN3ams4kern4init16JumpFromEL2ToEL1Ev, %function
_ZN3ams4kern4init16JumpFromEL2ToEL1Ev:
/* We're going to want to ERET to our caller. */
msr elr_el2, x30
/* Flush the entire data cache and invalidate the entire TLB. */
bl _ZN3ams4kern4arch5arm643cpu32FlushEntireDataCacheWithoutStackEv
/* Setup system registers for deprivileging. */
/* Check if we're on cortex A57 or A53. If we are, set ACTLR_EL2. */
mrs x1, midr_el1
/* Is the manufacturer ID 'A' (ARM)? */
ubfx x2, x1, #0x18, #8
cmp x2, #0x41
b.ne 2f
/* Is the board ID Cortex-A57? */
ubfx x2, x1, #4, #0xC
mov x3, #0xD07
cmp x2, x3
b.eq 1f
/* Is the board ID Cortex-A53? */
mov x3, #0xD03
cmp x2, x3
b.ne 2f
1:
/* ACTLR_EL2: */
/* - CPUACTLR access control = 1 */
/* - CPUECTLR access control = 1 */
/* - L2CTLR access control = 1 */
/* - L2ECTLR access control = 1 */
/* - L2ACTLR access control = 1 */
mov x0, #0x73
msr actlr_el2, x0
2:
/* HCR_EL2: */
/* - RW = 1 (el1 is aarch64) */
mov x0, #0x80000000
msr hcr_el2, x0
/* SCTLR_EL1: */
/* - EOS = 1 */
/* - EIS = 1 */
/* - SPAN = 1 */
LOAD_IMMEDIATE_32(x0, 0x00C00800)
msr sctlr_el1, x0
/* DACR32_EL2: */
/* - Manager access for all D<n> */
mov x0, #0xFFFFFFFF
msr dacr32_el2, x0
/* Set VPIDR_EL2 = MIDR_EL1 */
mrs x0, midr_el1
msr vpidr_el2, x0
/* SET VMPIDR_EL2 = MPIDR_EL1 */
mrs x0, mpidr_el1
msr vmpidr_el2, x0
/* SPSR_EL2: */
/* - EL1h */
/* - IRQ masked */
/* - FIQ masked */
mov x0, #0xC5
msr spsr_el2, x0
ERET_WITH_SPECULATION_BARRIER
#endif
/* ams::kern::init::DisableMmuAndCaches() */
.section .crt0.text._ZN3ams4kern4init19DisableMmuAndCachesEv, "ax", %progbits
.global _ZN3ams4kern4init19DisableMmuAndCachesEv
.type _ZN3ams4kern4init19DisableMmuAndCachesEv, %function
_ZN3ams4kern4init19DisableMmuAndCachesEv:
/* The stack isn't set up, so we'll need to trash a register. */
mov x22, x30
/* Set SCTLR_EL1 to disable the caches and mmu. */
/* SCTLR_EL1: */
/* - M = 0 */
/* - C = 0 */
/* - I = 0 */
mrs x0, sctlr_el1
LOAD_IMMEDIATE_64(x1, ~0x1005)
and x0, x0, x1
msr sctlr_el1, x0
/* Ensure instruction consistency. */
dsb sy
isb
mov x30, x22
ret
/* ams::kern::arch::arm64::cpu::FlushEntireDataCacheSharedWithoutStack() */
.section .crt0.text._ZN3ams4kern4arch5arm643cpu38FlushEntireDataCacheSharedWithoutStackEv, "ax", %progbits
.global _ZN3ams4kern4arch5arm643cpu38FlushEntireDataCacheSharedWithoutStackEv
.type _ZN3ams4kern4arch5arm643cpu38FlushEntireDataCacheSharedWithoutStackEv, %function
_ZN3ams4kern4arch5arm643cpu38FlushEntireDataCacheSharedWithoutStackEv:
/* The stack isn't set up, so we'll need to trash a register. */
mov x24, x30
/* CacheLineIdAccessor clidr_el1; */
mrs x10, clidr_el1
/* const int levels_of_coherency = clidr_el1.GetLevelsOfCoherency(); */
ubfx x9, x10, #0x15, 3
/* const int levels_of_unification = clidr_el1.GetLevelsOfUnification(); */
ubfx x10, x10, #0x18, 3
/* int level = levels_of_unification */
/* while (level <= levels_of_coherency) { */
cmp w9, w10
b.hi 1f
0:
/* FlushEntireDataCacheImplWithoutStack(level); */
mov w0, w9
bl _ZN3ams4kern4arch5arm643cpu36FlushEntireDataCacheImplWithoutStackEv
/* level++; */
cmp w9, w10
add w9, w9, #1
/* } */
b.cc 0b
/* cpu::DataSynchronizationBarrier(); */
dsb sy
1:
mov x30, x24
ret
/* ams::kern::arch::arm64::cpu::FlushEntireDataCacheLocalWithoutStack() */
.section .crt0.text._ZN3ams4kern4arch5arm643cpu37FlushEntireDataCacheLocalWithoutStackEv, "ax", %progbits
.global _ZN3ams4kern4arch5arm643cpu37FlushEntireDataCacheLocalWithoutStackEv
.type _ZN3ams4kern4arch5arm643cpu37FlushEntireDataCacheLocalWithoutStackEv, %function
_ZN3ams4kern4arch5arm643cpu37FlushEntireDataCacheLocalWithoutStackEv:
/* The stack isn't set up, so we'll need to trash a register. */
mov x24, x30
/* CacheLineIdAccessor clidr_el1; */
mrs x10, clidr_el1
/* const int levels_of_unification = clidr_el1.GetLevelsOfUnification(); */
ubfx x10, x10, #0x15, 3
/* int level = 0 */
mov x9, xzr
/* while (level <= levels_of_unification) { */
cmp x9, x10
b.eq 1f
0:
/* FlushEntireDataCacheImplWithoutStack(level); */
mov w0, w9
bl _ZN3ams4kern4arch5arm643cpu36FlushEntireDataCacheImplWithoutStackEv
/* level++; */
add w9, w9, #1
/* } */
cmp x9, x10
b.ne 0b
/* cpu::DataSynchronizationBarrier(); */
dsb sy
1:
mov x30, x24
ret
/* ams::kern::arch::arm64::cpu::FlushEntireDataCacheWithoutStack() */
.section .crt0.text._ZN3ams4kern4arch5arm643cpu32FlushEntireDataCacheWithoutStackEv, "ax", %progbits
.global _ZN3ams4kern4arch5arm643cpu32FlushEntireDataCacheWithoutStackEv
.type _ZN3ams4kern4arch5arm643cpu32FlushEntireDataCacheWithoutStackEv, %function
_ZN3ams4kern4arch5arm643cpu32FlushEntireDataCacheWithoutStackEv:
/* The stack isn't set up, so we'll need to trash a register. */
mov x23, x30
/* Ensure that the cache is coherent. */
bl _ZN3ams4kern4arch5arm643cpu37FlushEntireDataCacheLocalWithoutStackEv
bl _ZN3ams4kern4arch5arm643cpu38FlushEntireDataCacheSharedWithoutStackEv
bl _ZN3ams4kern4arch5arm643cpu37FlushEntireDataCacheLocalWithoutStackEv
/* Invalidate the entire TLB, and ensure instruction consistency. */
tlbi vmalle1is
dsb sy
isb
mov x30, x23
ret
/* ams::kern::arch::arm64::cpu::FlushEntireDataCacheImplWithoutStack() */
.section .crt0.text._ZN3ams4kern4arch5arm643cpu36FlushEntireDataCacheImplWithoutStackEv, "ax", %progbits
.global _ZN3ams4kern4arch5arm643cpu36FlushEntireDataCacheImplWithoutStackEv
.type _ZN3ams4kern4arch5arm643cpu36FlushEntireDataCacheImplWithoutStackEv, %function
_ZN3ams4kern4arch5arm643cpu36FlushEntireDataCacheImplWithoutStackEv:
/* const u64 level_sel_value = static_cast<u64>(level << 1); */
lsl w6, w0, #1
sxtw x6, w6
/* cpu::DataSynchronizationBarrier(); */
dsb sy
/* cpu::SetCsselrEl1(level_sel_value); */
msr csselr_el1, x6
/* cpu::InstructionMemoryBarrier(); */
isb
/* CacheSizeIdAccessor ccsidr_el1; */
mrs x3, ccsidr_el1
/* const int num_ways = ccsidr_el1.GetAssociativity(); */
ubfx x7, x3, #3, #0xA
mov w8, w7
/* const int line_size = ccsidr_el1.GetLineSize(); */
and x4, x3, #7
/* const u64 way_shift = static_cast<u64>(__builtin_clz(num_ways)); */
clz w7, w7
/* const u64 set_shift = static_cast<u64>(line_size + 4); */
add w4, w4, #4
/* const int num_sets = ccsidr_el1.GetNumberOfSets(); */
ubfx w3, w3, #0xD, #0xF
/* int way = 0; */
mov x5, #0
/* while (way <= num_ways) { */
0:
cmp w8, w5
b.lt 3f
/* int set = 0; */
mov x0, #0
/* while (set <= num_sets) { */
1:
cmp w3, w0
b.lt 2f
/* const u64 cisw_value = (static_cast<u64>(way) << way_shift) | (static_cast<u64>(set) << set_shift) | level_sel_value; */
lsl x2, x5, x7
lsl x1, x0, x4
orr x1, x1, x2
orr x1, x1, x6
/* __asm__ __volatile__("dc cisw, %0" :: "r"(cisw_value) : "memory"); */
dc cisw, x1
/* set++; */
add x0, x0, #1
/* } */
b 1b
2:
/* way++; */
add x5, x5, 1
/* } */
b 0b
3:
ret
/* System register values. */
.align 8
__sysreg_constant_begin:
__sysreg_constant_ttbr0_el1:
.quad 0 /* ttbr0_e11. */
__sysreg_constant_ttbr1_el1:
.quad 0 /* ttbr1_e11. */
__sysreg_constant_tcr_el1:
.quad 0 /* tcr_e11. */
__sysreg_constant_mair_el1:
.quad 0 /* mair_e11. */
__sysreg_constant_sctlr_el1:
.quad 0 /* sctlr_e11. */
.if (. - __sysreg_constant_begin) != 0x28
.error "Incorrect System Registers"
.endif
/* ================ Functions before this line remain identity-mapped after initialization finishes. ================ */
/* ams::kern::init::IdentityMappedFunctionAreaEnd() */
.global _ZN3ams4kern4init29IdentityMappedFunctionAreaEndEv
.type _ZN3ams4kern4init31IdentityMappedFunctionAreaEndEv, %function
_ZN3ams4kern4init29IdentityMappedFunctionAreaEndEv:
/* NOTE: This is not a real function, and only exists as a label for safety. */

659
Source/Atmosphere-MTC-Unlock/mesosphere/kernel/source/arch/arm64/kern_exception_handlers_asm.s
View File

@@ -1,659 +0,0 @@
/*
* Copyright (c) Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <mesosphere/kern_select_assembly_macros.h>
/* ams::kern::arch::arm64::EL1IrqExceptionHandler() */
.section .text._ZN3ams4kern4arch5arm6422EL1IrqExceptionHandlerEv, "ax", %progbits
.global _ZN3ams4kern4arch5arm6422EL1IrqExceptionHandlerEv
.type _ZN3ams4kern4arch5arm6422EL1IrqExceptionHandlerEv, %function
_ZN3ams4kern4arch5arm6422EL1IrqExceptionHandlerEv:
/* Save registers that need saving. */
sub sp, sp, #(8 * 24)
stp x0, x1, [sp, #(8 * 0)]
stp x2, x3, [sp, #(8 * 2)]
stp x4, x5, [sp, #(8 * 4)]
stp x6, x7, [sp, #(8 * 6)]
stp x8, x9, [sp, #(8 * 8)]
stp x10, x11, [sp, #(8 * 10)]
stp x12, x13, [sp, #(8 * 12)]
stp x14, x15, [sp, #(8 * 14)]
stp x16, x17, [sp, #(8 * 16)]
stp x19, x20, [sp, #(8 * 18)]
stp x21, x30, [sp, #(8 * 20)]
mrs x19, sp_el0
mrs x20, elr_el1
mrs x21, spsr_el1
mov w21, w21
/* Invoke KInterruptManager::HandleInterrupt(bool user_mode). */
mov x0, #0
bl _ZN3ams4kern4arch5arm6417KInterruptManager15HandleInterruptEb
/* Restore registers that we saved. */
msr sp_el0, x19
msr elr_el1, x20
msr spsr_el1, x21
ldp x0, x1, [sp, #(8 * 0)]
ldp x2, x3, [sp, #(8 * 2)]
ldp x4, x5, [sp, #(8 * 4)]
ldp x6, x7, [sp, #(8 * 6)]
ldp x8, x9, [sp, #(8 * 8)]
ldp x10, x11, [sp, #(8 * 10)]
ldp x12, x13, [sp, #(8 * 12)]
ldp x14, x15, [sp, #(8 * 14)]
ldp x16, x17, [sp, #(8 * 16)]
ldp x19, x20, [sp, #(8 * 18)]
ldp x21, x30, [sp, #(8 * 20)]
add sp, sp, #(8 * 24)
/* Return from the exception. */
ERET_WITH_SPECULATION_BARRIER
/* ams::kern::arch::arm64::EL0A64IrqExceptionHandler() */
.section .text._ZN3ams4kern4arch5arm6425EL0A64IrqExceptionHandlerEv, "ax", %progbits
.global _ZN3ams4kern4arch5arm6425EL0A64IrqExceptionHandlerEv
.type _ZN3ams4kern4arch5arm6425EL0A64IrqExceptionHandlerEv, %function
_ZN3ams4kern4arch5arm6425EL0A64IrqExceptionHandlerEv:
/* Save registers that need saving. */
sub sp, sp, #(EXCEPTION_CONTEXT_SIZE)
stp x0, x1, [sp, #(EXCEPTION_CONTEXT_X0_X1)]
stp x2, x3, [sp, #(EXCEPTION_CONTEXT_X2_X3)]
stp x4, x5, [sp, #(EXCEPTION_CONTEXT_X4_X5)]
stp x6, x7, [sp, #(EXCEPTION_CONTEXT_X6_X7)]
stp x8, x9, [sp, #(EXCEPTION_CONTEXT_X8_X9)]
stp x10, x11, [sp, #(EXCEPTION_CONTEXT_X10_X11)]
stp x12, x13, [sp, #(EXCEPTION_CONTEXT_X12_X13)]
stp x14, x15, [sp, #(EXCEPTION_CONTEXT_X14_X15)]
stp x16, x17, [sp, #(EXCEPTION_CONTEXT_X16_X17)]
stp x18, x19, [sp, #(EXCEPTION_CONTEXT_X18_X19)]
stp x20, x21, [sp, #(EXCEPTION_CONTEXT_X20_X21)]
stp x22, x23, [sp, #(EXCEPTION_CONTEXT_X22_X23)]
stp x24, x25, [sp, #(EXCEPTION_CONTEXT_X24_X25)]
stp x26, x27, [sp, #(EXCEPTION_CONTEXT_X26_X27)]
stp x28, x29, [sp, #(EXCEPTION_CONTEXT_X28_X29)]
mrs x20, sp_el0
mrs x21, elr_el1
mrs x22, spsr_el1
mrs x23, tpidr_el0
mov w22, w22
stp x30, x20, [sp, #(EXCEPTION_CONTEXT_X30_SP)]
stp x21, x22, [sp, #(EXCEPTION_CONTEXT_PC_PSR)]
str x23, [sp, #(EXCEPTION_CONTEXT_TPIDR)]
/* Invoke KInterruptManager::HandleInterrupt(bool user_mode). */
ldr x18, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_CUR_THREAD)]
mov x0, #1
bl _ZN3ams4kern4arch5arm6417KInterruptManager15HandleInterruptEb
/* If we don't need to restore the fpu, skip restoring it. */
ldrb w1, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
tbz w1, #(THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_FPU_CONTEXT_RESTORE_NEEDED), 1f
/* Clear the needs-fpu-restore flag. */
and w1, w1, #(~THREAD_EXCEPTION_FLAG_IS_FPU_CONTEXT_RESTORE_NEEDED)
strb w1, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
/* Perform a full fpu restore. */
ENABLE_AND_RESTORE_FPU64(x2, x0, x1, w0, w1)
1: /* Restore state from the context. */
ldp x30, x20, [sp, #(EXCEPTION_CONTEXT_X30_SP)]
ldp x21, x22, [sp, #(EXCEPTION_CONTEXT_PC_PSR)]
ldr x23, [sp, #(EXCEPTION_CONTEXT_TPIDR)]
#if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP)
/* Since we're returning from an exception, set SPSR.SS so that we advance an instruction if single-stepping. */
orr x22, x22, #(1 << 21)
#endif
msr sp_el0, x20
msr elr_el1, x21
msr spsr_el1, x22
msr tpidr_el0, x23
ldp x0, x1, [sp, #(EXCEPTION_CONTEXT_X0_X1)]
ldp x2, x3, [sp, #(EXCEPTION_CONTEXT_X2_X3)]
ldp x4, x5, [sp, #(EXCEPTION_CONTEXT_X4_X5)]
ldp x6, x7, [sp, #(EXCEPTION_CONTEXT_X6_X7)]
ldp x8, x9, [sp, #(EXCEPTION_CONTEXT_X8_X9)]
ldp x10, x11, [sp, #(EXCEPTION_CONTEXT_X10_X11)]
ldp x12, x13, [sp, #(EXCEPTION_CONTEXT_X12_X13)]
ldp x14, x15, [sp, #(EXCEPTION_CONTEXT_X14_X15)]
ldp x16, x17, [sp, #(EXCEPTION_CONTEXT_X16_X17)]
ldp x18, x19, [sp, #(EXCEPTION_CONTEXT_X18_X19)]
ldp x20, x21, [sp, #(EXCEPTION_CONTEXT_X20_X21)]
ldp x22, x23, [sp, #(EXCEPTION_CONTEXT_X22_X23)]
ldp x24, x25, [sp, #(EXCEPTION_CONTEXT_X24_X25)]
ldp x26, x27, [sp, #(EXCEPTION_CONTEXT_X26_X27)]
ldp x28, x29, [sp, #(EXCEPTION_CONTEXT_X28_X29)]
add sp, sp, #(EXCEPTION_CONTEXT_SIZE)
/* Return from the exception. */
ERET_WITH_SPECULATION_BARRIER
/* ams::kern::arch::arm64::EL0A32IrqExceptionHandler() */
.section .text._ZN3ams4kern4arch5arm6425EL0A32IrqExceptionHandlerEv, "ax", %progbits
.global _ZN3ams4kern4arch5arm6425EL0A32IrqExceptionHandlerEv
.type _ZN3ams4kern4arch5arm6425EL0A32IrqExceptionHandlerEv, %function
_ZN3ams4kern4arch5arm6425EL0A32IrqExceptionHandlerEv:
/* Save registers that need saving. */
sub sp, sp, #(EXCEPTION_CONTEXT_SIZE)
stp x0, x1, [sp, #(EXCEPTION_CONTEXT_X0_X1)]
stp x2, x3, [sp, #(EXCEPTION_CONTEXT_X2_X3)]
stp x4, x5, [sp, #(EXCEPTION_CONTEXT_X4_X5)]
stp x6, x7, [sp, #(EXCEPTION_CONTEXT_X6_X7)]
stp x8, x9, [sp, #(EXCEPTION_CONTEXT_X8_X9)]
stp x10, x11, [sp, #(EXCEPTION_CONTEXT_X10_X11)]
stp x12, x13, [sp, #(EXCEPTION_CONTEXT_X12_X13)]
stp x14, x15, [sp, #(EXCEPTION_CONTEXT_X14_X15)]
mrs x21, elr_el1
mrs x22, spsr_el1
mrs x23, tpidr_el0
mov w22, w22
stp x21, x22, [sp, #(EXCEPTION_CONTEXT_PC_PSR)]
str x23, [sp, #(EXCEPTION_CONTEXT_TPIDR)]
/* Invoke KInterruptManager::HandleInterrupt(bool user_mode). */
ldr x18, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_CUR_THREAD)]
mov x0, #1
bl _ZN3ams4kern4arch5arm6417KInterruptManager15HandleInterruptEb
/* If we don't need to restore the fpu, skip restoring it. */
ldrb w1, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
tbz w1, #(THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_FPU_CONTEXT_RESTORE_NEEDED), 1f
/* Clear the needs-fpu-restore flag. */
and w1, w1, #(~THREAD_EXCEPTION_FLAG_IS_FPU_CONTEXT_RESTORE_NEEDED)
strb w1, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
/* Perform a full fpu restore. */
ENABLE_AND_RESTORE_FPU32(x2, x0, x1, w0, w1)
1: /* Restore state from the context. */
ldp x21, x22, [sp, #(EXCEPTION_CONTEXT_PC_PSR)]
ldr x23, [sp, #(EXCEPTION_CONTEXT_TPIDR)]
#if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP)
/* Since we're returning from an exception, set SPSR.SS so that we advance an instruction if single-stepping. */
orr x22, x22, #(1 << 21)
#endif
msr elr_el1, x21
msr spsr_el1, x22
msr tpidr_el0, x23
ldp x0, x1, [sp, #(EXCEPTION_CONTEXT_X0_X1)]
ldp x2, x3, [sp, #(EXCEPTION_CONTEXT_X2_X3)]
ldp x4, x5, [sp, #(EXCEPTION_CONTEXT_X4_X5)]
ldp x6, x7, [sp, #(EXCEPTION_CONTEXT_X6_X7)]
ldp x8, x9, [sp, #(EXCEPTION_CONTEXT_X8_X9)]
ldp x10, x11, [sp, #(EXCEPTION_CONTEXT_X10_X11)]
ldp x12, x13, [sp, #(EXCEPTION_CONTEXT_X12_X13)]
ldp x14, x15, [sp, #(EXCEPTION_CONTEXT_X14_X15)]
add sp, sp, #(EXCEPTION_CONTEXT_SIZE)
/* Return from the exception. */
ERET_WITH_SPECULATION_BARRIER
/* ams::kern::arch::arm64::EL0SynchronousExceptionHandler() */
.section .text._ZN3ams4kern4arch5arm6430EL0SynchronousExceptionHandlerEv, "ax", %progbits
.global _ZN3ams4kern4arch5arm6430EL0SynchronousExceptionHandlerEv
.type _ZN3ams4kern4arch5arm6430EL0SynchronousExceptionHandlerEv, %function
_ZN3ams4kern4arch5arm6430EL0SynchronousExceptionHandlerEv:
/* Save x16 and x17, so that we can use them as scratch. */
stp x16, x17, [sp, #-16]!
/* Get and parse the exception syndrome register. */
mrs x16, esr_el1
lsr x17, x16, #0x1a
/* Is this an aarch32 SVC? */
cmp x17, #0x11
b.eq 4f
/* Is this an aarch64 SVC? */
cmp x17, #0x15
b.eq 5f
/* Is this an FPU error? */
cmp x17, #0x7
b.eq 6f
/* Is this a data abort? */
cmp x17, #0x24
b.eq 7f
/* Is this an instruction abort? */
cmp x17, #0x20
b.eq 7f
1: /* The exception is not a data abort or instruction abort caused by a TLB conflict. */
/* It is also not an SVC or an FPU exception. Handle it generically! */
/* Restore x16 and x17. */
ldp x16, x17, [sp], 16
/* Create a KExceptionContext to pass to HandleException. */
sub sp, sp, #(EXCEPTION_CONTEXT_SIZE)
stp x0, x1, [sp, #(EXCEPTION_CONTEXT_X0_X1)]
stp x2, x3, [sp, #(EXCEPTION_CONTEXT_X2_X3)]
stp x4, x5, [sp, #(EXCEPTION_CONTEXT_X4_X5)]
stp x6, x7, [sp, #(EXCEPTION_CONTEXT_X6_X7)]
stp x8, x9, [sp, #(EXCEPTION_CONTEXT_X8_X9)]
stp x10, x11, [sp, #(EXCEPTION_CONTEXT_X10_X11)]
stp x12, x13, [sp, #(EXCEPTION_CONTEXT_X12_X13)]
stp x14, x15, [sp, #(EXCEPTION_CONTEXT_X14_X15)]
stp x16, x17, [sp, #(EXCEPTION_CONTEXT_X16_X17)]
stp x18, x19, [sp, #(EXCEPTION_CONTEXT_X18_X19)]
stp x20, x21, [sp, #(EXCEPTION_CONTEXT_X20_X21)]
stp x22, x23, [sp, #(EXCEPTION_CONTEXT_X22_X23)]
stp x24, x25, [sp, #(EXCEPTION_CONTEXT_X24_X25)]
stp x26, x27, [sp, #(EXCEPTION_CONTEXT_X26_X27)]
stp x28, x29, [sp, #(EXCEPTION_CONTEXT_X28_X29)]
mrs x20, sp_el0
mrs x21, elr_el1
mrs x22, spsr_el1
mrs x23, tpidr_el0
mov w22, w22
stp x30, x20, [sp, #(EXCEPTION_CONTEXT_X30_SP)]
stp x21, x22, [sp, #(EXCEPTION_CONTEXT_PC_PSR)]
str x23, [sp, #(EXCEPTION_CONTEXT_TPIDR)]
/* Call ams::kern::arch::arm64::HandleException(ams::kern::arch::arm64::KExceptionContext *) */
ldr x18, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_CUR_THREAD)]
mov x0, sp
bl _ZN3ams4kern4arch5arm6415HandleExceptionEPNS2_17KExceptionContextE
/* If we don't need to restore the fpu, skip restoring it. */
ldrb w1, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
tbz w1, #(THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_FPU_CONTEXT_RESTORE_NEEDED), 3f
/* Clear the needs-fpu-restore flag. */
and w1, w1, #(~THREAD_EXCEPTION_FLAG_IS_FPU_CONTEXT_RESTORE_NEEDED)
strb w1, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
/* Enable and restore the fpu. */
ENABLE_AND_RESTORE_FPU(x2, x0, x1, w0, w1, 2, 3)
/* Restore state from the context. */
ldp x30, x20, [sp, #(EXCEPTION_CONTEXT_X30_SP)]
ldp x21, x22, [sp, #(EXCEPTION_CONTEXT_PC_PSR)]
ldr x23, [sp, #(EXCEPTION_CONTEXT_TPIDR)]
msr sp_el0, x20
msr elr_el1, x21
msr spsr_el1, x22
msr tpidr_el0, x23
ldp x0, x1, [sp, #(EXCEPTION_CONTEXT_X0_X1)]
ldp x2, x3, [sp, #(EXCEPTION_CONTEXT_X2_X3)]
ldp x4, x5, [sp, #(EXCEPTION_CONTEXT_X4_X5)]
ldp x6, x7, [sp, #(EXCEPTION_CONTEXT_X6_X7)]
ldp x8, x9, [sp, #(EXCEPTION_CONTEXT_X8_X9)]
ldp x10, x11, [sp, #(EXCEPTION_CONTEXT_X10_X11)]
ldp x12, x13, [sp, #(EXCEPTION_CONTEXT_X12_X13)]
ldp x14, x15, [sp, #(EXCEPTION_CONTEXT_X14_X15)]
ldp x16, x17, [sp, #(EXCEPTION_CONTEXT_X16_X17)]
ldp x18, x19, [sp, #(EXCEPTION_CONTEXT_X18_X19)]
ldp x20, x21, [sp, #(EXCEPTION_CONTEXT_X20_X21)]
ldp x22, x23, [sp, #(EXCEPTION_CONTEXT_X22_X23)]
ldp x24, x25, [sp, #(EXCEPTION_CONTEXT_X24_X25)]
ldp x26, x27, [sp, #(EXCEPTION_CONTEXT_X26_X27)]
ldp x28, x29, [sp, #(EXCEPTION_CONTEXT_X28_X29)]
add sp, sp, #(EXCEPTION_CONTEXT_SIZE)
/* Return from the exception. */
ERET_WITH_SPECULATION_BARRIER
4: /* SVC from aarch32. */
ldp x16, x17, [sp], 16
b _ZN3ams4kern4arch5arm6412SvcHandler32Ev
5: /* SVC from aarch64. */
ldp x16, x17, [sp], 16
b _ZN3ams4kern4arch5arm6412SvcHandler64Ev
6: /* FPU exception. */
ldp x16, x17, [sp], 16
b _ZN3ams4kern4arch5arm6425FpuAccessExceptionHandlerEv
7: /* Check if there's a TLB conflict that caused the abort. */
and x17, x16, #0x3F
cmp x17, #0x30
b.ne 1b
/* Get the ASID in x17. */
mrs x17, ttbr0_el1
and x17, x17, #(0xFFFF << 48)
/* Invalidate the address held by FAR (and assume it is valid). */
mrs x16, far_el1
lsr x16, x16, #12
orr x17, x16, x17
tlbi vae1, x17
/* Ensure instruction consistency. */
dsb ish
isb
/* Restore x16 and x17. */
ldp x16, x17, [sp], 16
/* Return from the exception. */
ERET_WITH_SPECULATION_BARRIER
/* ams::kern::arch::arm64::EL1SynchronousExceptionHandler() */
.section .text._ZN3ams4kern4arch5arm6430EL1SynchronousExceptionHandlerEv, "ax", %progbits
.global _ZN3ams4kern4arch5arm6430EL1SynchronousExceptionHandlerEv
.type _ZN3ams4kern4arch5arm6430EL1SynchronousExceptionHandlerEv, %function
_ZN3ams4kern4arch5arm6430EL1SynchronousExceptionHandlerEv:
/* Nintendo uses tpidr_el1 as a scratch register. */
msr tpidr_el1, x0
/* Get and parse the exception syndrome register. */
mrs x0, esr_el1
lsr x0, x0, #0x1a
/* Is this an instruction abort? */
cmp x0, #0x21
b.eq 4f
/* Is this a data abort? */
cmp x0, #0x25
b.eq 4f
1: /* The exception is not a data abort or instruction abort caused by a TLB conflict. */
/* Load the exception stack top from otherwise "unused" virtual timer compare value. */
mrs x0, cntv_cval_el0
/* Setup the stack for a generic exception handle */
lsl x0, x0, #8
asr x0, x0, #8
sub x0, x0, #0x20
str x1, [x0, #8]
mov x1, sp
str x1, [x0]
mov sp, x0
ldr x1, [x0, #8]
mrs x0, tpidr_el1
str x0, [sp, #8]
str x1, [sp, #16]
/* Check again if this is a data abort from EL1. */
mrs x0, esr_el1
lsr x1, x0, #0x1a
cmp x1, #0x25
b.ne 2f
/* Data abort. Check if it was from trying to access userspace memory. */
mrs x1, elr_el1
adr x0, _ZN3ams4kern4arch5arm6432UserspaceAccessFunctionAreaBeginEv
cmp x1, x0
b.lo 2f
adr x0, _ZN3ams4kern4arch5arm6430UserspaceAccessFunctionAreaEndEv
cmp x1, x0
b.hs 2f
/* We aborted trying to access userspace memory. */
/* All functions that access user memory return a boolean for whether they succeeded. */
/* With that in mind, we can simply restore the stack pointer and return false directly. */
ldr x0, [sp]
mov sp, x0
/* Return false. */
mov x0, #0x0
msr elr_el1, x30
ERET_WITH_SPECULATION_BARRIER
2: /* The exception wasn't an triggered by copying memory from userspace. */
/* NOTE: The following is, as of 19.0.0, now ifdef'd out on NX non-debug kernel. */
ldr x0, [sp, #8]
ldr x1, [sp, #16]
/* Create a KExceptionContext to pass to HandleException. */
sub sp, sp, #(EXCEPTION_CONTEXT_SIZE)
stp x0, x1, [sp, #(EXCEPTION_CONTEXT_X0_X1)]
stp x2, x3, [sp, #(EXCEPTION_CONTEXT_X2_X3)]
stp x4, x5, [sp, #(EXCEPTION_CONTEXT_X4_X5)]
stp x6, x7, [sp, #(EXCEPTION_CONTEXT_X6_X7)]
stp x8, x9, [sp, #(EXCEPTION_CONTEXT_X8_X9)]
stp x10, x11, [sp, #(EXCEPTION_CONTEXT_X10_X11)]
stp x12, x13, [sp, #(EXCEPTION_CONTEXT_X12_X13)]
stp x14, x15, [sp, #(EXCEPTION_CONTEXT_X14_X15)]
stp x16, x17, [sp, #(EXCEPTION_CONTEXT_X16_X17)]
stp x18, x19, [sp, #(EXCEPTION_CONTEXT_X18_X19)]
stp x20, x21, [sp, #(EXCEPTION_CONTEXT_X20_X21)]
stp x22, x23, [sp, #(EXCEPTION_CONTEXT_X22_X23)]
stp x24, x25, [sp, #(EXCEPTION_CONTEXT_X24_X25)]
stp x26, x27, [sp, #(EXCEPTION_CONTEXT_X26_X27)]
stp x28, x29, [sp, #(EXCEPTION_CONTEXT_X28_X29)]
ldr x20, [sp]
mrs x21, elr_el1
mrs x22, spsr_el1
mrs x23, tpidr_el0
mov w22, w22
stp x30, x20, [sp, #(EXCEPTION_CONTEXT_X30_SP)]
stp x21, x22, [sp, #(EXCEPTION_CONTEXT_PC_PSR)]
str x23, [sp, #(EXCEPTION_CONTEXT_TPIDR)]
/* Call ams::kern::arch::arm64::HandleException(ams::kern::arch::arm64::KExceptionContext *) */
mov x0, sp
bl _ZN3ams4kern4arch5arm6415HandleExceptionEPNS2_17KExceptionContextE
3: /* HandleException should never return. The best we can do is infinite loop. */
b 3b
4: /* Check if there's a TLB conflict that caused the abort. */
mrs x0, esr_el1
and x0, x0, #0x3F
cmp x0, #0x30
b.ne 1b
/* Invalidate the address held by FAR (and assume it is valid). */
mrs x0, far_el1
lsr x0, x0, #12
tlbi vaae1, x0
/* Ensure instruction consistency. */
dsb ish
isb
/* Restore x0 from scratch. */
mrs x0, tpidr_el1
/* Return from the exception. */
ERET_WITH_SPECULATION_BARRIER
/* ams::kern::arch::arm64::FpuAccessExceptionHandler() */
.section .text._ZN3ams4kern4arch5arm6425FpuAccessExceptionHandlerEv, "ax", %progbits
.global _ZN3ams4kern4arch5arm6425FpuAccessExceptionHandlerEv
.type _ZN3ams4kern4arch5arm6425FpuAccessExceptionHandlerEv, %function
_ZN3ams4kern4arch5arm6425FpuAccessExceptionHandlerEv:
/* Save registers. */
sub sp, sp, #(EXCEPTION_CONTEXT_SIZE)
stp x0, x1, [sp, #(EXCEPTION_CONTEXT_X0_X1)]
stp x2, x3, [sp, #(EXCEPTION_CONTEXT_X2_X3)]
ENABLE_AND_RESTORE_FPU(x2, x0, x1, w0, w1, 1, 2)
/* Restore registers. */
ldp x0, x1, [sp, #(EXCEPTION_CONTEXT_X0_X1)]
ldp x2, x3, [sp, #(EXCEPTION_CONTEXT_X2_X3)]
add sp, sp, #(EXCEPTION_CONTEXT_SIZE)
/* Return from the exception. */
ERET_WITH_SPECULATION_BARRIER
/* ams::kern::arch::arm64::EL1SystemErrorHandler() */
.section .text._ZN3ams4kern4arch5arm6421EL1SystemErrorHandlerEv, "ax", %progbits
.global _ZN3ams4kern4arch5arm6421EL1SystemErrorHandlerEv
.type _ZN3ams4kern4arch5arm6421EL1SystemErrorHandlerEv, %function
_ZN3ams4kern4arch5arm6421EL1SystemErrorHandlerEv:
/* Nintendo uses tpidr_el1 as a scratch register. */
msr tpidr_el1, x0
/* Load the exception stack top from otherwise "unused" virtual timer compare value. */
mrs x0, cntv_cval_el0
/* Setup the stack for a generic exception handle */
lsl x0, x0, #8
asr x0, x0, #8
sub x0, x0, #0x20
str x1, [x0, #8]
mov x1, sp
str x1, [x0]
mov sp, x0
ldr x1, [x0, #8]
mrs x0, tpidr_el1
/* Create a KExceptionContext to pass to HandleException. */
sub sp, sp, #(EXCEPTION_CONTEXT_SIZE)
stp x0, x1, [sp, #(EXCEPTION_CONTEXT_X0_X1)]
stp x2, x3, [sp, #(EXCEPTION_CONTEXT_X2_X3)]
stp x4, x5, [sp, #(EXCEPTION_CONTEXT_X4_X5)]
stp x6, x7, [sp, #(EXCEPTION_CONTEXT_X6_X7)]
stp x8, x9, [sp, #(EXCEPTION_CONTEXT_X8_X9)]
stp x10, x11, [sp, #(EXCEPTION_CONTEXT_X10_X11)]
stp x12, x13, [sp, #(EXCEPTION_CONTEXT_X12_X13)]
stp x14, x15, [sp, #(EXCEPTION_CONTEXT_X14_X15)]
stp x16, x17, [sp, #(EXCEPTION_CONTEXT_X16_X17)]
stp x18, x19, [sp, #(EXCEPTION_CONTEXT_X18_X19)]
stp x20, x21, [sp, #(EXCEPTION_CONTEXT_X20_X21)]
stp x22, x23, [sp, #(EXCEPTION_CONTEXT_X22_X23)]
stp x24, x25, [sp, #(EXCEPTION_CONTEXT_X24_X25)]
stp x26, x27, [sp, #(EXCEPTION_CONTEXT_X26_X27)]
stp x28, x29, [sp, #(EXCEPTION_CONTEXT_X28_X29)]
ldr x20, [sp]
mrs x21, elr_el1
mrs x22, spsr_el1
mrs x23, tpidr_el0
mov w22, w22
stp x30, x20, [sp, #(EXCEPTION_CONTEXT_X30_SP)]
stp x21, x22, [sp, #(EXCEPTION_CONTEXT_PC_PSR)]
str x23, [sp, #(EXCEPTION_CONTEXT_TPIDR)]
/* Invoke ams::kern::arch::arm64::HandleException(ams::kern::arch::arm64::KExceptionContext *). */
mov x0, sp
bl _ZN3ams4kern4arch5arm6415HandleExceptionEPNS2_17KExceptionContextE
1: /* HandleException should never return. The best we can do is infinite loop. */
b 1b
/* ams::kern::arch::arm64::EL0SystemErrorHandler() */
.section .text._ZN3ams4kern4arch5arm6421EL0SystemErrorHandlerEv, "ax", %progbits
.global _ZN3ams4kern4arch5arm6421EL0SystemErrorHandlerEv
.type _ZN3ams4kern4arch5arm6421EL0SystemErrorHandlerEv, %function
_ZN3ams4kern4arch5arm6421EL0SystemErrorHandlerEv:
/* Create a KExceptionContext to pass to HandleException. */
sub sp, sp, #(EXCEPTION_CONTEXT_SIZE)
stp x0, x1, [sp, #(EXCEPTION_CONTEXT_X0_X1)]
stp x2, x3, [sp, #(EXCEPTION_CONTEXT_X2_X3)]
stp x4, x5, [sp, #(EXCEPTION_CONTEXT_X4_X5)]
stp x6, x7, [sp, #(EXCEPTION_CONTEXT_X6_X7)]
stp x8, x9, [sp, #(EXCEPTION_CONTEXT_X8_X9)]
stp x10, x11, [sp, #(EXCEPTION_CONTEXT_X10_X11)]
stp x12, x13, [sp, #(EXCEPTION_CONTEXT_X12_X13)]
stp x14, x15, [sp, #(EXCEPTION_CONTEXT_X14_X15)]
stp x16, x17, [sp, #(EXCEPTION_CONTEXT_X16_X17)]
stp x18, x19, [sp, #(EXCEPTION_CONTEXT_X18_X19)]
stp x20, x21, [sp, #(EXCEPTION_CONTEXT_X20_X21)]
stp x22, x23, [sp, #(EXCEPTION_CONTEXT_X22_X23)]
stp x24, x25, [sp, #(EXCEPTION_CONTEXT_X24_X25)]
stp x26, x27, [sp, #(EXCEPTION_CONTEXT_X26_X27)]
stp x28, x29, [sp, #(EXCEPTION_CONTEXT_X28_X29)]
mrs x20, sp_el0
mrs x21, elr_el1
mrs x22, spsr_el1
mrs x23, tpidr_el0
mov w22, w22
stp x30, x20, [sp, #(EXCEPTION_CONTEXT_X30_SP)]
stp x21, x22, [sp, #(EXCEPTION_CONTEXT_PC_PSR)]
str x23, [sp, #(EXCEPTION_CONTEXT_TPIDR)]
/* Invoke ams::kern::arch::arm64::HandleException(ams::kern::arch::arm64::KExceptionContext *). */
ldr x18, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_CUR_THREAD)]
mov x0, sp
bl _ZN3ams4kern4arch5arm6415HandleExceptionEPNS2_17KExceptionContextE
/* If we don't need to restore the fpu, skip restoring it. */
ldrb w1, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
tbz w1, #(THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_FPU_CONTEXT_RESTORE_NEEDED), 2f
/* Clear the needs-fpu-restore flag. */
and w1, w1, #(~THREAD_EXCEPTION_FLAG_IS_FPU_CONTEXT_RESTORE_NEEDED)
strb w1, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
/* Enable and restore the fpu. */
ENABLE_AND_RESTORE_FPU(x2, x0, x1, w0, w1, 1, 2)
/* Restore state from the context. */
ldp x30, x20, [sp, #(EXCEPTION_CONTEXT_X30_SP)]
ldp x21, x22, [sp, #(EXCEPTION_CONTEXT_PC_PSR)]
ldr x23, [sp, #(EXCEPTION_CONTEXT_TPIDR)]
msr sp_el0, x20
msr elr_el1, x21
msr spsr_el1, x22
msr tpidr_el0, x23
ldp x0, x1, [sp, #(EXCEPTION_CONTEXT_X0_X1)]
ldp x2, x3, [sp, #(EXCEPTION_CONTEXT_X2_X3)]
ldp x4, x5, [sp, #(EXCEPTION_CONTEXT_X4_X5)]
ldp x6, x7, [sp, #(EXCEPTION_CONTEXT_X6_X7)]
ldp x8, x9, [sp, #(EXCEPTION_CONTEXT_X8_X9)]
ldp x10, x11, [sp, #(EXCEPTION_CONTEXT_X10_X11)]
ldp x12, x13, [sp, #(EXCEPTION_CONTEXT_X12_X13)]
ldp x14, x15, [sp, #(EXCEPTION_CONTEXT_X14_X15)]
ldp x16, x17, [sp, #(EXCEPTION_CONTEXT_X16_X17)]
ldp x18, x19, [sp, #(EXCEPTION_CONTEXT_X18_X19)]
ldp x20, x21, [sp, #(EXCEPTION_CONTEXT_X20_X21)]
ldp x22, x23, [sp, #(EXCEPTION_CONTEXT_X22_X23)]
ldp x24, x25, [sp, #(EXCEPTION_CONTEXT_X24_X25)]
ldp x26, x27, [sp, #(EXCEPTION_CONTEXT_X26_X27)]
ldp x28, x29, [sp, #(EXCEPTION_CONTEXT_X28_X29)]
add sp, sp, #(EXCEPTION_CONTEXT_SIZE)
/* Return from the exception. */
ERET_WITH_SPECULATION_BARRIER

283
Source/Atmosphere-MTC-Unlock/mesosphere/kernel/source/arch/arm64/kern_k_scheduler_asm.s
View File

@@ -1,283 +0,0 @@
/*
* Copyright (c) Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <mesosphere/kern_select_assembly_macros.h>
/* ams::kern::KScheduler::ScheduleImpl() */
.section .text._ZN3ams4kern10KScheduler12ScheduleImplEv, "ax", %progbits
.global _ZN3ams4kern10KScheduler12ScheduleImplEv
.type _ZN3ams4kern10KScheduler12ScheduleImplEv, %function
/* Ensure ScheduleImpl is aligned to 0x40 bytes. */
.balign 0x40
_ZN3ams4kern10KScheduler12ScheduleImplEv:
/* Right now, x0 contains (this). We want x1 to point to the scheduling state, */
/* KScheduler layout has state at +0x0, this is guaranteed statically by assembly offsets. */
mov x1, x0
/* First, clear the need's scheduling bool (and dmb ish after, as it's an atomic). */
strb wzr, [x1]
dmb ish
/* Check whether there are runnable interrupt tasks. */
ldrb w8, [x1, #(KSCHEDULER_INTERRUPT_TASK_RUNNABLE)]
cbnz w8, 0f
/* If it isn't, we want to check if the highest priority thread is the same as the current thread. */
ldr x7, [x1, #(KSCHEDULER_HIGHEST_PRIORITY_THREAD)]
cmp x7, x18
b.ne 1f
/* If they're the same, then we can just issue a memory barrier and return. */
dmb ish
ret
0: /* The interrupt task thread is runnable. */
/* We want to switch to the interrupt task/idle thread. */
mov x7, #0
1: /* The highest priority thread is not the same as the current thread. */
/* Get a reference to the current thread's stack parameters. */
add x2, sp, #0x1000
and x2, x2, #~(0x1000-1)
sub x2, x2, #(THREAD_STACK_PARAMETERS_SIZE)
/* Get a reference to the current thread's context. */
add x3, x2, #(THREAD_STACK_PARAMETERS_THREAD_CONTEXT)
/* Save the callee-save registers + SP. */
stp x19, x20, [x3, #(THREAD_CONTEXT_X19_X20)]
stp x21, x22, [x3, #(THREAD_CONTEXT_X21_X22)]
stp x23, x24, [x3, #(THREAD_CONTEXT_X23_X24)]
stp x25, x26, [x3, #(THREAD_CONTEXT_X25_X26)]
stp x27, x28, [x3, #(THREAD_CONTEXT_X27_X28)]
stp x29, x30, [x3, #(THREAD_CONTEXT_X29_X30)]
mov x4, sp
str x4, [x3, #(THREAD_CONTEXT_SP)]
/* Check if the fpu is enabled; if it is, we need to save it. */
mrs x5, cpacr_el1
and x4, x5, #0x300000
cbz x4, 8f
/* We need to save the fpu state; save fpsr/fpcr. */
mrs x4, fpcr
mrs x6, fpsr
stp w4, w6, [x3, #(THREAD_CONTEXT_FPCR_FPSR)]
/* Set fpu-restore-needed in our exception flags. */
ldrb w4, [x2, #(THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
orr w4, w4, #(THREAD_EXCEPTION_FLAG_IS_FPU_CONTEXT_RESTORE_NEEDED)
strb w4, [x2, #(THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
/* We need to save fpu state based on whether we're a 64-bit or 32-bit thread. */
tbz w4, #(THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_FPU_64_BIT), 4f
/* We have a 64-bit fpu. */
/* If we're in a usermode exception, we need to save the caller-save fpu registers. */
tbz w4, #(THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_IN_USERMODE_EXCEPTION_HANDLER), 2f
/* If we're in an SVC (and not a usermode exception), we only need to save the callee-save fpu registers. */
tbz w4, #(THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_CALLING_SVC), 3f
2: /* Save the 64-bit caller-save fpu registers. */
ldr x6, [x2, #(THREAD_STACK_PARAMETERS_CALLER_SAVE_FPU_REGISTERS)]
stp q0, q1, [x6, #(THREAD_FPU64_CONTEXT_Q0_Q1)]
stp q2, q3, [x6, #(THREAD_FPU64_CONTEXT_Q2_Q3)]
stp q4, q5, [x6, #(THREAD_FPU64_CONTEXT_Q4_Q5)]
stp q6, q7, [x6, #(THREAD_FPU64_CONTEXT_Q6_Q7)]
stp q16, q17, [x6, #(THREAD_FPU64_CONTEXT_Q16_Q17)]
stp q18, q19, [x6, #(THREAD_FPU64_CONTEXT_Q18_Q19)]
stp q20, q21, [x6, #(THREAD_FPU64_CONTEXT_Q20_Q21)]
stp q22, q23, [x6, #(THREAD_FPU64_CONTEXT_Q22_Q23)]
stp q24, q25, [x6, #(THREAD_FPU64_CONTEXT_Q24_Q25)]
stp q26, q27, [x6, #(THREAD_FPU64_CONTEXT_Q26_Q27)]
stp q28, q29, [x6, #(THREAD_FPU64_CONTEXT_Q28_Q29)]
stp q30, q31, [x6, #(THREAD_FPU64_CONTEXT_Q30_Q31)]
3: /* Save the 64-bit callee-save fpu registers. */
stp q8, q9, [x3, #(THREAD_CONTEXT_FPU64_Q8_Q9)]
stp q10, q11, [x3, #(THREAD_CONTEXT_FPU64_Q10_Q11)]
stp q12, q13, [x3, #(THREAD_CONTEXT_FPU64_Q12_Q13)]
stp q14, q15, [x3, #(THREAD_CONTEXT_FPU64_Q14_Q15)]
b 7f
4: /* We have a 32-bit fpu. */
/* If we're in a usermode exception, we need to save the caller-save fpu registers. */
tbz w4, #(THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_IN_USERMODE_EXCEPTION_HANDLER), 5f
/* If we're in an SVC (and not a usermode exception), we only need to save the callee-save fpu registers. */
tbz w4, #(THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_CALLING_SVC), 6f
5: /* Save the 32-bit caller-save fpu registers. */
ldr x6, [x2, #(THREAD_STACK_PARAMETERS_CALLER_SAVE_FPU_REGISTERS)]
stp q0, q1, [x6, #(THREAD_FPU32_CONTEXT_Q0_Q1)]
stp q2, q3, [x6, #(THREAD_FPU32_CONTEXT_Q2_Q3)]
stp q8, q9, [x6, #(THREAD_FPU32_CONTEXT_Q8_Q9)]
stp q10, q11, [x6, #(THREAD_FPU32_CONTEXT_Q10_Q11)]
stp q12, q13, [x6, #(THREAD_FPU32_CONTEXT_Q12_Q13)]
stp q14, q15, [x6, #(THREAD_FPU32_CONTEXT_Q14_Q15)]
6: /* Save the 32-bit callee-save fpu registers. */
stp q4, q5, [x3, #(THREAD_CONTEXT_FPU32_Q4_Q5)]
stp q6, q7, [x3, #(THREAD_CONTEXT_FPU32_Q6_Q7)]
7: /* With the fpu state saved, disable the fpu. */
and x5, x5, #(~0x300000)
msr cpacr_el1, x5
8: /* We're done saving this thread's context. */
/* Check if the thread is terminated by checking the DPC flags for DpcFlag_Terminated. */
ldrb w4, [x2, #(THREAD_STACK_PARAMETERS_DPC_FLAGS)]
tbnz w4, #1, 9f
/* The thread isn't terminated, so we want to unlock it. */
/* Write atomically to the context's locked member. */
add x3, x3, #(THREAD_CONTEXT_LOCKED)
stlrb wzr, [x3]
9: /* The current thread's context has been entirely taken care of. */
/* Now we want to loop until we successfully switch the thread context. */
/* Start by saving all the values we care about in callee-save registers. */
mov x19, x0 /* this */
mov x20, x1 /* this->state */
mov x21, x7 /* highest priority thread */
/* Set our stack to the idle thread stack. */
ldr x3, [x20, #(KSCHEDULER_IDLE_THREAD_STACK)]
mov sp, x3
b 11f
10: /* We failed to successfully do the context switch, and need to retry. */
/* Clear the exclusive monitor. */
clrex
/* Clear the need's scheduling bool (and dmb ish after, as it's an atomic). */
strb wzr, [x20]
dmb ish
/* Refresh the highest priority thread. */
ldr x21, [x20, #(KSCHEDULER_HIGHEST_PRIORITY_THREAD)]
11: /* We're starting to try to do the context switch. */
/* Check if the highest priority thread if null. */
/* If it is, we want to branch to a special idle thread loop. */
cbz x21, 16f
/* Get the highest priority thread's context, and save it. */
/* ams::kern::KThread::GetContextForSchedulerLoop() */
ldr x22, [x21, #(THREAD_KERNEL_STACK_TOP)]
sub x22, x22, #(THREAD_STACK_PARAMETERS_SIZE - THREAD_STACK_PARAMETERS_THREAD_CONTEXT)
/* Prepare to try to acquire the context lock. */
add x1, x22, #(THREAD_CONTEXT_LOCKED)
mov w2, #1
12: /* We want to try to lock the highest priority thread's context. */
/* Check if the lock is already held. */
ldaxrb w3, [x1]
cbnz w3, 13f
/* If it's not, try to take it. */
stxrb w3, w2, [x1]
cbnz w3, 12b
/* We hold the lock, so we can now switch the thread. */
b 14f
13: /* The highest priority thread's context is already locked. */
/* Check if we need scheduling. If we don't, we can retry directly. */
ldarb w3, [x20] // ldarb w3, [x20, #(KSCHEDULER_NEEDS_SCHEDULING)]
cbz w3, 12b
/* If we do, another core is interfering, and we must start from the top. */
b 10b
14: /* It's time to switch the thread. */
/* Switch to the highest priority thread. */
mov x0, x19
mov x1, x21
/* Call ams::kern::KScheduler::SwitchThread(ams::kern::KThread *) */
bl _ZN3ams4kern10KScheduler12SwitchThreadEPNS0_7KThreadE
/* Check if we need scheduling. If we do, then we can't complete the switch and should retry. */
ldarb w1, [x20] // ldarb w1, [x20, #(KSCHEDULER_NEEDS_SCHEDULING)]
cbnz w1, 15f
/* Restore the thread context. */
mov x0, x22
ldp x19, x20, [x0, #(THREAD_CONTEXT_X19_X20)]
ldp x21, x22, [x0, #(THREAD_CONTEXT_X21_X22)]
ldp x23, x24, [x0, #(THREAD_CONTEXT_X23_X24)]
ldp x25, x26, [x0, #(THREAD_CONTEXT_X25_X26)]
ldp x27, x28, [x0, #(THREAD_CONTEXT_X27_X28)]
ldp x29, x30, [x0, #(THREAD_CONTEXT_X29_X30)]
ldr x1, [x0, #(THREAD_CONTEXT_SP)]
mov sp, x1
/* Return. */
ret
15: /* Our switch failed. */
/* We should unlock the thread context, and then retry. */
add x1, x22, #(THREAD_CONTEXT_LOCKED)
stlrb wzr, [x1]
b 10b
16: /* The next thread is nullptr! */
/* Switch to nullptr. This will actually switch to the idle thread. */
mov x0, x19
mov x1, #0
/* Call ams::kern::KScheduler::SwitchThread(ams::kern::KThread *) */
bl _ZN3ams4kern10KScheduler12SwitchThreadEPNS0_7KThreadE
17: /* We've switched to the idle thread, so we want to process interrupt tasks until we schedule a non-idle thread. */
/* Check whether there are runnable interrupt tasks. */
ldrb w3, [x20, #(KSCHEDULER_INTERRUPT_TASK_RUNNABLE)]
cbnz w3, 18f
/* Check if we need scheduling. */
ldarb w3, [x20] // ldarb w3, [x20, #(KSCHEDULER_NEEDS_SCHEDULING)]
cbnz w3, 10b
/* Clear the previous thread. */
str xzr, [x20, #(KSCHEDULER_PREVIOUS_THREAD)]
/* Wait for an interrupt and check again. */
wfi
msr daifclr, #2
msr daifset, #2
b 17b
18: /* We have interrupt tasks to execute! */
/* Execute any pending interrupt tasks. */
ldr x0, [x20, #(KSCHEDULER_INTERRUPT_TASK_MANAGER)]
bl _ZN3ams4kern21KInterruptTaskManager7DoTasksEv
/* Clear the interrupt task thread as runnable. */
strb wzr, [x20, #(KSCHEDULER_INTERRUPT_TASK_RUNNABLE)]
/* Retry the scheduling loop. */
b 10b

92
Source/Atmosphere-MTC-Unlock/mesosphere/kernel/source/arch/arm64/kern_k_thread_context_asm.s
View File

@@ -1,92 +0,0 @@
/*
* Copyright (c) Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <mesosphere/kern_select_assembly_offsets.h>
#include <mesosphere/kern_select_assembly_macros.h>
/* ams::kern::arch::arm64::UserModeThreadStarter() */
.section .text._ZN3ams4kern4arch5arm6421UserModeThreadStarterEv, "ax", %progbits
.global _ZN3ams4kern4arch5arm6421UserModeThreadStarterEv
.type _ZN3ams4kern4arch5arm6421UserModeThreadStarterEv, %function
_ZN3ams4kern4arch5arm6421UserModeThreadStarterEv:
/* NOTE: Stack layout on entry looks like following: */
/* SP */
/* | */
/* v */
/* | KExceptionContext (size 0x120) | KThread::StackParameters (size 0x30) | */
/* Clear the disable count for this thread's stack parameters. */
strh wzr, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_DISABLE_COUNT)]
/* Call ams::kern::arch::arm64::OnThreadStart() */
bl _ZN3ams4kern4arch5arm6413OnThreadStartEv
/* Restore thread state from the KExceptionContext on stack */
ldp x30, x19, [sp, #(EXCEPTION_CONTEXT_X30_SP)] /* x30 = lr, x19 = sp */
ldp x20, x21, [sp, #(EXCEPTION_CONTEXT_PC_PSR)] /* x20 = pc, x21 = psr */
ldr x22, [sp, #(EXCEPTION_CONTEXT_TPIDR)] /* x22 = tpidr */
msr sp_el0, x19
msr elr_el1, x20
msr spsr_el1, x21
msr tpidr_el0, x22
ldp x0, x1, [sp, #(EXCEPTION_CONTEXT_X0_X1)]
ldp x2, x3, [sp, #(EXCEPTION_CONTEXT_X2_X3)]
ldp x4, x5, [sp, #(EXCEPTION_CONTEXT_X4_X5)]
ldp x6, x7, [sp, #(EXCEPTION_CONTEXT_X6_X7)]
ldp x8, x9, [sp, #(EXCEPTION_CONTEXT_X8_X9)]
ldp x10, x11, [sp, #(EXCEPTION_CONTEXT_X10_X11)]
ldp x12, x13, [sp, #(EXCEPTION_CONTEXT_X12_X13)]
ldp x14, x15, [sp, #(EXCEPTION_CONTEXT_X14_X15)]
ldp x16, x17, [sp, #(EXCEPTION_CONTEXT_X16_X17)]
ldp x18, x19, [sp, #(EXCEPTION_CONTEXT_X18_X19)]
ldp x20, x21, [sp, #(EXCEPTION_CONTEXT_X20_X21)]
ldp x22, x23, [sp, #(EXCEPTION_CONTEXT_X22_X23)]
ldp x24, x25, [sp, #(EXCEPTION_CONTEXT_X24_X25)]
ldp x26, x27, [sp, #(EXCEPTION_CONTEXT_X26_X27)]
ldp x28, x29, [sp, #(EXCEPTION_CONTEXT_X28_X29)]
/* Increment stack pointer above the KExceptionContext */
add sp, sp, #(EXCEPTION_CONTEXT_SIZE)
/* Return to EL0 */
ERET_WITH_SPECULATION_BARRIER
/* ams::kern::arch::arm64::SupervisorModeThreadStarter() */
.section .text._ZN3ams4kern4arch5arm6427SupervisorModeThreadStarterEv, "ax", %progbits
.global _ZN3ams4kern4arch5arm6427SupervisorModeThreadStarterEv
.type _ZN3ams4kern4arch5arm6427SupervisorModeThreadStarterEv, %function
_ZN3ams4kern4arch5arm6427SupervisorModeThreadStarterEv:
/* NOTE: Stack layout on entry looks like following: */
/* SP */
/* | */
/* v */
/* | u64 argument | u64 entrypoint | KThread::StackParameters (size 0x30) | */
/* Clear the link register. */
mov x30, #0
/* Load the argument and entrypoint. */
ldp x0, x1, [sp], #0x10
/* Clear the disable count for this thread's stack parameters. */
strh wzr, [sp, #(THREAD_STACK_PARAMETERS_DISABLE_COUNT)]
/* Mask I bit in DAIF */
msr daifclr, #2
/* Invoke the function (by calling ams::kern::arch::arm64::InvokeSupervisorModeThread(argument, entrypoint)). */
b _ZN3ams4kern4arch5arm6426InvokeSupervisorModeThreadEmm

27
Source/Atmosphere-MTC-Unlock/mesosphere/kernel/source/arch/arm64/svc/kern_svc_tables_asm.s
View File

@@ -1,27 +0,0 @@
/*
* Copyright (c) Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
/* ams::kern::svc::PatchSvcTableEntry(void (* const*)(), unsigned int, void (*)()) */
.section .text._ZN3ams4kern3svc18PatchSvcTableEntryEPKPFvvEjS3_, "ax", %progbits
.global _ZN3ams4kern3svc18PatchSvcTableEntryEPKPFvvEjS3_
.type _ZN3ams4kern3svc18PatchSvcTableEntryEPKPFvvEjS3_, %function
_ZN3ams4kern3svc18PatchSvcTableEntryEPKPFvvEjS3_:
/* This function violates const correctness by design, to patch the svc tables. */
/* The svc tables live in .rodata (.rel.ro), but must be patched by initial constructors */
/* to support firmware-specific table entries. */
mov w1, w1
str x2, [x0, x1, lsl #3]
ret

118
Source/Atmosphere-MTC-Unlock/mesosphere/kernel/source/kern_kernel_instantiations.cpp
View File

@@ -1,118 +0,0 @@
/*
* Copyright (c) Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <mesosphere.hpp>
namespace ams::kern {
/* Declare kernel data members in kernel TU. */
constinit Kernel::State Kernel::s_state = Kernel::State::Invalid;
constinit KResourceLimit Kernel::s_system_resource_limit{util::ConstantInitialize};
KMemoryManager Kernel::s_memory_manager;
constinit KSupervisorPageTable Kernel::s_supervisor_page_table;
constinit KUnsafeMemory Kernel::s_unsafe_memory;
constinit KWorkerTaskManager Kernel::s_worker_task_managers[KWorkerTaskManager::WorkerType_Count];
constinit KInterruptManager Kernel::s_interrupt_manager;
constinit KScheduler Kernel::s_schedulers[cpu::NumCores];
constinit KInterruptTaskManager Kernel::s_interrupt_task_managers[cpu::NumCores];
constinit KHardwareTimer Kernel::s_hardware_timers[cpu::NumCores];
constinit KPageTableSlabHeap Kernel::s_page_table_heap;
constinit KMemoryBlockSlabHeap Kernel::s_app_memory_block_heap;
constinit KMemoryBlockSlabHeap Kernel::s_sys_memory_block_heap;
constinit KBlockInfoSlabHeap Kernel::s_block_info_heap;
constinit KPageTableManager Kernel::s_app_page_table_manager{util::ConstantInitialize};
constinit KPageTableManager Kernel::s_sys_page_table_manager{util::ConstantInitialize};
constinit KMemoryBlockSlabManager Kernel::s_app_memory_block_manager;
constinit KMemoryBlockSlabManager Kernel::s_sys_memory_block_manager;
constinit KBlockInfoManager Kernel::s_app_block_info_manager;
constinit KBlockInfoManager Kernel::s_sys_block_info_manager;
constinit KSystemResource Kernel::s_app_system_resource{util::ConstantInitialize};
constinit KSystemResource Kernel::s_sys_system_resource{util::ConstantInitialize};
namespace {
template<size_t N> requires (N > 0)
union KThreadArray {
struct RecursiveHolder {
KThread m_thread;
KThreadArray<N - 1> m_next;
consteval RecursiveHolder() : m_thread{util::ConstantInitialize}, m_next() { /* ... */ }
} m_holder;
KThread m_arr[N];
consteval KThreadArray() : m_holder() { /* ... */ }
};
template<>
union KThreadArray<1>{
struct RecursiveHolder {
KThread m_thread;
consteval RecursiveHolder() : m_thread{util::ConstantInitialize} { /* ... */ }
} m_holder;
KThread m_arr[1];
consteval KThreadArray() : m_holder() { /* ... */ }
};
template<size_t Ix>
consteval bool IsKThreadArrayValid(const KThreadArray<Ix> &v, const KThread *thread) {
if (std::addressof(v.m_holder.m_thread) != thread) {
return false;
}
if constexpr (Ix == 1) {
return true;
} else {
return IsKThreadArrayValid(v.m_holder.m_next, thread + 1);
}
}
template<size_t N>
consteval bool IsKThreadArrayValid() {
const KThreadArray<N> v{};
if (!IsKThreadArrayValid(v, v.m_arr)) {
return false;
}
if constexpr (N == 1) {
return true;
} else {
return IsKThreadArrayValid<N - 1>();
}
}
static_assert(IsKThreadArrayValid<cpu::NumCores>());
constinit KThreadArray<cpu::NumCores> g_main_threads;
constinit KThreadArray<cpu::NumCores> g_idle_threads;
static_assert(sizeof(g_main_threads) == cpu::NumCores * sizeof(KThread));
static_assert(sizeof(g_main_threads.m_holder) == sizeof(g_main_threads.m_arr));
}
KThread &Kernel::GetMainThread(s32 core_id) { return g_main_threads.m_arr[core_id]; }
KThread &Kernel::GetIdleThread(s32 core_id) { return g_idle_threads.m_arr[core_id]; }
__attribute__((constructor)) void ConfigureKTargetSystem() {
KSystemControl::ConfigureKTargetSystem();
}
}

31
Source/Atmosphere-MTC-Unlock/mesosphere/kernel/source/libc/arch/arm64/asmdefs.h
View File

@@ -1,31 +0,0 @@
/*
* Macros for asm code.
*
* Copyright (c) 2019, Arm Limited.
* SPDX-License-Identifier: MIT
*/
#ifndef _ASMDEFS_H
#define _ASMDEFS_H
#define ENTRY_ALIGN(name, alignment) \
.global name; \
.type name,%function; \
.align alignment; \
name: \
.cfi_startproc;
#define ENTRY(name) ENTRY_ALIGN(name, 6)
#define ENTRY_ALIAS(name) \
.global name; \
.type name,%function; \
name:
#define END(name) \
.cfi_endproc; \
.size name, .-name;
#define L(l) .L ## l
#endif

133
Source/Atmosphere-MTC-Unlock/mesosphere/kernel/source/libc/arch/arm64/memcmp.arch.arm64.s
View File

@@ -1,133 +0,0 @@
/* memcmp - compare memory
*
* Copyright (c) 2013, Arm Limited.
* SPDX-License-Identifier: MIT
*/
/* Assumptions:
*
* ARMv8-a, AArch64, unaligned accesses.
*/
#include "asmdefs.h"
/* Parameters and result. */
#define src1 x0
#define src2 x1
#define limit x2
#define result w0
/* Internal variables. */
#define data1 x3
#define data1w w3
#define data1h x4
#define data2 x5
#define data2w w5
#define data2h x6
#define tmp1 x7
#define tmp2 x8
ENTRY (memcmp)
subs limit, limit, 8
b.lo L(less8)
ldr data1, [src1], 8
ldr data2, [src2], 8
cmp data1, data2
b.ne L(return)
subs limit, limit, 8
b.gt L(more16)
ldr data1, [src1, limit]
ldr data2, [src2, limit]
b L(return)
L(more16):
ldr data1, [src1], 8
ldr data2, [src2], 8
cmp data1, data2
bne L(return)
/* Jump directly to comparing the last 16 bytes for 32 byte (or less)
strings. */
subs limit, limit, 16
b.ls L(last_bytes)
/* We overlap loads between 0-32 bytes at either side of SRC1 when we
try to align, so limit it only to strings larger than 128 bytes. */
cmp limit, 96
b.ls L(loop16)
/* Align src1 and adjust src2 with bytes not yet done. */
and tmp1, src1, 15
add limit, limit, tmp1
sub src1, src1, tmp1
sub src2, src2, tmp1
/* Loop performing 16 bytes per iteration using aligned src1.
Limit is pre-decremented by 16 and must be larger than zero.
Exit if <= 16 bytes left to do or if the data is not equal. */
.p2align 4
L(loop16):
ldp data1, data1h, [src1], 16
ldp data2, data2h, [src2], 16
subs limit, limit, 16
ccmp data1, data2, 0, hi
ccmp data1h, data2h, 0, eq
b.eq L(loop16)
cmp data1, data2
bne L(return)
mov data1, data1h
mov data2, data2h
cmp data1, data2
bne L(return)
/* Compare last 1-16 bytes using unaligned access. */
L(last_bytes):
add src1, src1, limit
add src2, src2, limit
ldp data1, data1h, [src1]
ldp data2, data2h, [src2]
cmp data1, data2
bne L(return)
mov data1, data1h
mov data2, data2h
cmp data1, data2
/* Compare data bytes and set return value to 0, -1 or 1. */
L(return):
#ifndef __AARCH64EB__
rev data1, data1
rev data2, data2
#endif
cmp data1, data2
L(ret_eq):
cset result, ne
cneg result, result, lo
ret
.p2align 4
/* Compare up to 8 bytes. Limit is [-8..-1]. */
L(less8):
adds limit, limit, 4
b.lo L(less4)
ldr data1w, [src1], 4
ldr data2w, [src2], 4
cmp data1w, data2w
b.ne L(return)
sub limit, limit, 4
L(less4):
adds limit, limit, 4
beq L(ret_eq)
L(byte_loop):
ldrb data1w, [src1], 1
ldrb data2w, [src2], 1
subs limit, limit, 1
ccmp data1w, data2w, 0, ne /* NZCV = 0b0000. */
b.eq L(byte_loop)
sub result, data1w, data2w
ret
END (memcmp)

239
Source/Atmosphere-MTC-Unlock/mesosphere/kernel/source/libc/arch/arm64/memcpy.arch.arm64.s
View File

@@ -1,239 +0,0 @@
/*
* memcpy - copy memory area
*
* Copyright (c) 2012-2020, Arm Limited.
* SPDX-License-Identifier: MIT
*/
/* Assumptions:
*
* ARMv8-a, AArch64, unaligned accesses.
*
*/
#include "asmdefs.h"
#define dstin x0
#define src x1
#define count x2
#define dst x3
#define srcend x4
#define dstend x5
#define A_l x6
#define A_lw w6
#define A_h x7
#define B_l x8
#define B_lw w8
#define B_h x9
#define C_l x10
#define C_lw w10
#define C_h x11
#define D_l x12
#define D_h x13
#define E_l x14
#define E_h x15
#define F_l x16
#define F_h x17
#define G_l count
#define G_h dst
#define H_l src
#define H_h srcend
#define tmp1 x14
/* This implementation handles overlaps and supports both memcpy and memmove
from a single entry point. It uses unaligned accesses and branchless
sequences to keep the code small, simple and improve performance.
Copies are split into 3 main cases: small copies of up to 32 bytes, medium
copies of up to 128 bytes, and large copies. The overhead of the overlap
check is negligible since it is only required for large copies.
Large copies use a software pipelined loop processing 64 bytes per iteration.
The destination pointer is 16-byte aligned to minimize unaligned accesses.
The loop tail is handled by always copying 64 bytes from the end.
*/
ENTRY (memcpy)
ENTRY_ALIAS (memmove)
add srcend, src, count
add dstend, dstin, count
cmp count, 128
b.hi L(copy_long)
cmp count, 32
b.hi L(copy32_128)
/* Small copies: 0..32 bytes. */
cmp count, 16
b.lo L(copy16)
ldp A_l, A_h, [src]
ldp D_l, D_h, [srcend, -16]
stp A_l, A_h, [dstin]
stp D_l, D_h, [dstend, -16]
ret
/* Copy 8-15 bytes. */
L(copy16):
tbz count, 3, L(copy8)
ldr A_l, [src]
ldr A_h, [srcend, -8]
str A_l, [dstin]
str A_h, [dstend, -8]
ret
.p2align 3
/* Copy 4-7 bytes. */
L(copy8):
tbz count, 2, L(copy4)
ldr A_lw, [src]
ldr B_lw, [srcend, -4]
str A_lw, [dstin]
str B_lw, [dstend, -4]
ret
/* Copy 0..3 bytes using a branchless sequence. */
L(copy4):
cbz count, L(copy0)
lsr tmp1, count, 1
ldrb A_lw, [src]
ldrb C_lw, [srcend, -1]
ldrb B_lw, [src, tmp1]
strb A_lw, [dstin]
strb B_lw, [dstin, tmp1]
strb C_lw, [dstend, -1]
L(copy0):
ret
.p2align 4
/* Medium copies: 33..128 bytes. */
L(copy32_128):
ldp A_l, A_h, [src]
ldp B_l, B_h, [src, 16]
ldp C_l, C_h, [srcend, -32]
ldp D_l, D_h, [srcend, -16]
cmp count, 64
b.hi L(copy128)
stp A_l, A_h, [dstin]
stp B_l, B_h, [dstin, 16]
stp C_l, C_h, [dstend, -32]
stp D_l, D_h, [dstend, -16]
ret
.p2align 4
/* Copy 65..128 bytes. */
L(copy128):
ldp E_l, E_h, [src, 32]
ldp F_l, F_h, [src, 48]
cmp count, 96
b.ls L(copy96)
ldp G_l, G_h, [srcend, -64]
ldp H_l, H_h, [srcend, -48]
stp G_l, G_h, [dstend, -64]
stp H_l, H_h, [dstend, -48]
L(copy96):
stp A_l, A_h, [dstin]
stp B_l, B_h, [dstin, 16]
stp E_l, E_h, [dstin, 32]
stp F_l, F_h, [dstin, 48]
stp C_l, C_h, [dstend, -32]
stp D_l, D_h, [dstend, -16]
ret
.p2align 4
/* Copy more than 128 bytes. */
L(copy_long):
/* Use backwards copy if there is an overlap. */
sub tmp1, dstin, src
cbz tmp1, L(copy0)
cmp tmp1, count
b.lo L(copy_long_backwards)
/* Copy 16 bytes and then align dst to 16-byte alignment. */
ldp D_l, D_h, [src]
and tmp1, dstin, 15
bic dst, dstin, 15
sub src, src, tmp1
add count, count, tmp1 /* Count is now 16 too large. */
ldp A_l, A_h, [src, 16]
stp D_l, D_h, [dstin]
ldp B_l, B_h, [src, 32]
ldp C_l, C_h, [src, 48]
ldp D_l, D_h, [src, 64]!
subs count, count, 128 + 16 /* Test and readjust count. */
b.ls L(copy64_from_end)
L(loop64):
stp A_l, A_h, [dst, 16]
ldp A_l, A_h, [src, 16]
stp B_l, B_h, [dst, 32]
ldp B_l, B_h, [src, 32]
stp C_l, C_h, [dst, 48]
ldp C_l, C_h, [src, 48]
stp D_l, D_h, [dst, 64]!
ldp D_l, D_h, [src, 64]!
subs count, count, 64
b.hi L(loop64)
/* Write the last iteration and copy 64 bytes from the end. */
L(copy64_from_end):
ldp E_l, E_h, [srcend, -64]
stp A_l, A_h, [dst, 16]
ldp A_l, A_h, [srcend, -48]
stp B_l, B_h, [dst, 32]
ldp B_l, B_h, [srcend, -32]
stp C_l, C_h, [dst, 48]
ldp C_l, C_h, [srcend, -16]
stp D_l, D_h, [dst, 64]
stp E_l, E_h, [dstend, -64]
stp A_l, A_h, [dstend, -48]
stp B_l, B_h, [dstend, -32]
stp C_l, C_h, [dstend, -16]
ret
.p2align 4
/* Large backwards copy for overlapping copies.
Copy 16 bytes and then align dst to 16-byte alignment. */
L(copy_long_backwards):
ldp D_l, D_h, [srcend, -16]
and tmp1, dstend, 15
sub srcend, srcend, tmp1
sub count, count, tmp1
ldp A_l, A_h, [srcend, -16]
stp D_l, D_h, [dstend, -16]
ldp B_l, B_h, [srcend, -32]
ldp C_l, C_h, [srcend, -48]
ldp D_l, D_h, [srcend, -64]!
sub dstend, dstend, tmp1
subs count, count, 128
b.ls L(copy64_from_start)
L(loop64_backwards):
stp A_l, A_h, [dstend, -16]
ldp A_l, A_h, [srcend, -16]
stp B_l, B_h, [dstend, -32]
ldp B_l, B_h, [srcend, -32]
stp C_l, C_h, [dstend, -48]
ldp C_l, C_h, [srcend, -48]
stp D_l, D_h, [dstend, -64]!
ldp D_l, D_h, [srcend, -64]!
subs count, count, 64
b.hi L(loop64_backwards)
/* Write the last iteration and copy 64 bytes from the start. */
L(copy64_from_start):
ldp G_l, G_h, [src, 48]
stp A_l, A_h, [dstend, -16]
ldp A_l, A_h, [src, 32]
stp B_l, B_h, [dstend, -32]
ldp B_l, B_h, [src, 16]
stp C_l, C_h, [dstend, -48]
ldp C_l, C_h, [src]
stp D_l, D_h, [dstend, -64]
stp G_l, G_h, [dstin, 48]
stp A_l, A_h, [dstin, 32]
stp B_l, B_h, [dstin, 16]
stp C_l, C_h, [dstin]
ret
END (memcpy)

172
Source/Atmosphere-MTC-Unlock/mesosphere/kernel/source/libc/arch/arm64/memset.arch.arm64.s
View File

@@ -1,172 +0,0 @@
/*
* memset - fill memory with a constant byte
*
* Copyright (c) 2012-2020, Arm Limited.
* SPDX-License-Identifier: MIT
*/
/* Assumptions:
*
* ARMv8-a, AArch64, Advanced SIMD, unaligned accesses.
*
*/
#include "asmdefs.h"
#define DC_ZVA_THRESHOLD 512
#define dstin x0
#define val x1
#define valw w1
#define count x2
#define dst x3
#define dstend x4
#define zva_val x5
ENTRY (memset)
bfi valw, valw, 8, 8
bfi valw, valw, 16, 16
bfi val, val, 32, 32
add dstend, dstin, count
cmp count, 96
b.hi L(set_long)
cmp count, 16
b.hs L(set_medium)
/* Set 0..15 bytes. */
tbz count, 3, 1f
str val, [dstin]
str val, [dstend, -8]
ret
1: tbz count, 2, 2f
str valw, [dstin]
str valw, [dstend, -4]
ret
2: cbz count, 3f
strb valw, [dstin]
tbz count, 1, 3f
strh valw, [dstend, -2]
3: ret
/* Set 16..96 bytes. */
.p2align 4
L(set_medium):
stp val, val, [dstin]
tbnz count, 6, L(set96)
stp val, val, [dstend, -16]
tbz count, 5, 1f
stp val, val, [dstin, 16]
stp val, val, [dstend, -32]
1: ret
.p2align 4
/* Set 64..96 bytes. Write 64 bytes from the start and
32 bytes from the end. */
L(set96):
stp val, val, [dstin, 16]
stp val, val, [dstin, 32]
stp val, val, [dstin, 48]
stp val, val, [dstend, -32]
stp val, val, [dstend, -16]
ret
.p2align 4
L(set_long):
stp val, val, [dstin]
#if DC_ZVA_THRESHOLD
cmp count, DC_ZVA_THRESHOLD
ccmp val, 0, 0, cs
bic dst, dstin, 15
b.eq L(zva_64)
#else
bic dst, dstin, 15
#endif
/* Small-size or non-zero memset does not use DC ZVA. */
sub count, dstend, dst
/*
* Adjust count and bias for loop. By substracting extra 1 from count,
* it is easy to use tbz instruction to check whether loop tailing
* count is less than 33 bytes, so as to bypass 2 unneccesary stps.
*/
sub count, count, 64+16+1
#if DC_ZVA_THRESHOLD
/* Align loop on 16-byte boundary, this might be friendly to i-cache. */
nop
#endif
1: stp val, val, [dst, 16]
stp val, val, [dst, 32]
stp val, val, [dst, 48]
stp val, val, [dst, 64]!
subs count, count, 64
b.hs 1b
tbz count, 5, 1f /* Remaining count is less than 33 bytes? */
stp val, val, [dst, 16]
stp val, val, [dst, 32]
1: stp val, val, [dstend, -32]
stp val, val, [dstend, -16]
ret
#if DC_ZVA_THRESHOLD
.p2align 4
L(zva_64):
stp val, val, [dst, 16]
stp val, val, [dst, 32]
stp val, val, [dst, 48]
bic dst, dst, 63
/*
* Previous memory writes might cross cache line boundary, and cause
* cache line partially dirty. Zeroing this kind of cache line using
* DC ZVA will incur extra cost, for it requires loading untouched
* part of the line from memory before zeoring.
*
* So, write the first 64 byte aligned block using stp to force
* fully dirty cache line.
*/
stp val, val, [dst, 64]
stp val, val, [dst, 80]
stp val, val, [dst, 96]
stp val, val, [dst, 112]
sub count, dstend, dst
/*
* Adjust count and bias for loop. By substracting extra 1 from count,
* it is easy to use tbz instruction to check whether loop tailing
* count is less than 33 bytes, so as to bypass 2 unneccesary stps.
*/
sub count, count, 128+64+64+1
add dst, dst, 128
nop
/* DC ZVA sets 64 bytes each time. */
1: dc zva, dst
add dst, dst, 64
subs count, count, 64
b.hs 1b
/*
* Write the last 64 byte aligned block using stp to force fully
* dirty cache line.
*/
stp val, val, [dst, 0]
stp val, val, [dst, 16]
stp val, val, [dst, 32]
stp val, val, [dst, 48]
tbz count, 5, 1f /* Remaining count is less than 33 bytes? */
stp val, val, [dst, 64]
stp val, val, [dst, 80]
1: stp val, val, [dstend, -32]
stp val, val, [dstend, -16]
ret
#endif
END (memset)

32
Source/Atmosphere-MTC-Unlock/mesosphere/kernel/source/libc/kern_env.cpp
View File

@@ -1,32 +0,0 @@
/*
* Copyright (c) Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <mesosphere.hpp>
void operator delete (void *deleted) throw() {
MESOSPHERE_PANIC("operator delete(void *) was called: %p", deleted);
}
void operator delete (void *deleted, size_t size) throw() {
MESOSPHERE_PANIC("operator delete(void *, size_t) was called: %p %zu", deleted, size);
}
void operator delete (void *deleted, size_t size, std::align_val_t align) throw() {
MESOSPHERE_PANIC("operator delete(void *, size_t, std::align_val_t) was called: %p %zu, %zu", deleted, size, static_cast<size_t>(align));
}
extern "C" void abort() {
MESOSPHERE_PANIC("abort() was called");
}

24
Source/Atmosphere-MTC-Unlock/mesosphere/kernel/source/libc/kern_libc_config.arch.arm64.h
View File

@@ -1,24 +0,0 @@
/*
* Copyright (c) Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
/* Definitions for libc genericity. */
#define MESOSPHERE_LIBC_MEMCPY_GENERIC 0
#define MESOSPHERE_LIBC_MEMCMP_GENERIC 0
#define MESOSPHERE_LIBC_MEMMOVE_GENERIC 0
#define MESOSPHERE_LIBC_MEMSET_GENERIC 0
#define MESOSPHERE_LIBC_STRNCPY_GENERIC 1
#define MESOSPHERE_LIBC_STRNCMP_GENERIC 1

26
Source/Atmosphere-MTC-Unlock/mesosphere/kernel/source/libc/kern_libc_config.h
View File

@@ -1,26 +0,0 @@
/*
* Copyright (c) Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#if defined(ATMOSPHERE_ARCH_ARM64)
#include "kern_libc_config.arch.arm64.h"
#else
#error "Unknown architecture for libc"
#endif

673
Source/Atmosphere-MTC-Unlock/mesosphere/kernel/source/libc/kern_libc_generic.c
View File

@@ -1,673 +0,0 @@
/*
* Copyright (c) Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <string.h>
#include <stddef.h>
#include <limits.h>
#include "kern_libc_config.h"
/* Note: copied from newlib */
#ifdef __cplusplus
extern "C" {
#endif
/*
FUNCTION
<<memmove>>---move possibly overlapping memory
INDEX
memmove
SYNOPSIS
#include <string.h>
void *memmove(void *<[dst]>, const void *<[src]>, size_t <[length]>);
DESCRIPTION
This function moves <[length]> characters from the block of
memory starting at <<*<[src]>>> to the memory starting at
<<*<[dst]>>>. <<memmove>> reproduces the characters correctly
at <<*<[dst]>>> even if the two areas overlap.
RETURNS
The function returns <[dst]> as passed.
PORTABILITY
<<memmove>> is ANSI C.
<<memmove>> requires no supporting OS subroutines.
QUICKREF
memmove ansi pure
*/
/* Nonzero if either X or Y is not aligned on a "long" boundary. */
#define UNALIGNED(X, Y) \
(((long)X & (sizeof (long) - 1)) | ((long)Y & (sizeof (long) - 1)))
/* How many bytes are copied each iteration of the 4X unrolled loop. */
#define BIGBLOCKSIZE (sizeof (long) << 2)
/* How many bytes are copied each iteration of the word copy loop. */
#define LITTLEBLOCKSIZE (sizeof (long))
/* Threshhold for punting to the byte copier. */
#undef TOO_SMALL
#define TOO_SMALL(LEN) ((LEN) < BIGBLOCKSIZE)
#if MESOSPHERE_LIBC_MEMMOVE_GENERIC
/*SUPPRESS 20*/
void *
//__inhibit_loop_to_libcall
__attribute__((weak))
memmove (void *dst_void,
const void *src_void,
size_t length)
{
#if defined(PREFER_SIZE_OVER_SPEED) || defined(__OPTIMIZE_SIZE__)
char *dst = dst_void;
const char *src = src_void;
if (src < dst && dst < src + length)
{
/* Have to copy backwards */
src += length;
dst += length;
while (length--)
{
*--dst = *--src;
}
}
else
{
while (length--)
{
*dst++ = *src++;
}
}
return dst_void;
#else
char *dst = dst_void;
const char *src = src_void;
long *aligned_dst;
const long *aligned_src;
if (src < dst && dst < src + length)
{
/* Destructive overlap...have to copy backwards */
src += length;
dst += length;
while (length--)
{
*--dst = *--src;
}
}
else
{
/* Use optimizing algorithm for a non-destructive copy to closely
match memcpy. If the size is small or either SRC or DST is unaligned,
then punt into the byte copy loop. This should be rare. */
if (!TOO_SMALL(length) && !UNALIGNED (src, dst))
{
aligned_dst = (long*)dst;
aligned_src = (long*)src;
/* Copy 4X long words at a time if possible. */
while (length >= BIGBLOCKSIZE)
{
*aligned_dst++ = *aligned_src++;
*aligned_dst++ = *aligned_src++;
*aligned_dst++ = *aligned_src++;
*aligned_dst++ = *aligned_src++;
length -= BIGBLOCKSIZE;
}
/* Copy one long word at a time if possible. */
while (length >= LITTLEBLOCKSIZE)
{
*aligned_dst++ = *aligned_src++;
length -= LITTLEBLOCKSIZE;
}
/* Pick up any residual with a byte copier. */
dst = (char*)aligned_dst;
src = (char*)aligned_src;
}
while (length--)
{
*dst++ = *src++;
}
}
return dst_void;
#endif /* not PREFER_SIZE_OVER_SPEED */
}
#endif /* MESOSPHERE_LIBC_MEMMOVE_GENERIC */
/*
FUNCTION
<<memcpy>>---copy memory regions
SYNOPSIS
#include <string.h>
void* memcpy(void *restrict <[out]>, const void *restrict <[in]>,
size_t <[n]>);
DESCRIPTION
This function copies <[n]> bytes from the memory region
pointed to by <[in]> to the memory region pointed to by
<[out]>.
If the regions overlap, the behavior is undefined.
RETURNS
<<memcpy>> returns a pointer to the first byte of the <[out]>
region.
PORTABILITY
<<memcpy>> is ANSI C.
<<memcpy>> requires no supporting OS subroutines.
QUICKREF
memcpy ansi pure
*/
#if MESOSPHERE_LIBC_MEMCPY_GENERIC
void *
__attribute__((weak))
memcpy (void * dst0,
const void * __restrict src0,
size_t len0)
{
#if defined(PREFER_SIZE_OVER_SPEED) || defined(__OPTIMIZE_SIZE__)
char *dst = (char *) dst0;
char *src = (char *) src0;
void *save = dst0;
while (len0--)
{
*dst++ = *src++;
}
return save;
#else
char *dst = dst0;
const char *src = src0;
long *aligned_dst;
const long *aligned_src;
/* If the size is small, or either SRC or DST is unaligned,
then punt into the byte copy loop. This should be rare. */
if (!TOO_SMALL(len0) && !UNALIGNED (src, dst))
{
aligned_dst = (long*)dst;
aligned_src = (long*)src;
/* Copy 4X long words at a time if possible. */
while (len0 >= BIGBLOCKSIZE)
{
*aligned_dst++ = *aligned_src++;
*aligned_dst++ = *aligned_src++;
*aligned_dst++ = *aligned_src++;
*aligned_dst++ = *aligned_src++;
len0 -= BIGBLOCKSIZE;
}
/* Copy one long word at a time if possible. */
while (len0 >= LITTLEBLOCKSIZE)
{
*aligned_dst++ = *aligned_src++;
len0 -= LITTLEBLOCKSIZE;
}
/* Pick up any residual with a byte copier. */
dst = (char*)aligned_dst;
src = (char*)aligned_src;
}
while (len0--)
*dst++ = *src++;
return dst0;
#endif /* not PREFER_SIZE_OVER_SPEED */
}
#endif /* MESOSPHERE_LIBC_MEMCPY_GENERIC */
/*
FUNCTION
<<memset>>---set an area of memory
INDEX
memset
SYNOPSIS
#include <string.h>
void *memset(void *<[dst]>, int <[c]>, size_t <[length]>);
DESCRIPTION
This function converts the argument <[c]> into an unsigned
char and fills the first <[length]> characters of the array
pointed to by <[dst]> to the value.
RETURNS
<<memset>> returns the value of <[dst]>.
PORTABILITY
<<memset>> is ANSI C.
<<memset>> requires no supporting OS subroutines.
QUICKREF
memset ansi pure
*/
#include <string.h>
#undef LBLOCKSIZE
#undef UNALIGNED
#undef TOO_SMALL
#define LBLOCKSIZE (sizeof(long))
#define UNALIGNED(X) ((long)X & (LBLOCKSIZE - 1))
#define TOO_SMALL(LEN) ((LEN) < LBLOCKSIZE)
#if MESOSPHERE_LIBC_MEMSET_GENERIC
void *
__attribute__((weak))
memset (void *m,
int c,
size_t n)
{
char *s = (char *) m;
#if !defined(PREFER_SIZE_OVER_SPEED) && !defined(__OPTIMIZE_SIZE__)
unsigned int i;
unsigned long buffer;
unsigned long *aligned_addr;
unsigned int d = c & 0xff; /* To avoid sign extension, copy C to an
unsigned variable. */
while (UNALIGNED (s))
{
if (n--)
*s++ = (char) c;
else
return m;
}
if (!TOO_SMALL (n))
{
/* If we get this far, we know that n is large and s is word-aligned. */
aligned_addr = (unsigned long *) s;
/* Store D into each char sized location in BUFFER so that
we can set large blocks quickly. */
buffer = (d << 8) | d;
buffer |= (buffer << 16);
for (i = 32; i < LBLOCKSIZE * 8; i <<= 1)
buffer = (buffer << i) | buffer;
/* Unroll the loop. */
while (n >= LBLOCKSIZE*4)
{
*aligned_addr++ = buffer;
*aligned_addr++ = buffer;
*aligned_addr++ = buffer;
*aligned_addr++ = buffer;
n -= 4*LBLOCKSIZE;
}
while (n >= LBLOCKSIZE)
{
*aligned_addr++ = buffer;
n -= LBLOCKSIZE;
}
/* Pick up the remainder with a bytewise loop. */
s = (char*)aligned_addr;
}
#endif /* not PREFER_SIZE_OVER_SPEED */
while (n--)
*s++ = (char) c;
return m;
}
#endif /* MESOSPHERE_LIBC_MEMSET_GENERIC */
/*
FUNCTION
<<memcmp>>---compare two memory areas
INDEX
memcmp
SYNOPSIS
#include <string.h>
int memcmp(const void *<[s1]>, const void *<[s2]>, size_t <[n]>);
DESCRIPTION
This function compares not more than <[n]> characters of the
object pointed to by <[s1]> with the object pointed to by <[s2]>.
RETURNS
The function returns an integer greater than, equal to or
less than zero according to whether the object pointed to by
<[s1]> is greater than, equal to or less than the object
pointed to by <[s2]>.
PORTABILITY
<<memcmp>> is ANSI C.
<<memcmp>> requires no supporting OS subroutines.
QUICKREF
memcmp ansi pure
*/
#undef LBLOCKSIZE
#undef UNALIGNED
#undef TOO_SMALL
/* Nonzero if either X or Y is not aligned on a "long" boundary. */
#define UNALIGNED(X, Y) \
(((long)X & (sizeof (long) - 1)) | ((long)Y & (sizeof (long) - 1)))
/* How many bytes are copied each iteration of the word copy loop. */
#define LBLOCKSIZE (sizeof (long))
/* Threshhold for punting to the byte copier. */
#define TOO_SMALL(LEN) ((LEN) < LBLOCKSIZE)
#if MESOSPHERE_LIBC_MEMCMP_GENERIC
int
__attribute__((weak))
memcmp (const void *m1,
const void *m2,
size_t n)
{
#if defined(PREFER_SIZE_OVER_SPEED) || defined(__OPTIMIZE_SIZE__)
unsigned char *s1 = (unsigned char *) m1;
unsigned char *s2 = (unsigned char *) m2;
while (n--)
{
if (*s1 != *s2)
{
return *s1 - *s2;
}
s1++;
s2++;
}
return 0;
#else
unsigned char *s1 = (unsigned char *) m1;
unsigned char *s2 = (unsigned char *) m2;
unsigned long *a1;
unsigned long *a2;
/* If the size is too small, or either pointer is unaligned,
then we punt to the byte compare loop. Hopefully this will
not turn up in inner loops. */
if (!TOO_SMALL(n) && !UNALIGNED(s1,s2))
{
/* Otherwise, load and compare the blocks of memory one
word at a time. */
a1 = (unsigned long*) s1;
a2 = (unsigned long*) s2;
while (n >= LBLOCKSIZE)
{
if (*a1 != *a2)
break;
a1++;
a2++;
n -= LBLOCKSIZE;
}
/* check m mod LBLOCKSIZE remaining characters */
s1 = (unsigned char*)a1;
s2 = (unsigned char*)a2;
}
while (n--)
{
if (*s1 != *s2)
return *s1 - *s2;
s1++;
s2++;
}
return 0;
#endif /* not PREFER_SIZE_OVER_SPEED */
}
#endif /* MESOSPHERE_LIBC_MEMCMP_GENERIC */
/*
FUNCTION
<<strncpy>>---counted copy string
INDEX
strncpy
SYNOPSIS
#include <string.h>
char *strncpy(char *restrict <[dst]>, const char *restrict <[src]>,
size_t <[length]>);
DESCRIPTION
<<strncpy>> copies not more than <[length]> characters from the
the string pointed to by <[src]> (including the terminating
null character) to the array pointed to by <[dst]>. If the
string pointed to by <[src]> is shorter than <[length]>
characters, null characters are appended to the destination
array until a total of <[length]> characters have been
written.
RETURNS
This function returns the initial value of <[dst]>.
PORTABILITY
<<strncpy>> is ANSI C.
<<strncpy>> requires no supporting OS subroutines.
QUICKREF
strncpy ansi pure
*/
#include <string.h>
#include <limits.h>
/*SUPPRESS 560*/
/*SUPPRESS 530*/
/* Nonzero if either X or Y is not aligned on a "long" boundary. */
#define UNALIGNED(X, Y) \
(((long)X & (sizeof (long) - 1)) | ((long)Y & (sizeof (long) - 1)))
#if LONG_MAX == 2147483647L
#define DETECTNULL(X) (((X) - 0x01010101) & ~(X) & 0x80808080)
#else
#if LONG_MAX == 9223372036854775807L
/* Nonzero if X (a long int) contains a NULL byte. */
#define DETECTNULL(X) (((X) - 0x0101010101010101) & ~(X) & 0x8080808080808080)
#else
#error long int is not a 32bit or 64bit type.
#endif
#endif
#ifndef DETECTNULL
#error long int is not a 32bit or 64bit byte
#endif
#undef TOO_SMALL
#define TOO_SMALL(LEN) ((LEN) < sizeof (long))
#if MESOSPHERE_LIBC_STRNCMP_GENERIC
char *
strncpy (char *__restrict dst0,
const char *__restrict src0,
size_t count)
{
#if defined(PREFER_SIZE_OVER_SPEED) || defined(__OPTIMIZE_SIZE__)
char *dscan;
const char *sscan;
dscan = dst0;
sscan = src0;
while (count > 0)
{
--count;
if ((*dscan++ = *sscan++) == '\0')
break;
}
while (count-- > 0)
*dscan++ = '\0';
return dst0;
#else
char *dst = dst0;
const char *src = src0;
long *aligned_dst;
const long *aligned_src;
/* If SRC and DEST is aligned and count large enough, then copy words. */
if (!UNALIGNED (src, dst) && !TOO_SMALL (count))
{
aligned_dst = (long*)dst;
aligned_src = (long*)src;
/* SRC and DEST are both "long int" aligned, try to do "long int"
sized copies. */
while (count >= sizeof (long int) && !DETECTNULL(*aligned_src))
{
count -= sizeof (long int);
*aligned_dst++ = *aligned_src++;
}
dst = (char*)aligned_dst;
src = (char*)aligned_src;
}
while (count > 0)
{
--count;
if ((*dst++ = *src++) == '\0')
break;
}
while (count-- > 0)
*dst++ = '\0';
return dst0;
#endif /* not PREFER_SIZE_OVER_SPEED */
}
#endif /* MESOSPHERE_LIBC_STRNCPY_GENERIC */
/*
FUNCTION
<<strncmp>>---character string compare
INDEX
strncmp
SYNOPSIS
#include <string.h>
int strncmp(const char *<[a]>, const char * <[b]>, size_t <[length]>);
DESCRIPTION
<<strncmp>> compares up to <[length]> characters
from the string at <[a]> to the string at <[b]>.
RETURNS
If <<*<[a]>>> sorts lexicographically after <<*<[b]>>>,
<<strncmp>> returns a number greater than zero. If the two
strings are equivalent, <<strncmp>> returns zero. If <<*<[a]>>>
sorts lexicographically before <<*<[b]>>>, <<strncmp>> returns a
number less than zero.
PORTABILITY
<<strncmp>> is ANSI C.
<<strncmp>> requires no supporting OS subroutines.
QUICKREF
strncmp ansi pure
*/
#include <string.h>
#include <limits.h>
/* Nonzero if either X or Y is not aligned on a "long" boundary. */
#define UNALIGNED(X, Y) \
(((long)X & (sizeof (long) - 1)) | ((long)Y & (sizeof (long) - 1)))
/* DETECTNULL returns nonzero if (long)X contains a NULL byte. */
#if LONG_MAX == 2147483647L
#define DETECTNULL(X) (((X) - 0x01010101) & ~(X) & 0x80808080)
#else
#if LONG_MAX == 9223372036854775807L
#define DETECTNULL(X) (((X) - 0x0101010101010101) & ~(X) & 0x8080808080808080)
#else
#error long int is not a 32bit or 64bit type.
#endif
#endif
#ifndef DETECTNULL
#error long int is not a 32bit or 64bit byte
#endif
#if MESOSPHERE_LIBC_STRNCMP_GENERIC
int
strncmp (const char *s1,
const char *s2,
size_t n)
{
#if defined(PREFER_SIZE_OVER_SPEED) || defined(__OPTIMIZE_SIZE__)
if (n == 0)
return 0;
while (n-- != 0 && *s1 == *s2)
{
if (n == 0 || *s1 == '\0')
break;
s1++;
s2++;
}
return (*(unsigned char *) s1) - (*(unsigned char *) s2);
#else
unsigned long *a1;
unsigned long *a2;
if (n == 0)
return 0;
/* If s1 or s2 are unaligned, then compare bytes. */
if (!UNALIGNED (s1, s2))
{
/* If s1 and s2 are word-aligned, compare them a word at a time. */
a1 = (unsigned long*)s1;
a2 = (unsigned long*)s2;
while (n >= sizeof (long) && *a1 == *a2)
{
n -= sizeof (long);
/* If we've run out of bytes or hit a null, return zero
since we already know *a1 == *a2. */
if (n == 0 || DETECTNULL (*a1))
return 0;
a1++;
a2++;
}
/* A difference was detected in last few bytes of s1, so search bytewise */
s1 = (char*)a1;
s2 = (char*)a2;
}
while (n-- > 0 && *s1 == *s2)
{
/* If we've run out of bytes or hit a null, return zero
since we already know *s1 == *s2. */
if (n == 0 || *s1 == '\0')
return 0;
s1++;
s2++;
}
return (*(unsigned char *) s1) - (*(unsigned char *) s2);
#endif /* not PREFER_SIZE_OVER_SPEED */
}
#endif /* MESOSPHERE_LIBC_STRNCMP_GENERIC */
#ifdef __cplusplus
} /* extern "C" */
#endif