OmniNX/Horizon-OC
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add entire atmosphere source code and also add ram timing fixes

Browse Source
This commit is contained in:
souldbminersmwc
2025-08-31 14:07:54 -04:00
parent fd91bc07d8
commit ca285351db
3743 changed files with 565998 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

210
Source/Atmosphere/libraries/libmesosphere/source/arch/arm/kern_generic_interrupt_controller.inc Normal file
View File

@@ -0,0 +1,210 @@
/*
* Copyright (c) Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <mesosphere.hpp>
namespace ams::kern::arch::arm {
void KInterruptController::SetupInterruptLines(s32 core_id) const {
const size_t ITLines = (core_id == 0) ? 32 * ((m_gicd->typer & 0x1F) + 1) : NumLocalInterrupts;
for (size_t i = 0; i < ITLines / 32; i++) {
m_gicd->icenabler[i] = 0xFFFFFFFF;
m_gicd->icpendr[i] = 0xFFFFFFFF;
m_gicd->icactiver[i] = 0xFFFFFFFF;
m_gicd->igroupr[i] = 0;
}
for (size_t i = 0; i < ITLines; i++) {
m_gicd->ipriorityr.bytes[i] = 0xFF;
m_gicd->itargetsr.bytes[i] = 0x00;
}
for (size_t i = 0; i < ITLines / 16; i++) {
m_gicd->icfgr[i] = 0x00000000;
}
}
void KInterruptController::Initialize(s32 core_id) {
/* Setup pointers to ARM mmio. */
m_gicd = GetPointer<volatile GicDistributor >(KMemoryLayout::GetDeviceVirtualAddress(KMemoryRegionType_InterruptDistributor));
m_gicc = GetPointer<volatile GicCpuInterface>(KMemoryLayout::GetDeviceVirtualAddress(KMemoryRegionType_InterruptCpuInterface));
/* Clear CTLRs. */
m_gicc->ctlr = 0;
if (core_id == 0) {
m_gicd->ctlr = 0;
}
m_gicc->pmr = 0;
m_gicc->bpr = 7;
/* Setup all interrupt lines. */
SetupInterruptLines(core_id);
/* Set CTLRs. */
if (core_id == 0) {
m_gicd->ctlr = 1;
}
m_gicc->ctlr = 1;
/* Set the mask for this core. */
SetGicMask(core_id);
/* Set the priority level. */
SetPriorityLevel(PriorityLevel_Low);
}
void KInterruptController::Finalize(s32 core_id) {
/* Clear CTLRs. */
if (core_id == 0) {
m_gicd->ctlr = 0;
}
m_gicc->ctlr = 0;
/* Set the priority level. */
SetPriorityLevel(PriorityLevel_High);
/* Setup all interrupt lines. */
SetupInterruptLines(core_id);
/* Clear pointers, if needed. */
if (core_id == 0) {
m_gicd = nullptr;
m_gicc = nullptr;
}
}
void KInterruptController::SaveCoreLocal(LocalState *state) const {
/* Save isenabler. */
for (size_t i = 0; i < util::size(state->isenabler); ++i) {
constexpr size_t Offset = 0;
state->isenabler[i] = m_gicd->isenabler[i + Offset];
m_gicd->isenabler[i + Offset] = 0xFFFFFFFF;
}
/* Save ipriorityr. */
for (size_t i = 0; i < util::size(state->ipriorityr); ++i) {
constexpr size_t Offset = 0;
state->ipriorityr[i] = m_gicd->ipriorityr.words[i + Offset];
m_gicd->ipriorityr.words[i + Offset] = 0xFFFFFFFF;
}
/* Save itargetsr. */
for (size_t i = 0; i < util::size(state->itargetsr); ++i) {
constexpr size_t Offset = 0;
state->itargetsr[i] = m_gicd->itargetsr.words[i + Offset];
}
/* Save icfgr. */
for (size_t i = 0; i < util::size(state->icfgr); ++i) {
constexpr size_t Offset = 0;
state->icfgr[i] = m_gicd->icfgr[i + Offset];
}
/* Save spendsgir. */
for (size_t i = 0; i < util::size(state->spendsgir); ++i) {
state->spendsgir[i] = m_gicd->spendsgir[i];
}
}
void KInterruptController::SaveGlobal(GlobalState *state) const {
/* Save isenabler. */
for (size_t i = 0; i < util::size(state->isenabler); ++i) {
constexpr size_t Offset = util::size(LocalState{}.isenabler);
state->isenabler[i] = m_gicd->isenabler[i + Offset];
m_gicd->isenabler[i + Offset] = 0xFFFFFFFF;
}
/* Save ipriorityr. */
for (size_t i = 0; i < util::size(state->ipriorityr); ++i) {
constexpr size_t Offset = util::size(LocalState{}.ipriorityr);
state->ipriorityr[i] = m_gicd->ipriorityr.words[i + Offset];
m_gicd->ipriorityr.words[i + Offset] = 0xFFFFFFFF;
}
/* Save itargetsr. */
for (size_t i = 0; i < util::size(state->itargetsr); ++i) {
constexpr size_t Offset = util::size(LocalState{}.itargetsr);
state->itargetsr[i] = m_gicd->itargetsr.words[i + Offset];
}
/* Save icfgr. */
for (size_t i = 0; i < util::size(state->icfgr); ++i) {
constexpr size_t Offset = util::size(LocalState{}.icfgr);
state->icfgr[i] = m_gicd->icfgr[i + Offset];
}
}
void KInterruptController::RestoreCoreLocal(const LocalState *state) const {
/* Restore ipriorityr. */
for (size_t i = 0; i < util::size(state->ipriorityr); ++i) {
constexpr size_t Offset = 0;
m_gicd->ipriorityr.words[i + Offset] = state->ipriorityr[i];
}
/* Restore itargetsr. */
for (size_t i = 0; i < util::size(state->itargetsr); ++i) {
constexpr size_t Offset = 0;
m_gicd->itargetsr.words[i + Offset] = state->itargetsr[i];
}
/* Restore icfgr. */
for (size_t i = 0; i < util::size(state->icfgr); ++i) {
constexpr size_t Offset = 0;
m_gicd->icfgr[i + Offset] = state->icfgr[i];
}
/* Restore isenabler. */
for (size_t i = 0; i < util::size(state->isenabler); ++i) {
constexpr size_t Offset = 0;
m_gicd->icenabler[i + Offset] = 0xFFFFFFFF;
m_gicd->isenabler[i + Offset] = state->isenabler[i];
}
/* Restore spendsgir. */
for (size_t i = 0; i < util::size(state->spendsgir); ++i) {
m_gicd->spendsgir[i] = state->spendsgir[i];
}
}
void KInterruptController::RestoreGlobal(const GlobalState *state) const {
/* Restore ipriorityr. */
for (size_t i = 0; i < util::size(state->ipriorityr); ++i) {
constexpr size_t Offset = util::size(LocalState{}.ipriorityr);
m_gicd->ipriorityr.words[i + Offset] = state->ipriorityr[i];
}
/* Restore itargetsr. */
for (size_t i = 0; i < util::size(state->itargetsr); ++i) {
constexpr size_t Offset = util::size(LocalState{}.itargetsr);
m_gicd->itargetsr.words[i + Offset] = state->itargetsr[i];
}
/* Restore icfgr. */
for (size_t i = 0; i < util::size(state->icfgr); ++i) {
constexpr size_t Offset = util::size(LocalState{}.icfgr);
m_gicd->icfgr[i + Offset] = state->icfgr[i];
}
/* Restore isenabler. */
for (size_t i = 0; i < util::size(state->isenabler); ++i) {
constexpr size_t Offset = util::size(LocalState{}.isenabler);
m_gicd->icenabler[i + Offset] = 0xFFFFFFFF;
m_gicd->isenabler[i + Offset] = state->isenabler[i];
}
}
}

19
Source/Atmosphere/libraries/libmesosphere/source/arch/arm/kern_k_interrupt_controller.board.generic.cpp Normal file
View File

@@ -0,0 +1,19 @@
/*
* Copyright (c) Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <mesosphere.hpp>
/* Include the common implementation. */
#include "../arm/kern_generic_interrupt_controller.inc"

528
Source/Atmosphere/libraries/libmesosphere/source/arch/arm64/kern_cpu.cpp Normal file
View File

@@ -0,0 +1,528 @@
/*
* Copyright (c) Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <mesosphere.hpp>
namespace ams::kern::arch::arm64::cpu {
/* Declare prototype to be implemented in asm. */
void SynchronizeAllCoresImpl(s32 *sync_var, s32 num_cores);
namespace {
ALWAYS_INLINE void SetEventLocally() {
__asm__ __volatile__("sevl" ::: "memory");
}
ALWAYS_INLINE void WaitForEvent() {
__asm__ __volatile__("wfe" ::: "memory");
}
class KScopedCoreMigrationDisable {
public:
ALWAYS_INLINE KScopedCoreMigrationDisable() { GetCurrentThread().DisableCoreMigration(); }
ALWAYS_INLINE ~KScopedCoreMigrationDisable() { GetCurrentThread().EnableCoreMigration(); }
};
class KScopedCacheMaintenance {
private:
bool m_active;
public:
ALWAYS_INLINE KScopedCacheMaintenance() {
__asm__ __volatile__("" ::: "memory");
if (m_active = !GetCurrentThread().IsInCacheMaintenanceOperation(); m_active) {
GetCurrentThread().SetInCacheMaintenanceOperation();
}
}
ALWAYS_INLINE ~KScopedCacheMaintenance() {
if (m_active) {
GetCurrentThread().ClearInCacheMaintenanceOperation();
}
__asm__ __volatile__("" ::: "memory");
}
};
/* Nintendo registers a handler for a SGI on thread termination, but does not handle anything. */
/* This is sufficient, because post-interrupt scheduling is all they really intend to occur. */
class KThreadTerminationInterruptHandler : public KInterruptHandler {
public:
constexpr KThreadTerminationInterruptHandler() : KInterruptHandler() { /* ... */ }
virtual KInterruptTask *OnInterrupt(s32 interrupt_id) override {
MESOSPHERE_UNUSED(interrupt_id);
return nullptr;
}
};
class KPerformanceCounterInterruptHandler : public KInterruptHandler {
private:
static constinit inline KLightLock s_lock;
private:
u64 m_counter;
s32 m_which;
bool m_done;
public:
constexpr KPerformanceCounterInterruptHandler() : KInterruptHandler(), m_counter(), m_which(), m_done() { /* ... */ }
static KLightLock &GetLock() { return s_lock; }
void Setup(s32 w) {
m_done = false;
m_which = w;
}
void Wait() {
while (!m_done) {
cpu::Yield();
}
}
u64 GetCounter() const { return m_counter; }
/* Nintendo misuses this per their own API, but it's functional. */
virtual KInterruptTask *OnInterrupt(s32 interrupt_id) override {
MESOSPHERE_UNUSED(interrupt_id);
if (m_which < 0) {
m_counter = cpu::GetCycleCounter();
} else {
m_counter = cpu::GetPerformanceCounter(m_which);
}
DataMemoryBarrierInnerShareable();
m_done = true;
return nullptr;
}
};
class KCoreBarrierInterruptHandler : public KInterruptHandler {
private:
util::Atomic<u64> m_target_cores;
KLightLock m_lock;
public:
constexpr KCoreBarrierInterruptHandler() : KInterruptHandler(), m_target_cores(0), m_lock() { /* ... */ }
virtual KInterruptTask *OnInterrupt(s32 interrupt_id) override {
MESOSPHERE_UNUSED(interrupt_id);
m_target_cores &= ~(1ul << GetCurrentCoreId());
return nullptr;
}
void SynchronizeCores(u64 core_mask) {
/* Acquire exclusive access to ourselves. */
KScopedLightLock lk(m_lock);
/* If necessary, force synchronization with other cores. */
if (const u64 other_cores_mask = core_mask & ~(1ul << GetCurrentCoreId()); other_cores_mask != 0) {
/* Send an interrupt to the other cores. */
m_target_cores = other_cores_mask;
cpu::DataSynchronizationBarrierInnerShareable();
Kernel::GetInterruptManager().SendInterProcessorInterrupt(KInterruptName_CoreBarrier, other_cores_mask);
/* Wait for all cores to acknowledge. */
{
u64 v;
__asm__ __volatile__("ldaxr %[v], %[p]\n"
"cbz %[v], 1f\n"
"0:\n"
"wfe\n"
"ldaxr %[v], %[p]\n"
"cbnz %[v], 0b\n"
"1:\n"
: [v]"=&r"(v)
: [p]"Q"(*reinterpret_cast<u64 *>(std::addressof(m_target_cores)))
: "memory");
}
}
}
};
class KCacheHelperInterruptHandler : public KInterruptHandler {
private:
static constexpr s32 ThreadPriority = 8;
public:
enum class Operation {
Idle,
InstructionMemoryBarrier,
StoreDataCache,
FlushDataCache,
};
private:
KLightLock m_lock;
KLightLock m_cv_lock;
KLightConditionVariable m_cv;
util::Atomic<u64> m_target_cores;
volatile Operation m_operation;
private:
static void ThreadFunction(uintptr_t _this) {
reinterpret_cast<KCacheHelperInterruptHandler *>(_this)->ThreadFunctionImpl();
}
void ThreadFunctionImpl() {
const u64 core_mask = (1ul << GetCurrentCoreId());
while (true) {
/* Wait for a request to come in. */
{
KScopedLightLock lk(m_cv_lock);
while ((m_target_cores.Load() & core_mask) == 0) {
m_cv.Wait(std::addressof(m_cv_lock));
}
}
/* Process the request. */
this->ProcessOperation();
/* Broadcast, if there's nothing pending. */
{
KScopedLightLock lk(m_cv_lock);
m_target_cores &= ~core_mask;
if (m_target_cores.Load() == 0) {
m_cv.Broadcast();
}
}
}
}
void ProcessOperation();
public:
constexpr KCacheHelperInterruptHandler() : KInterruptHandler(), m_lock(), m_cv_lock(), m_cv(util::ConstantInitialize), m_target_cores(0), m_operation(Operation::Idle) { /* ... */ }
void Initialize(s32 core_id) {
/* Reserve a thread from the system limit. */
MESOSPHERE_ABORT_UNLESS(Kernel::GetSystemResourceLimit().Reserve(ams::svc::LimitableResource_ThreadCountMax, 1));
/* Create a new thread. */
KThread *new_thread = KThread::Create();
MESOSPHERE_ABORT_UNLESS(new_thread != nullptr);
MESOSPHERE_R_ABORT_UNLESS(KThread::InitializeKernelThread(new_thread, ThreadFunction, reinterpret_cast<uintptr_t>(this), ThreadPriority, core_id));
/* Register the new thread. */
KThread::Register(new_thread);
/* Run the thread. */
new_thread->Run();
}
virtual KInterruptTask *OnInterrupt(s32 interrupt_id) override {
MESOSPHERE_UNUSED(interrupt_id);
this->ProcessOperation();
m_target_cores &= ~(1ul << GetCurrentCoreId());
return nullptr;
}
void RequestOperation(Operation op) {
KScopedLightLock lk(m_lock);
/* Create core masks for us to use. */
constexpr u64 AllCoresMask = (1ul << cpu::NumCores) - 1ul;
const u64 other_cores_mask = AllCoresMask & ~(1ul << GetCurrentCoreId());
if ((op == Operation::InstructionMemoryBarrier) || (Kernel::GetState() == Kernel::State::Initializing)) {
/* Check that there's no on-going operation. */
MESOSPHERE_ABORT_UNLESS(m_operation == Operation::Idle);
MESOSPHERE_ABORT_UNLESS(m_target_cores.Load() == 0);
/* Set operation. */
m_operation = op;
/* For certain operations, we want to send an interrupt. */
m_target_cores = other_cores_mask;
const u64 target_mask = m_target_cores.Load();
DataSynchronizationBarrierInnerShareable();
Kernel::GetInterruptManager().SendInterProcessorInterrupt(KInterruptName_CacheOperation, target_mask);
this->ProcessOperation();
while (m_target_cores.Load() != 0) {
cpu::Yield();
}
/* Go idle again. */
m_operation = Operation::Idle;
} else {
/* Lock condvar so that we can send and wait for acknowledgement of request. */
KScopedLightLock cv_lk(m_cv_lock);
/* Check that there's no on-going operation. */
MESOSPHERE_ABORT_UNLESS(m_operation == Operation::Idle);
MESOSPHERE_ABORT_UNLESS(m_target_cores.Load() == 0);
/* Set operation. */
m_operation = op;
/* Request all cores. */
m_target_cores = AllCoresMask;
/* Use the condvar. */
m_cv.Broadcast();
while (m_target_cores.Load() != 0) {
m_cv.Wait(std::addressof(m_cv_lock));
}
/* Go idle again. */
m_operation = Operation::Idle;
}
}
};
/* Instances of the interrupt handlers. */
constinit KThreadTerminationInterruptHandler g_thread_termination_handler;
constinit KCacheHelperInterruptHandler g_cache_operation_handler;
constinit KCoreBarrierInterruptHandler g_core_barrier_handler;
#if defined(MESOSPHERE_ENABLE_PERFORMANCE_COUNTER)
constinit KPerformanceCounterInterruptHandler g_performance_counter_handler[cpu::NumCores];
#endif
/* Expose this as a global, for asm to use. */
constinit s32 g_all_core_sync_count;
template<typename F>
ALWAYS_INLINE void PerformCacheOperationBySetWayImpl(int level, F f) {
/* Used in multiple locations. */
const u64 level_sel_value = static_cast<u64>(level << 1);
/* Get the cache size id register value with interrupts disabled. */
u64 ccsidr_value;
{
/* Disable interrupts. */
KScopedInterruptDisable di;
/* Configure the cache select register for our level. */
cpu::SetCsselrEl1(level_sel_value);
/* Ensure our configuration takes before reading the cache size id register. */
cpu::InstructionMemoryBarrier();
/* Get the cache size id register. */
ccsidr_value = cpu::GetCcsidrEl1();
}
/* Ensure that no memory inconsistencies occur between cache management invocations. */
cpu::DataSynchronizationBarrier();
/* Get cache size id info. */
CacheSizeIdRegisterAccessor ccsidr_el1(ccsidr_value);
const int num_sets = ccsidr_el1.GetNumberOfSets();
const int num_ways = ccsidr_el1.GetAssociativity();
const int line_size = ccsidr_el1.GetLineSize();
const u64 way_shift = static_cast<u64>(__builtin_clz(num_ways));
const u64 set_shift = static_cast<u64>(line_size + 4);
for (int way = 0; way <= num_ways; way++) {
for (int set = 0; set <= num_sets; set++) {
const u64 way_value = static_cast<u64>(way) << way_shift;
const u64 set_value = static_cast<u64>(set) << set_shift;
f(way_value | set_value | level_sel_value);
}
}
}
ALWAYS_INLINE void FlushDataCacheLineBySetWayImpl(const u64 sw_value) {
__asm__ __volatile__("dc cisw, %[v]" :: [v]"r"(sw_value) : "memory");
}
ALWAYS_INLINE void StoreDataCacheLineBySetWayImpl(const u64 sw_value) {
__asm__ __volatile__("dc csw, %[v]" :: [v]"r"(sw_value) : "memory");
}
void StoreDataCacheBySetWay(int level) {
PerformCacheOperationBySetWayImpl(level, StoreDataCacheLineBySetWayImpl);
}
void FlushDataCacheBySetWay(int level) {
PerformCacheOperationBySetWayImpl(level, FlushDataCacheLineBySetWayImpl);
}
void KCacheHelperInterruptHandler::ProcessOperation() {
switch (m_operation) {
case Operation::Idle:
break;
case Operation::InstructionMemoryBarrier:
InstructionMemoryBarrier();
break;
case Operation::StoreDataCache:
StoreDataCacheBySetWay(0);
cpu::DataSynchronizationBarrier();
break;
case Operation::FlushDataCache:
FlushDataCacheBySetWay(0);
cpu::DataSynchronizationBarrier();
break;
}
}
ALWAYS_INLINE Result InvalidateDataCacheRange(uintptr_t start, uintptr_t end) {
MESOSPHERE_ASSERT(util::IsAligned(start, DataCacheLineSize));
MESOSPHERE_ASSERT(util::IsAligned(end, DataCacheLineSize));
R_UNLESS(UserspaceAccess::InvalidateDataCache(start, end), svc::ResultInvalidCurrentMemory());
DataSynchronizationBarrier();
R_SUCCEED();
}
ALWAYS_INLINE Result StoreDataCacheRange(uintptr_t start, uintptr_t end) {
MESOSPHERE_ASSERT(util::IsAligned(start, DataCacheLineSize));
MESOSPHERE_ASSERT(util::IsAligned(end, DataCacheLineSize));
R_UNLESS(UserspaceAccess::StoreDataCache(start, end), svc::ResultInvalidCurrentMemory());
DataSynchronizationBarrier();
R_SUCCEED();
}
ALWAYS_INLINE Result FlushDataCacheRange(uintptr_t start, uintptr_t end) {
MESOSPHERE_ASSERT(util::IsAligned(start, DataCacheLineSize));
MESOSPHERE_ASSERT(util::IsAligned(end, DataCacheLineSize));
R_UNLESS(UserspaceAccess::FlushDataCache(start, end), svc::ResultInvalidCurrentMemory());
DataSynchronizationBarrier();
R_SUCCEED();
}
ALWAYS_INLINE void InvalidateEntireInstructionCacheLocalImpl() {
__asm__ __volatile__("ic iallu" ::: "memory");
}
ALWAYS_INLINE void InvalidateEntireInstructionCacheGlobalImpl() {
__asm__ __volatile__("ic ialluis" ::: "memory");
}
}
void SynchronizeCores(u64 core_mask) {
/* Request a core barrier interrupt. */
g_core_barrier_handler.SynchronizeCores(core_mask);
}
void StoreCacheForInit(void *addr, size_t size) {
/* Store the data cache for the specified range. */
const uintptr_t start = util::AlignDown(reinterpret_cast<uintptr_t>(addr), DataCacheLineSize);
const uintptr_t end = start + size;
for (uintptr_t cur = start; cur < end; cur += DataCacheLineSize) {
__asm__ __volatile__("dc cvac, %[cur]" :: [cur]"r"(cur) : "memory");
}
/* Data synchronization barrier. */
DataSynchronizationBarrierInnerShareable();
/* Invalidate instruction cache. */
InvalidateEntireInstructionCacheLocalImpl();
/* Ensure local instruction consistency. */
EnsureInstructionConsistency();
}
void FlushEntireDataCache() {
KScopedCoreMigrationDisable dm;
CacheLineIdRegisterAccessor clidr_el1;
const int levels_of_coherency = clidr_el1.GetLevelsOfCoherency();
/* Store cache from L2 up to the level of coherence (if there's an L3 cache or greater). */
for (int level = 2; level < levels_of_coherency; ++level) {
StoreDataCacheBySetWay(level - 1);
}
/* Flush cache from the level of coherence down to L2. */
for (int level = levels_of_coherency; level > 1; --level) {
FlushDataCacheBySetWay(level - 1);
}
/* Data synchronization barrier for full system. */
DataSynchronizationBarrier();
}
Result InvalidateDataCache(void *addr, size_t size) {
/* Mark ourselves as in a cache maintenance operation, and prevent re-ordering. */
KScopedCacheMaintenance cm;
const uintptr_t start = reinterpret_cast<uintptr_t>(addr);
const uintptr_t end = start + size;
uintptr_t aligned_start = util::AlignDown(start, DataCacheLineSize);
uintptr_t aligned_end = util::AlignUp(end, DataCacheLineSize);
if (aligned_start != start) {
R_TRY(FlushDataCacheRange(aligned_start, aligned_start + DataCacheLineSize));
aligned_start += DataCacheLineSize;
}
if (aligned_start < aligned_end && (aligned_end != end)) {
aligned_end -= DataCacheLineSize;
R_TRY(FlushDataCacheRange(aligned_end, aligned_end + DataCacheLineSize));
}
if (aligned_start < aligned_end) {
R_TRY(InvalidateDataCacheRange(aligned_start, aligned_end));
}
R_SUCCEED();
}
Result StoreDataCache(const void *addr, size_t size) {
/* Mark ourselves as in a cache maintenance operation, and prevent re-ordering. */
KScopedCacheMaintenance cm;
const uintptr_t start = util::AlignDown(reinterpret_cast<uintptr_t>(addr), DataCacheLineSize);
const uintptr_t end = util::AlignUp( reinterpret_cast<uintptr_t>(addr) + size, DataCacheLineSize);
R_RETURN(StoreDataCacheRange(start, end));
}
Result FlushDataCache(const void *addr, size_t size) {
/* Mark ourselves as in a cache maintenance operation, and prevent re-ordering. */
KScopedCacheMaintenance cm;
const uintptr_t start = util::AlignDown(reinterpret_cast<uintptr_t>(addr), DataCacheLineSize);
const uintptr_t end = util::AlignUp( reinterpret_cast<uintptr_t>(addr) + size, DataCacheLineSize);
R_RETURN(FlushDataCacheRange(start, end));
}
void InvalidateEntireInstructionCache() {
KScopedCoreMigrationDisable dm;
/* Invalidate the instruction cache on all cores. */
InvalidateEntireInstructionCacheGlobalImpl();
EnsureInstructionConsistency();
/* Request the interrupt helper to perform an instruction memory barrier. */
g_cache_operation_handler.RequestOperation(KCacheHelperInterruptHandler::Operation::InstructionMemoryBarrier);
}
void InitializeInterruptThreads(s32 core_id) {
/* Initialize the cache operation handler. */
g_cache_operation_handler.Initialize(core_id);
/* Bind all handlers to the relevant interrupts. */
Kernel::GetInterruptManager().BindHandler(std::addressof(g_cache_operation_handler), KInterruptName_CacheOperation, core_id, KInterruptController::PriorityLevel_High, false, false);
Kernel::GetInterruptManager().BindHandler(std::addressof(g_thread_termination_handler), KInterruptName_ThreadTerminate, core_id, KInterruptController::PriorityLevel_Scheduler, false, false);
Kernel::GetInterruptManager().BindHandler(std::addressof(g_core_barrier_handler), KInterruptName_CoreBarrier, core_id, KInterruptController::PriorityLevel_Scheduler, false, false);
/* If we should, enable user access to the performance counter registers. */
if (KTargetSystem::IsUserPmuAccessEnabled()) { SetPmUserEnrEl0(1ul); }
/* If we should, enable the kernel performance counter interrupt handler. */
#if defined(MESOSPHERE_ENABLE_PERFORMANCE_COUNTER)
Kernel::GetInterruptManager().BindHandler(std::addressof(g_performance_counter_handler[core_id]), KInterruptName_PerformanceCounter, core_id, KInterruptController::PriorityLevel_Timer, false, false);
#endif
}
void SynchronizeAllCores() {
SynchronizeAllCoresImpl(&g_all_core_sync_count, static_cast<s32>(cpu::NumCores));
}
}

198
Source/Atmosphere/libraries/libmesosphere/source/arch/arm64/kern_cpu_asm.s Normal file
View File

@@ -0,0 +1,198 @@
/*
* Copyright (c) Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
/* ams::kern::arch::arm64::cpu::SynchronizeAllCoresImpl(int *sync_var, int num_cores) */
.section .text._ZN3ams4kern4arch5arm643cpu23SynchronizeAllCoresImplEPii, "ax", %progbits
.global _ZN3ams4kern4arch5arm643cpu23SynchronizeAllCoresImplEPii
.type _ZN3ams4kern4arch5arm643cpu23SynchronizeAllCoresImplEPii, %function
_ZN3ams4kern4arch5arm643cpu23SynchronizeAllCoresImplEPii:
/* Loop until the sync var is less than num cores. */
sevl
1:
wfe
ldaxr w2, [x0]
cmp w2, w1
b.gt 1b
/* Increment the sync var. */
2:
ldaxr w2, [x0]
add w3, w2, #1
stlxr w4, w3, [x0]
cbnz w4, 2b
/* Loop until the sync var matches our ticket. */
add w3, w2, w1
sevl
3:
wfe
ldaxr w2, [x0]
cmp w2, w3
b.ne 3b
/* Check if the ticket is the last. */
sub w2, w1, #1
add w2, w2, w1
cmp w3, w2
b.eq 5f
/* Our ticket is not the last one. Increment. */
4:
ldaxr w2, [x0]
add w3, w2, #1
stlxr w4, w3, [x0]
cbnz w4, 4b
ret
/* Our ticket is the last one. */
5:
stlr wzr, [x0]
ret
/* ams::kern::arch::arm64::cpu::ClearPageToZeroImpl(void *) */
.section .text._ZN3ams4kern4arch5arm643cpu19ClearPageToZeroImplEPv, "ax", %progbits
.global _ZN3ams4kern4arch5arm643cpu19ClearPageToZeroImplEPv
.type _ZN3ams4kern4arch5arm643cpu19ClearPageToZeroImplEPv, %function
_ZN3ams4kern4arch5arm643cpu19ClearPageToZeroImplEPv:
/* Efficiently clear the page using dc zva. */
dc zva, x0
add x8, x0, #0x040
dc zva, x8
add x8, x0, #0x080
dc zva, x8
add x8, x0, #0x0c0
dc zva, x8
add x8, x0, #0x100
dc zva, x8
add x8, x0, #0x140
dc zva, x8
add x8, x0, #0x180
dc zva, x8
add x8, x0, #0x1c0
dc zva, x8
add x8, x0, #0x200
dc zva, x8
add x8, x0, #0x240
dc zva, x8
add x8, x0, #0x280
dc zva, x8
add x8, x0, #0x2c0
dc zva, x8
add x8, x0, #0x300
dc zva, x8
add x8, x0, #0x340
dc zva, x8
add x8, x0, #0x380
dc zva, x8
add x8, x0, #0x3c0
dc zva, x8
add x8, x0, #0x400
dc zva, x8
add x8, x0, #0x440
dc zva, x8
add x8, x0, #0x480
dc zva, x8
add x8, x0, #0x4c0
dc zva, x8
add x8, x0, #0x500
dc zva, x8
add x8, x0, #0x540
dc zva, x8
add x8, x0, #0x580
dc zva, x8
add x8, x0, #0x5c0
dc zva, x8
add x8, x0, #0x600
dc zva, x8
add x8, x0, #0x640
dc zva, x8
add x8, x0, #0x680
dc zva, x8
add x8, x0, #0x6c0
dc zva, x8
add x8, x0, #0x700
dc zva, x8
add x8, x0, #0x740
dc zva, x8
add x8, x0, #0x780
dc zva, x8
add x8, x0, #0x7c0
dc zva, x8
add x8, x0, #0x800
dc zva, x8
add x8, x0, #0x840
dc zva, x8
add x8, x0, #0x880
dc zva, x8
add x8, x0, #0x8c0
dc zva, x8
add x8, x0, #0x900
dc zva, x8
add x8, x0, #0x940
dc zva, x8
add x8, x0, #0x980
dc zva, x8
add x8, x0, #0x9c0
dc zva, x8
add x8, x0, #0xa00
dc zva, x8
add x8, x0, #0xa40
dc zva, x8
add x8, x0, #0xa80
dc zva, x8
add x8, x0, #0xac0
dc zva, x8
add x8, x0, #0xb00
dc zva, x8
add x8, x0, #0xb40
dc zva, x8
add x8, x0, #0xb80
dc zva, x8
add x8, x0, #0xbc0
dc zva, x8
add x8, x0, #0xc00
dc zva, x8
add x8, x0, #0xc40
dc zva, x8
add x8, x0, #0xc80
dc zva, x8
add x8, x0, #0xcc0
dc zva, x8
add x8, x0, #0xd00
dc zva, x8
add x8, x0, #0xd40
dc zva, x8
add x8, x0, #0xd80
dc zva, x8
add x8, x0, #0xdc0
dc zva, x8
add x8, x0, #0xe00
dc zva, x8
add x8, x0, #0xe40
dc zva, x8
add x8, x0, #0xe80
dc zva, x8
add x8, x0, #0xec0
dc zva, x8
add x8, x0, #0xf00
dc zva, x8
add x8, x0, #0xf40
dc zva, x8
add x8, x0, #0xf80
dc zva, x8
add x8, x0, #0xfc0
dc zva, x8
ret

656
Source/Atmosphere/libraries/libmesosphere/source/arch/arm64/kern_exception_handlers.cpp Normal file
View File

@@ -0,0 +1,656 @@
/*
* Copyright (c) Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <mesosphere.hpp>
namespace ams::kern::svc {
void RestoreContext(uintptr_t sp);
}
namespace ams::kern::arch::arm64 {
namespace {
enum EsrEc : u32 {
EsrEc_Unknown = 0b000000,
EsrEc_WaitForInterruptOrEvent = 0b000001,
EsrEc_Cp15McrMrc = 0b000011,
EsrEc_Cp15McrrMrrc = 0b000100,
EsrEc_Cp14McrMrc = 0b000101,
EsrEc_FpAccess = 0b000111,
EsrEc_Cp14Mrrc = 0b001100,
EsrEc_BranchTarget = 0b001101,
EsrEc_IllegalExecution = 0b001110,
EsrEc_Svc32 = 0b010001,
EsrEc_Svc64 = 0b010101,
EsrEc_SystemInstruction64 = 0b011000,
EsrEc_SveZen = 0b011001,
EsrEc_PointerAuthInstruction = 0b011100,
EsrEc_InstructionAbortEl0 = 0b100000,
EsrEc_InstructionAbortEl1 = 0b100001,
EsrEc_PcAlignmentFault = 0b100010,
EsrEc_DataAbortEl0 = 0b100100,
EsrEc_DataAbortEl1 = 0b100101,
EsrEc_SpAlignmentFault = 0b100110,
EsrEc_FpException32 = 0b101000,
EsrEc_FpException64 = 0b101100,
EsrEc_SErrorInterrupt = 0b101111,
EsrEc_BreakPointEl0 = 0b110000,
EsrEc_BreakPointEl1 = 0b110001,
EsrEc_SoftwareStepEl0 = 0b110010,
EsrEc_SoftwareStepEl1 = 0b110011,
EsrEc_WatchPointEl0 = 0b110100,
EsrEc_WatchPointEl1 = 0b110101,
EsrEc_BkptInstruction = 0b111000,
EsrEc_BrkInstruction = 0b111100,
};
u32 GetInstructionDataSupervisorMode(const KExceptionContext *context, u64 esr) {
/* Check for THUMB usermode */
if ((context->psr & 0x3F) == 0x30) {
u32 insn = *reinterpret_cast<u16 *>(context->pc & ~0x1);
/* Check if the instruction was 32-bit. */
if ((esr >> 25) & 1) {
insn = (insn << 16) | *reinterpret_cast<u16 *>((context->pc & ~0x1) + sizeof(u16));
}
return insn;
} else {
/* Not thumb, so just get the instruction. */
return *reinterpret_cast<u32 *>(context->pc);
}
}
u32 GetInstructionDataUserMode(const KExceptionContext *context) {
/* Check for THUMB usermode */
u32 insn = 0;
if ((context->psr & 0x3F) == 0x30) {
u16 insn_high = 0;
if (UserspaceAccess::CopyMemoryFromUser(std::addressof(insn_high), reinterpret_cast<u16 *>(context->pc & ~0x1), sizeof(insn_high))) {
insn = insn_high;
/* Check if the instruction was a THUMB mode branch prefix. */
if (((insn >> 11) & 0b11110) == 0b11110) {
u16 insn_low = 0;
if (UserspaceAccess::CopyMemoryFromUser(std::addressof(insn_low), reinterpret_cast<u16 *>((context->pc & ~0x1) + sizeof(u16)), sizeof(insn_low))) {
insn = (static_cast<u32>(insn_high) << 16) | (static_cast<u32>(insn_low) << 0);
} else {
insn = 0;
}
}
} else {
insn = 0;
}
} else {
u32 insn_value = 0;
if (UserspaceAccess::CopyMemoryFromUser(std::addressof(insn_value), reinterpret_cast<u32 *>(context->pc), sizeof(insn_value))) {
insn = insn_value;
} else if (KTargetSystem::IsDebugMode() && (context->pc & 3) == 0 && UserspaceAccess::CopyMemoryFromUserSize32BitWithSupervisorAccess(std::addressof(insn_value), reinterpret_cast<u32 *>(context->pc))) {
insn = insn_value;
} else {
insn = 0;
}
}
return insn;
}
void HandleUserException(KExceptionContext *context, u64 raw_esr, u64 raw_far, u64 afsr0, u64 afsr1, u32 data) {
/* Pre-process exception registers as needed. */
u64 esr = raw_esr;
u64 far = raw_far;
const u64 ec = (esr >> 26) & 0x3F;
if (ec == EsrEc_InstructionAbortEl0 || ec == EsrEc_DataAbortEl0) {
/* Adjust registers if a synchronous external abort has occurred with far not valid. */
/* Mask 0x03F = Low 6 bits IFSC == 0x10: "Synchronous External abort, */
/* not on translation table walk or hardware update of translation table. */
/* Mask 0x400 = FnV = "FAR Not Valid" */
/* TODO: How would we perform this check using named register accesses? */
if ((esr & 0x43F) == 0x410) {
/* Clear the faulting register on memory tagging exception. */
far = 0;
} else {
/* If the faulting address is a kernel address, set ISFC = 4. */
if (far >= ams::svc::AddressMemoryRegion39Size) {
esr = (esr & 0xFFFFFFC0) | 4;
}
}
}
KProcess &cur_process = GetCurrentProcess();
bool should_process_user_exception = KTargetSystem::IsUserExceptionHandlersEnabled();
/* In the event that we return from this exception, we want SPSR.SS set so that we advance an instruction if single-stepping. */
#if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP)
context->psr |= (1ul << 21);
#endif
/* If we should process the user exception (and it's not a breakpoint), try to enter. */
const bool is_software_break = (ec == EsrEc_Unknown || ec == EsrEc_IllegalExecution || ec == EsrEc_BkptInstruction || ec == EsrEc_BrkInstruction);
const bool is_breakpoint = (ec == EsrEc_BreakPointEl0 || ec == EsrEc_SoftwareStepEl0 || ec == EsrEc_WatchPointEl0);
if ((should_process_user_exception) &&
!(is_software_break && cur_process.IsAttachedToDebugger() && KDebug::IsBreakInstruction(data, context->psr)) &&
!(is_breakpoint))
{
if (cur_process.EnterUserException()) {
/* Fill out the exception info. */
const bool is_aarch64 = (context->psr & 0x10) == 0;
if (is_aarch64) {
/* 64-bit. */
ams::svc::aarch64::ExceptionInfo *info = std::addressof(static_cast<ams::svc::aarch64::ProcessLocalRegion *>(cur_process.GetProcessLocalRegionHeapAddress())->exception_info);
for (size_t i = 0; i < util::size(info->r); ++i) {
info->r[i] = context->x[i];
}
info->sp = context->sp;
info->lr = context->x[30];
info->pc = context->pc;
info->pstate = (context->psr & cpu::El0Aarch64PsrMask);
info->afsr0 = afsr0;
info->afsr1 = afsr1;
info->esr = esr;
info->far = far;
} else {
/* 32-bit. */
ams::svc::aarch32::ExceptionInfo *info = std::addressof(static_cast<ams::svc::aarch32::ProcessLocalRegion *>(cur_process.GetProcessLocalRegionHeapAddress())->exception_info);
for (size_t i = 0; i < util::size(info->r); ++i) {
info->r[i] = context->x[i];
}
info->sp = context->x[13];
info->lr = context->x[14];
info->pc = context->pc;
info->flags = 1;
info->status_64.pstate = (context->psr & cpu::El0Aarch32PsrMask);
info->status_64.afsr0 = afsr0;
info->status_64.afsr1 = afsr1;
info->status_64.esr = esr;
info->status_64.far = far;
}
/* Save the debug parameters to the current thread. */
GetCurrentThread().SaveDebugParams(raw_far, raw_esr, data);
/* Get the exception type. */
u32 type;
switch (ec) {
case EsrEc_Unknown:
case EsrEc_IllegalExecution:
case EsrEc_Cp15McrMrc:
case EsrEc_Cp15McrrMrrc:
case EsrEc_Cp14McrMrc:
case EsrEc_Cp14Mrrc:
case EsrEc_SystemInstruction64:
case EsrEc_BkptInstruction:
case EsrEc_BrkInstruction:
type = ams::svc::ExceptionType_InstructionAbort;
break;
case EsrEc_PcAlignmentFault:
type = ams::svc::ExceptionType_UnalignedInstruction;
break;
case EsrEc_SpAlignmentFault:
type = ams::svc::ExceptionType_UnalignedData;
break;
case EsrEc_Svc32:
case EsrEc_Svc64:
type = ams::svc::ExceptionType_InvalidSystemCall;
break;
case EsrEc_SErrorInterrupt:
type = ams::svc::ExceptionType_MemorySystemError;
break;
case EsrEc_InstructionAbortEl0:
type = ams::svc::ExceptionType_InstructionAbort;
break;
case EsrEc_DataAbortEl0:
/* If esr.IFSC is "Alignment Fault", return UnalignedData instead of DataAbort. */
if ((esr & 0x3F) == 0b100001) {
type = ams::svc::ExceptionType_UnalignedData;
} else {
type = ams::svc::ExceptionType_DataAbort;
}
break;
default:
type = ams::svc::ExceptionType_DataAbort;
break;
}
/* We want to enter at the process entrypoint, with x0 = type. */
context->pc = GetInteger(cur_process.GetEntryPoint());
context->x[0] = type;
if (is_aarch64) {
context->x[1] = GetInteger(cur_process.GetProcessLocalRegionAddress() + AMS_OFFSETOF(ams::svc::aarch64::ProcessLocalRegion, exception_info));
const auto *plr = GetPointer<ams::svc::aarch64::ProcessLocalRegion>(cur_process.GetProcessLocalRegionAddress());
context->sp = util::AlignDown(reinterpret_cast<uintptr_t>(plr->data) + sizeof(plr->data), 0x10);
context->psr = 0;
} else {
context->x[1] = GetInteger(cur_process.GetProcessLocalRegionAddress() + AMS_OFFSETOF(ams::svc::aarch32::ProcessLocalRegion, exception_info));
const auto *plr = GetPointer<ams::svc::aarch32::ProcessLocalRegion>(cur_process.GetProcessLocalRegionAddress());
context->x[13] = util::AlignDown(reinterpret_cast<uintptr_t>(plr->data) + sizeof(plr->data), 0x08);
context->psr = 0x10;
}
/* Process that we're entering a usermode exception on the current thread. */
GetCurrentThread().OnEnterUsermodeException();
return;
}
}
/* If we should, clear the thread's state as single-step. */
#if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP)
if (AMS_UNLIKELY(GetCurrentThread().IsHardwareSingleStep())) {
GetCurrentThread().ClearHardwareSingleStep();
cpu::MonitorDebugSystemControlRegisterAccessor().SetSoftwareStep(false).Store();
cpu::InstructionMemoryBarrier();
}
#endif
{
/* Collect additional information based on the ec. */
uintptr_t params[3] = {};
switch (ec) {
case EsrEc_Unknown:
case EsrEc_IllegalExecution:
case EsrEc_BkptInstruction:
case EsrEc_BrkInstruction:
{
params[0] = ams::svc::DebugException_UndefinedInstruction;
params[1] = far;
params[2] = data;
}
break;
case EsrEc_PcAlignmentFault:
case EsrEc_SpAlignmentFault:
{
params[0] = ams::svc::DebugException_AlignmentFault;
params[1] = far;
}
break;
case EsrEc_Svc32:
case EsrEc_Svc64:
{
params[0] = ams::svc::DebugException_UndefinedSystemCall;
params[1] = far;
params[2] = (esr & 0xFF);
}
break;
case EsrEc_BreakPointEl0:
case EsrEc_SoftwareStepEl0:
{
params[0] = ams::svc::DebugException_BreakPoint;
params[1] = far;
params[2] = ams::svc::BreakPointType_HardwareInstruction;
}
break;
case EsrEc_WatchPointEl0:
{
params[0] = ams::svc::DebugException_BreakPoint;
params[1] = far;
params[2] = ams::svc::BreakPointType_HardwareData;
}
break;
case EsrEc_SErrorInterrupt:
{
params[0] = ams::svc::DebugException_MemorySystemError;
params[1] = far;
}
break;
case EsrEc_InstructionAbortEl0:
{
params[0] = ams::svc::DebugException_InstructionAbort;
params[1] = far;
}
break;
case EsrEc_DataAbortEl0:
default:
{
params[0] = ams::svc::DebugException_DataAbort;
params[1] = far;
}
break;
}
/* Process the debug event. */
Result result = KDebug::OnDebugEvent(ams::svc::DebugEvent_Exception, params, util::size(params));
/* If we should stop processing the exception, do so. */
if (svc::ResultStopProcessingException::Includes(result)) {
return;
}
#if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP)
{
if (ec != EsrEc_SoftwareStepEl0) {
/* If the exception wasn't single-step, print details. */
MESOSPHERE_EXCEPTION_LOG("Exception occurred. ");
{
/* Print the current thread's registers. */
KDebug::PrintRegister();
/* Print a backtrace. */
KDebug::PrintBacktrace();
}
} else {
/* If the exception was single-step and we have no debug object, we should just return. */
if (AMS_UNLIKELY(!cur_process.IsAttachedToDebugger())) {
return;
}
}
}
#else
{
/* Print that an exception occurred. */
MESOSPHERE_EXCEPTION_LOG("Exception occurred. ");
{
/* Print the current thread's registers. */
KDebug::PrintRegister();
/* Print a backtrace. */
KDebug::PrintBacktrace();
}
}
#endif
/* If the SVC is handled, handle it. */
if (!svc::ResultNotHandled::Includes(result)) {
/* If we successfully enter jit debug, stop processing the exception. */
if (cur_process.EnterJitDebug(ams::svc::DebugEvent_Exception, static_cast<ams::svc::DebugException>(params[0]), params[1], params[2])) {
return;
}
}
}
/* Exit the current process. */
cur_process.Exit();
}
}
/* NOTE: This function is called from ASM. */
void FpuContextSwitchHandler() {
KThreadContext::FpuContextSwitchHandler(GetCurrentThreadPointer());
}
/* NOTE: This function is called from ASM. */
void ReturnFromException(Result user_result) {
/* Get the current thread. */
KThread *cur_thread = GetCurrentThreadPointer();
/* Get the current exception context. */
KExceptionContext *e_ctx = GetExceptionContext(cur_thread);
/* Get the current process. */
KProcess &cur_process = GetCurrentProcess();
/* Read the exception info that userland put in tls. */
union {
ams::svc::aarch64::ExceptionInfo info64;
ams::svc::aarch32::ExceptionInfo info32;
} info = {};
const bool is_aarch64 = (e_ctx->psr & 0x10) == 0;
if (is_aarch64) {
/* We're 64-bit. */
info.info64 = static_cast<const ams::svc::aarch64::ProcessLocalRegion *>(cur_process.GetProcessLocalRegionHeapAddress())->exception_info;
} else {
/* We're 32-bit. */
info.info32 = static_cast<const ams::svc::aarch32::ProcessLocalRegion *>(cur_process.GetProcessLocalRegionHeapAddress())->exception_info;
}
/* Try to leave the user exception. */
if (cur_process.LeaveUserException()) {
/* Process that we're leaving a usermode exception on the current thread. */
GetCurrentThread().OnLeaveUsermodeException();
/* Copy the user context to the thread context. */
if (is_aarch64) {
for (size_t i = 0; i < util::size(info.info64.r); ++i) {
e_ctx->x[i] = info.info64.r[i];
}
e_ctx->x[30] = info.info64.lr;
e_ctx->sp = info.info64.sp;
e_ctx->pc = info.info64.pc;
e_ctx->psr = (info.info64.pstate & cpu::El0Aarch64PsrMask) | (e_ctx->psr & ~cpu::El0Aarch64PsrMask);
} else {
for (size_t i = 0; i < util::size(info.info32.r); ++i) {
e_ctx->x[i] = info.info32.r[i];
}
e_ctx->x[14] = info.info32.lr;
e_ctx->x[13] = info.info32.sp;
e_ctx->pc = info.info32.pc;
e_ctx->psr = (info.info32.status_64.pstate & cpu::El0Aarch32PsrMask) | (e_ctx->psr & ~cpu::El0Aarch32PsrMask);
}
/* Note that PC was adjusted. */
e_ctx->write = 1;
if (R_SUCCEEDED(user_result)) {
/* If result handling succeeded, just restore the context. */
svc::RestoreContext(reinterpret_cast<uintptr_t>(e_ctx));
} else {
/* Restore the debug params for the exception. */
uintptr_t far, esr, data;
GetCurrentThread().RestoreDebugParams(std::addressof(far), std::addressof(esr), std::addressof(data));
/* Pre-process exception registers as needed. */
const u64 ec = (esr >> 26) & 0x3F;
if (ec == EsrEc_InstructionAbortEl0 || ec == EsrEc_DataAbortEl0) {
/* Adjust registers if a synchronous external abort has occurred with far not valid. */
/* Mask 0x03F = Low 6 bits IFSC == 0x10: "Synchronous External abort, */
/* not on translation table walk or hardware update of translation table. */
/* Mask 0x400 = FnV = "FAR Not Valid" */
/* TODO: How would we perform this check using named register accesses? */
if ((esr & 0x43F) == 0x410) {
/* Clear the faulting register on memory tagging exception. */
far = 0;
} else {
/* If the faulting address is a kernel address, set ISFC = 4. */
if (far >= ams::svc::AddressMemoryRegion39Size) {
esr = (esr & 0xFFFFFFC0) | 4;
}
}
}
/* Collect additional information based on the ec. */
uintptr_t params[3] = {};
switch (ec) {
case EsrEc_Unknown:
case EsrEc_IllegalExecution:
case EsrEc_BkptInstruction:
case EsrEc_BrkInstruction:
{
params[0] = ams::svc::DebugException_UndefinedInstruction;
params[1] = far;
params[2] = data;
}
break;
case EsrEc_PcAlignmentFault:
case EsrEc_SpAlignmentFault:
{
params[0] = ams::svc::DebugException_AlignmentFault;
params[1] = far;
}
break;
case EsrEc_Svc32:
case EsrEc_Svc64:
{
params[0] = ams::svc::DebugException_UndefinedSystemCall;
params[1] = far;
params[2] = (esr & 0xFF);
}
break;
case EsrEc_SErrorInterrupt:
{
params[0] = ams::svc::DebugException_MemorySystemError;
params[1] = far;
}
break;
case EsrEc_InstructionAbortEl0:
{
params[0] = ams::svc::DebugException_InstructionAbort;
params[1] = far;
}
break;
case EsrEc_DataAbortEl0:
default:
{
params[0] = ams::svc::DebugException_DataAbort;
params[1] = far;
}
break;
}
/* Process the debug event. */
Result result = KDebug::OnDebugEvent(ams::svc::DebugEvent_Exception, params, util::size(params));
/* If the SVC is handled, handle it. */
if (!svc::ResultNotHandled::Includes(result)) {
/* If we should stop processing the exception, restore. */
if (svc::ResultStopProcessingException::Includes(result)) {
svc::RestoreContext(reinterpret_cast<uintptr_t>(e_ctx));
}
/* If we successfully enter jit debug, restore. */
if (cur_process.EnterJitDebug(ams::svc::DebugEvent_Exception, static_cast<ams::svc::DebugException>(params[0]), params[1], params[2])) {
svc::RestoreContext(reinterpret_cast<uintptr_t>(e_ctx));
}
}
/* Otherwise, if result debug was returned, restore. */
if (svc::ResultDebug::Includes(result)) {
svc::RestoreContext(reinterpret_cast<uintptr_t>(e_ctx));
}
}
}
/* Print that an exception occurred. */
MESOSPHERE_EXCEPTION_LOG("Exception occurred. ");
/* Exit the current process. */
GetCurrentProcess().Exit();
}
/* NOTE: This function is called from ASM. */
void HandleException(KExceptionContext *context) {
MESOSPHERE_ASSERT(!KInterruptManager::AreInterruptsEnabled());
/* Retrieve information about the exception. */
const bool is_user_mode = (context->psr & 0xF) == 0;
const u64 esr = cpu::GetEsrEl1();
const u64 afsr0 = cpu::GetAfsr0El1();
const u64 afsr1 = cpu::GetAfsr1El1();
u64 far = 0;
u32 data = 0;
/* Collect far and data based on the ec. */
switch ((esr >> 26) & 0x3F) {
case EsrEc_Unknown:
case EsrEc_IllegalExecution:
case EsrEc_BkptInstruction:
case EsrEc_BrkInstruction:
far = context->pc;
/* NOTE: Nintendo always calls GetInstructionDataUserMode. */
if (is_user_mode) {
data = GetInstructionDataUserMode(context);
} else {
data = GetInstructionDataSupervisorMode(context, esr);
}
break;
case EsrEc_Svc32:
if (context->psr & 0x20) {
/* Thumb mode. */
context->pc -= 2;
} else {
/* ARM mode. */
context->pc -= 4;
}
far = context->pc;
break;
case EsrEc_Svc64:
context->pc -= 4;
far = context->pc;
break;
case EsrEc_BreakPointEl0:
far = context->pc;
break;
default:
far = cpu::GetFarEl1();
break;
}
/* Note that we're in an exception handler. */
GetCurrentThread().SetInExceptionHandler();
/* Verify that spsr's M is allowable (EL0t). */
{
if (is_user_mode) {
/* If the user disable count is set, we may need to pin the current thread. */
if (GetCurrentThread().GetUserDisableCount() != 0 && GetCurrentProcess().GetPinnedThread(GetCurrentCoreId()) == nullptr) {
KScopedSchedulerLock lk;
/* Pin the current thread. */
GetCurrentProcess().PinCurrentThread();
/* Set the interrupt flag for the thread. */
GetCurrentThread().SetInterruptFlag();
}
/* Enable interrupts while we process the usermode exception. */
{
KScopedInterruptEnable ei;
/* Terminate the thread, if we should. */
if (GetCurrentThread().IsTerminationRequested()) {
GetCurrentThread().Exit();
}
HandleUserException(context, esr, far, afsr0, afsr1, data);
}
} else {
const s32 core_id = GetCurrentCoreId();
MESOSPHERE_LOG("%d: Unhandled Exception in Supervisor Mode\n", core_id);
if (GetCurrentProcessPointer() != nullptr) {
MESOSPHERE_LOG("%d: Current Process = %s\n", core_id, GetCurrentProcess().GetName());
}
for (size_t i = 0; i < 31; i++) {
MESOSPHERE_LOG("%d: X[%02zu] = %016lx\n", core_id, i, context->x[i]);
}
MESOSPHERE_LOG("%d: PC = %016lx\n", core_id, context->pc);
MESOSPHERE_LOG("%d: SP = %016lx\n", core_id, context->sp);
MESOSPHERE_PANIC("Unhandled Exception in Supervisor Mode\n");
}
MESOSPHERE_ASSERT(!KInterruptManager::AreInterruptsEnabled());
/* Handle any DPC requests. */
while (GetCurrentThread().HasDpc()) {
KDpcManager::HandleDpc();
}
}
/* Note that we're no longer in an exception handler. */
GetCurrentThread().ClearInExceptionHandler();
}
}

949
Source/Atmosphere/libraries/libmesosphere/source/arch/arm64/kern_k_debug.cpp Normal file
View File

@@ -0,0 +1,949 @@
/*
* Copyright (c) Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <mesosphere.hpp>
/* <stratosphere/rocrt/rocrt.hpp> */
namespace ams::rocrt {
constexpr inline const u32 ModuleHeaderVersion = util::FourCC<'M','O','D','0'>::Code;
struct ModuleHeader {
u32 signature;
u32 dynamic_offset;
u32 bss_start_offset;
u32 bss_end_offset;
u32 exception_info_start_offset;
u32 exception_info_end_offset;
u32 module_offset;
};
struct ModuleHeaderLocation {
u32 pad;
u32 header_offset;
};
}
namespace ams::kern::arch::arm64 {
namespace {
constexpr inline u64 ForbiddenBreakPointFlagsMask = (((1ul << 40) - 1) << 24) | /* Reserved upper bits. */
(((1ul << 1) - 1) << 23) | /* Match VMID BreakPoint Type. */
(((1ul << 2) - 1) << 14) | /* Security State Control. */
(((1ul << 1) - 1) << 13) | /* Hyp Mode Control. */
(((1ul << 4) - 1) << 9) | /* Reserved middle bits. */
(((1ul << 2) - 1) << 3) | /* Reserved lower bits. */
(((1ul << 2) - 1) << 1); /* Privileged Mode Control. */
static_assert(ForbiddenBreakPointFlagsMask == 0xFFFFFFFFFF80FE1Eul);
constexpr inline u64 ForbiddenWatchPointFlagsMask = (((1ul << 32) - 1) << 32) | /* Reserved upper bits. */
(((1ul << 4) - 1) << 20) | /* WatchPoint Type. */
(((1ul << 2) - 1) << 14) | /* Security State Control. */
(((1ul << 1) - 1) << 13) | /* Hyp Mode Control. */
(((1ul << 2) - 1) << 1); /* Privileged Access Control. */
static_assert(ForbiddenWatchPointFlagsMask == 0xFFFFFFFF00F0E006ul);
}
uintptr_t KDebug::GetProgramCounter(const KThread &thread) {
return GetExceptionContext(std::addressof(thread))->pc;
}
void KDebug::SetPreviousProgramCounter() {
/* Get the current thread. */
KThread *thread = GetCurrentThreadPointer();
MESOSPHERE_ASSERT(thread->IsCallingSvc());
/* Get the exception context. */
KExceptionContext *e_ctx = GetExceptionContext(thread);
/* Set the previous pc. */
if (e_ctx->write == 0) {
/* Subtract from the program counter. */
if (thread->GetOwnerProcess()->Is64Bit()) {
e_ctx->pc -= sizeof(u32);
} else {
e_ctx->pc -= (e_ctx->psr & 0x20) ? sizeof(u16) : sizeof(u32);
}
/* Mark that we've set. */
e_ctx->write = 1;
}
}
Result KDebug::GetThreadContextImpl(ams::svc::ThreadContext *out, KThread *thread, u32 context_flags) {
MESOSPHERE_ASSERT(KScheduler::IsSchedulerLockedByCurrentThread());
MESOSPHERE_ASSERT(thread != GetCurrentThreadPointer());
/* Get the exception context. */
const KExceptionContext *e_ctx = GetExceptionContext(thread);
/* Get whether we're 64-bit. */
const bool is_64_bit = this->Is64Bit();
/* If general registers are requested, get them. */
if ((context_flags & ams::svc::ThreadContextFlag_General) != 0) {
/* We can always get X0-X7/R0-R7. */
auto register_count = 8;
if (!thread->IsCallingSvc() || thread->GetSvcId() == svc::SvcId_ReturnFromException) {
if (is_64_bit) {
/* We're not in an SVC, so we can get X0-X29. */
register_count = 29;
} else {
/* We're 32-bit, so we should get R0-R12. */
register_count = 13;
}
}
/* Get the registers. */
for (auto i = 0; i < register_count; ++i) {
out->r[i] = is_64_bit ? e_ctx->x[i] : static_cast<u32>(e_ctx->x[i]);
}
}
/* If control flags are requested, get them. */
if ((context_flags & ams::svc::ThreadContextFlag_Control) != 0) {
if (is_64_bit) {
out->fp = e_ctx->x[29];
out->lr = e_ctx->x[30];
out->sp = e_ctx->sp;
out->pc = e_ctx->pc;
out->pstate = (e_ctx->psr & cpu::El0Aarch64PsrMask);
/* Adjust PC if we should. */
if (e_ctx->write == 0 && thread->IsCallingSvc()) {
out->pc -= sizeof(u32);
}
out->tpidr = e_ctx->tpidr;
} else {
out->r[11] = static_cast<u32>(e_ctx->x[11]);
out->r[13] = static_cast<u32>(e_ctx->x[13]);
out->r[14] = static_cast<u32>(e_ctx->x[14]);
out->lr = 0;
out->sp = 0;
out->pc = e_ctx->pc;
out->pstate = (e_ctx->psr & cpu::El0Aarch32PsrMask);
/* Adjust PC if we should. */
if (e_ctx->write == 0 && thread->IsCallingSvc()) {
out->pc -= (e_ctx->psr & 0x20) ? sizeof(u16) : sizeof(u32);
}
out->tpidr = static_cast<u32>(e_ctx->tpidr);
}
}
/* Get the FPU context. */
R_RETURN(this->GetFpuContext(out, thread, context_flags));
}
Result KDebug::SetThreadContextImpl(const ams::svc::ThreadContext &ctx, KThread *thread, u32 context_flags) {
MESOSPHERE_ASSERT(KScheduler::IsSchedulerLockedByCurrentThread());
MESOSPHERE_ASSERT(thread != GetCurrentThreadPointer());
/* Get the exception context. */
KExceptionContext *e_ctx = GetExceptionContext(thread);
/* If general registers are requested, set them. */
if ((context_flags & ams::svc::ThreadContextFlag_General) != 0) {
if (this->Is64Bit()) {
/* Set X0-X28. */
for (auto i = 0; i <= 28; ++i) {
e_ctx->x[i] = ctx.r[i];
}
} else {
/* Set R0-R12. */
for (auto i = 0; i <= 12; ++i) {
e_ctx->x[i] = static_cast<u32>(ctx.r[i]);
}
}
}
/* If control flags are requested, set them. */
if ((context_flags & ams::svc::ThreadContextFlag_Control) != 0) {
/* Mark ourselve as having adjusted pc. */
e_ctx->write = 1;
if (this->Is64Bit()) {
e_ctx->x[29] = ctx.fp;
e_ctx->x[30] = ctx.lr;
e_ctx->sp = ctx.sp;
e_ctx->pc = ctx.pc;
e_ctx->psr = ((ctx.pstate & cpu::El0Aarch64PsrMask) | (e_ctx->psr & ~cpu::El0Aarch64PsrMask));
e_ctx->tpidr = ctx.tpidr;
} else {
e_ctx->x[13] = static_cast<u32>(ctx.r[13]);
e_ctx->x[14] = static_cast<u32>(ctx.r[14]);
e_ctx->x[30] = 0;
e_ctx->sp = 0;
e_ctx->pc = static_cast<u32>(ctx.pc);
e_ctx->psr = ((ctx.pstate & cpu::El0Aarch32PsrMask) | (e_ctx->psr & ~cpu::El0Aarch32PsrMask));
e_ctx->tpidr = ctx.tpidr;
}
}
/* Set the FPU context. */
R_RETURN(this->SetFpuContext(ctx, thread, context_flags));
}
Result KDebug::GetFpuContext(ams::svc::ThreadContext *out, KThread *thread, u32 context_flags) {
MESOSPHERE_ASSERT(KScheduler::IsSchedulerLockedByCurrentThread());
MESOSPHERE_ASSERT(thread != GetCurrentThreadPointer());
/* Succeed if there's nothing to do. */
R_SUCCEED_IF((context_flags & (ams::svc::ThreadContextFlag_Fpu | ams::svc::ThreadContextFlag_FpuControl)) == 0);
/* Get the thread context. */
KThreadContext *t_ctx = std::addressof(thread->GetContext());
/* Get the FPU control registers, if required. */
if ((context_flags & ams::svc::ThreadContextFlag_FpuControl) != 0) {
out->fpsr = t_ctx->GetFpsr();
out->fpcr = t_ctx->GetFpcr();
}
/* Get the FPU registers, if required. */
if ((context_flags & ams::svc::ThreadContextFlag_Fpu) != 0) {
static_assert(util::size(ams::svc::ThreadContext{}.v) == KThreadContext::NumFpuRegisters);
const auto &caller_save = thread->GetCallerSaveFpuRegisters();
const auto &callee_save = t_ctx->GetCalleeSaveFpuRegisters();
if (this->Is64Bit()) {
KThreadContext::GetFpuRegisters(out->v, caller_save.fpu64, callee_save.fpu64);
} else {
KThreadContext::GetFpuRegisters(out->v, caller_save.fpu32, callee_save.fpu32);
for (size_t i = KThreadContext::NumFpuRegisters / 2; i < KThreadContext::NumFpuRegisters; ++i) {
out->v[i] = 0;
}
}
}
R_SUCCEED();
}
Result KDebug::SetFpuContext(const ams::svc::ThreadContext &ctx, KThread *thread, u32 context_flags) {
MESOSPHERE_ASSERT(KScheduler::IsSchedulerLockedByCurrentThread());
MESOSPHERE_ASSERT(thread != GetCurrentThreadPointer());
/* Succeed if there's nothing to do. */
R_SUCCEED_IF((context_flags & (ams::svc::ThreadContextFlag_Fpu | ams::svc::ThreadContextFlag_FpuControl)) == 0);
/* Get the thread context. */
KThreadContext *t_ctx = std::addressof(thread->GetContext());
/* Set the FPU control registers, if required. */
if ((context_flags & ams::svc::ThreadContextFlag_FpuControl) != 0) {
t_ctx->SetFpsr(ctx.fpsr);
t_ctx->SetFpcr(ctx.fpcr);
}
/* Set the FPU registers, if required. */
if ((context_flags & ams::svc::ThreadContextFlag_Fpu) != 0) {
static_assert(util::size(ams::svc::ThreadContext{}.v) == KThreadContext::NumFpuRegisters);
auto &caller_save = thread->GetCallerSaveFpuRegisters();
auto &callee_save = t_ctx->GetCalleeSaveFpuRegisters();
if (this->Is64Bit()) {
KThreadContext::SetFpuRegisters(caller_save.fpu64, callee_save.fpu64, ctx.v);
} else {
KThreadContext::SetFpuRegisters(caller_save.fpu32, callee_save.fpu32, ctx.v);
}
}
R_SUCCEED();
}
Result KDebug::BreakIfAttached(ams::svc::BreakReason break_reason, uintptr_t address, size_t size) {
const uintptr_t params[5] = { ams::svc::DebugException_UserBreak, GetProgramCounter(GetCurrentThread()), break_reason, address, size };
R_RETURN(KDebugBase::OnDebugEvent(ams::svc::DebugEvent_Exception, params, util::size(params)));
}
#define MESOSPHERE_SET_HW_BREAK_POINT(ID, FLAGS, VALUE) \
({ \
cpu::SetDbgBcr##ID##El1(0); \
cpu::EnsureInstructionConsistencyFullSystem(); \
cpu::SetDbgBvr##ID##El1(VALUE); \
cpu::EnsureInstructionConsistencyFullSystem(); \
cpu::SetDbgBcr##ID##El1(FLAGS); \
cpu::EnsureInstructionConsistencyFullSystem(); \
})
#define MESOSPHERE_SET_HW_WATCH_POINT(ID, FLAGS, VALUE) \
({ \
cpu::SetDbgWcr##ID##El1(0); \
cpu::EnsureInstructionConsistencyFullSystem(); \
cpu::SetDbgWvr##ID##El1(VALUE); \
cpu::EnsureInstructionConsistencyFullSystem(); \
cpu::SetDbgWcr##ID##El1(FLAGS); \
cpu::EnsureInstructionConsistencyFullSystem(); \
})
Result KDebug::SetHardwareBreakPoint(ams::svc::HardwareBreakPointRegisterName name, u64 flags, u64 value) {
/* Get the debug feature register. */
cpu::DebugFeatureRegisterAccessor dfr0;
/* Extract interesting info from the debug feature register. */
const auto num_bp = dfr0.GetNumBreakpoints();
const auto num_wp = dfr0.GetNumWatchpoints();
const auto num_ctx = dfr0.GetNumContextAwareBreakpoints();
if (ams::svc::HardwareBreakPointRegisterName_I0 <= name && name <= ams::svc::HardwareBreakPointRegisterName_I15) {
/* Check that the name is a valid instruction breakpoint. */
R_UNLESS((name - ams::svc::HardwareBreakPointRegisterName_I0) <= num_bp, svc::ResultNotSupported());
/* Configure flags/value. */
if ((flags & 1) != 0) {
/* We're enabling the breakpoint. Check that the flags are allowable. */
R_UNLESS((flags & ForbiddenBreakPointFlagsMask) == 0, svc::ResultInvalidCombination());
/* Require that the breakpoint be linked or match context id. */
R_UNLESS((flags & ((1ul << 21) | (1ul << 20))) != 0, svc::ResultInvalidCombination());
/* If the breakpoint matches context id, we need to get the context id. */
if ((flags & (1ul << 21)) != 0) {
/* Ensure that the breakpoint is context-aware. */
R_UNLESS((name - ams::svc::HardwareBreakPointRegisterName_I0) >= (num_bp - num_ctx), svc::ResultNotSupported());
/* Check that the breakpoint does not have the mismatch bit. */
R_UNLESS((flags & (1ul << 22)) == 0, svc::ResultInvalidCombination());
/* Get the debug object from the current handle table. */
KScopedAutoObject debug = GetCurrentProcess().GetHandleTable().GetObject<KDebug>(static_cast<ams::svc::Handle>(value));
R_UNLESS(debug.IsNotNull(), svc::ResultInvalidHandle());
/* Get the process from the debug object. */
R_UNLESS(debug->IsAttached(), svc::ResultProcessTerminated());
R_UNLESS(debug->OpenProcess(), svc::ResultProcessTerminated());
/* Close the process when we're done. */
ON_SCOPE_EXIT { debug->CloseProcess(); };
/* Get the proces. */
KProcess * const process = debug->GetProcessUnsafe();
/* Set the value to be the context id. */
value = process->GetId() & 0xFFFFFFFF;
}
/* Set the breakpoint as non-secure EL0-only. */
flags |= (1ul << 14) | (2ul << 1);
} else {
/* We're disabling the breakpoint. */
flags = 0;
value = 0;
}
/* Set the breakpoint. */
switch (name) {
case ams::svc::HardwareBreakPointRegisterName_I0: MESOSPHERE_SET_HW_BREAK_POINT( 0, flags, value); break;
case ams::svc::HardwareBreakPointRegisterName_I1: MESOSPHERE_SET_HW_BREAK_POINT( 1, flags, value); break;
case ams::svc::HardwareBreakPointRegisterName_I2: MESOSPHERE_SET_HW_BREAK_POINT( 2, flags, value); break;
case ams::svc::HardwareBreakPointRegisterName_I3: MESOSPHERE_SET_HW_BREAK_POINT( 3, flags, value); break;
case ams::svc::HardwareBreakPointRegisterName_I4: MESOSPHERE_SET_HW_BREAK_POINT( 4, flags, value); break;
case ams::svc::HardwareBreakPointRegisterName_I5: MESOSPHERE_SET_HW_BREAK_POINT( 5, flags, value); break;
case ams::svc::HardwareBreakPointRegisterName_I6: MESOSPHERE_SET_HW_BREAK_POINT( 6, flags, value); break;
case ams::svc::HardwareBreakPointRegisterName_I7: MESOSPHERE_SET_HW_BREAK_POINT( 7, flags, value); break;
case ams::svc::HardwareBreakPointRegisterName_I8: MESOSPHERE_SET_HW_BREAK_POINT( 8, flags, value); break;
case ams::svc::HardwareBreakPointRegisterName_I9: MESOSPHERE_SET_HW_BREAK_POINT( 9, flags, value); break;
case ams::svc::HardwareBreakPointRegisterName_I10: MESOSPHERE_SET_HW_BREAK_POINT(10, flags, value); break;
case ams::svc::HardwareBreakPointRegisterName_I11: MESOSPHERE_SET_HW_BREAK_POINT(11, flags, value); break;
case ams::svc::HardwareBreakPointRegisterName_I12: MESOSPHERE_SET_HW_BREAK_POINT(12, flags, value); break;
case ams::svc::HardwareBreakPointRegisterName_I13: MESOSPHERE_SET_HW_BREAK_POINT(13, flags, value); break;
case ams::svc::HardwareBreakPointRegisterName_I14: MESOSPHERE_SET_HW_BREAK_POINT(14, flags, value); break;
case ams::svc::HardwareBreakPointRegisterName_I15: MESOSPHERE_SET_HW_BREAK_POINT(15, flags, value); break;
default: break;
}
} else if (ams::svc::HardwareBreakPointRegisterName_D0 <= name && name <= ams::svc::HardwareBreakPointRegisterName_D15) {
/* Check that the name is a valid data breakpoint. */
R_UNLESS((name - ams::svc::HardwareBreakPointRegisterName_D0) <= num_wp, svc::ResultNotSupported());
/* Configure flags/value. */
if ((flags & 1) != 0) {
/* We're enabling the watchpoint. Check that the flags are allowable. */
R_UNLESS((flags & ForbiddenWatchPointFlagsMask) == 0, svc::ResultInvalidCombination());
/* Set the breakpoint as linked non-secure EL0-only. */
flags |= (1ul << 20) | (1ul << 14) | (2ul << 1);
} else {
/* We're disabling the watchpoint. */
flags = 0;
value = 0;
}
/* Set the watchpoint. */
switch (name) {
case ams::svc::HardwareBreakPointRegisterName_D0: MESOSPHERE_SET_HW_WATCH_POINT( 0, flags, value); break;
case ams::svc::HardwareBreakPointRegisterName_D1: MESOSPHERE_SET_HW_WATCH_POINT( 1, flags, value); break;
case ams::svc::HardwareBreakPointRegisterName_D2: MESOSPHERE_SET_HW_WATCH_POINT( 2, flags, value); break;
case ams::svc::HardwareBreakPointRegisterName_D3: MESOSPHERE_SET_HW_WATCH_POINT( 3, flags, value); break;
case ams::svc::HardwareBreakPointRegisterName_D4: MESOSPHERE_SET_HW_WATCH_POINT( 4, flags, value); break;
case ams::svc::HardwareBreakPointRegisterName_D5: MESOSPHERE_SET_HW_WATCH_POINT( 5, flags, value); break;
case ams::svc::HardwareBreakPointRegisterName_D6: MESOSPHERE_SET_HW_WATCH_POINT( 6, flags, value); break;
case ams::svc::HardwareBreakPointRegisterName_D7: MESOSPHERE_SET_HW_WATCH_POINT( 7, flags, value); break;
case ams::svc::HardwareBreakPointRegisterName_D8: MESOSPHERE_SET_HW_WATCH_POINT( 8, flags, value); break;
case ams::svc::HardwareBreakPointRegisterName_D9: MESOSPHERE_SET_HW_WATCH_POINT( 9, flags, value); break;
case ams::svc::HardwareBreakPointRegisterName_D10: MESOSPHERE_SET_HW_WATCH_POINT(10, flags, value); break;
case ams::svc::HardwareBreakPointRegisterName_D11: MESOSPHERE_SET_HW_WATCH_POINT(11, flags, value); break;
case ams::svc::HardwareBreakPointRegisterName_D12: MESOSPHERE_SET_HW_WATCH_POINT(12, flags, value); break;
case ams::svc::HardwareBreakPointRegisterName_D13: MESOSPHERE_SET_HW_WATCH_POINT(13, flags, value); break;
case ams::svc::HardwareBreakPointRegisterName_D14: MESOSPHERE_SET_HW_WATCH_POINT(14, flags, value); break;
case ams::svc::HardwareBreakPointRegisterName_D15: MESOSPHERE_SET_HW_WATCH_POINT(15, flags, value); break;
default: break;
}
} else {
/* Invalid name. */
R_THROW(svc::ResultInvalidEnumValue());
}
R_SUCCEED();
}
#undef MESOSPHERE_SET_HW_WATCH_POINT
#undef MESOSPHERE_SET_HW_BREAK_POINT
void KDebug::PrintRegister(KThread *thread) {
#if defined(MESOSPHERE_BUILD_FOR_DEBUGGING)
{
/* Treat no thread as current thread. */
if (thread == nullptr) {
thread = GetCurrentThreadPointer();
}
/* Get the exception context. */
KExceptionContext *e_ctx = GetExceptionContext(thread);
/* Get the owner process. */
if (auto *process = thread->GetOwnerProcess(); process != nullptr) {
/* Lock the owner process. */
KScopedLightLock state_lk(process->GetStateLock());
KScopedLightLock list_lk(process->GetListLock());
/* Suspend all the process's threads. */
{
KScopedSchedulerLock sl;
auto end = process->GetThreadList().end();
for (auto it = process->GetThreadList().begin(); it != end; ++it) {
if (std::addressof(*it) != GetCurrentThreadPointer()) {
it->RequestSuspend(KThread::SuspendType_Backtrace);
}
}
}
/* Print the registers. */
MESOSPHERE_RELEASE_LOG("Registers\n");
if ((e_ctx->psr & 0x10) == 0) {
/* 64-bit thread. */
for (auto i = 0; i < 31; ++i) {
MESOSPHERE_RELEASE_LOG(" X[%2d]: 0x%016lx\n", i, e_ctx->x[i]);
}
MESOSPHERE_RELEASE_LOG(" SP: 0x%016lx\n", e_ctx->sp);
MESOSPHERE_RELEASE_LOG(" PC: 0x%016lx\n", e_ctx->pc - sizeof(u32));
MESOSPHERE_RELEASE_LOG(" PSR: 0x%08x\n", e_ctx->psr);
MESOSPHERE_RELEASE_LOG(" TPIDR_EL0: 0x%016lx\n", e_ctx->tpidr);
} else {
/* 32-bit thread. */
for (auto i = 0; i < 13; ++i) {
MESOSPHERE_RELEASE_LOG(" R[%2d]: 0x%08x\n", i, static_cast<u32>(e_ctx->x[i]));
}
MESOSPHERE_RELEASE_LOG(" SP: 0x%08x\n", static_cast<u32>(e_ctx->x[13]));
MESOSPHERE_RELEASE_LOG(" LR: 0x%08x\n", static_cast<u32>(e_ctx->x[14]));
MESOSPHERE_RELEASE_LOG(" PC: 0x%08x\n", static_cast<u32>(e_ctx->pc) - static_cast<u32>((e_ctx->psr & 0x20) ? sizeof(u16) : sizeof(u32)));
MESOSPHERE_RELEASE_LOG(" PSR: 0x%08x\n", e_ctx->psr);
MESOSPHERE_RELEASE_LOG(" TPIDR: 0x%08x\n", static_cast<u32>(e_ctx->tpidr));
}
/* Resume the threads that we suspended. */
{
KScopedSchedulerLock sl;
auto end = process->GetThreadList().end();
for (auto it = process->GetThreadList().begin(); it != end; ++it) {
if (std::addressof(*it) != GetCurrentThreadPointer()) {
it->Resume(KThread::SuspendType_Backtrace);
}
}
}
}
}
#else
MESOSPHERE_UNUSED(thread);
#endif
}
#if defined(MESOSPHERE_BUILD_FOR_DEBUGGING)
namespace {
bool IsHeapPhysicalAddress(KPhysicalAddress phys_addr) {
const KMemoryRegion *cached = nullptr;
return KMemoryLayout::IsHeapPhysicalAddress(cached, phys_addr);
}
template<typename T>
bool ReadValue(T *out, KProcess *process, uintptr_t address) {
KPhysicalAddress phys_addr;
KMemoryInfo mem_info;
ams::svc::PageInfo page_info;
if (!util::IsAligned(address, sizeof(T))) {
return false;
}
if (R_FAILED(process->GetPageTable().QueryInfo(std::addressof(mem_info), std::addressof(page_info), address))) {
return false;
}
if ((mem_info.GetPermission() & KMemoryPermission_UserRead) != KMemoryPermission_UserRead) {
return false;
}
if (!process->GetPageTable().GetPhysicalAddress(std::addressof(phys_addr), address)) {
return false;
}
if (!IsHeapPhysicalAddress(phys_addr)) {
return false;
}
*out = *GetPointer<T>(process->GetPageTable().GetHeapVirtualAddress(phys_addr));
return true;
}
bool GetModuleName(char *dst, size_t dst_size, KProcess *process, uintptr_t base_address) {
/* Locate .rodata. */
KMemoryInfo mem_info;
ams::svc::PageInfo page_info;
KMemoryState mem_state = KMemoryState_None;
while (true) {
if (R_FAILED(process->GetPageTable().QueryInfo(std::addressof(mem_info), std::addressof(page_info), base_address))) {
return false;
}
if (mem_state == KMemoryState_None) {
mem_state = mem_info.GetState();
if (mem_state != KMemoryState_Code && mem_state != KMemoryState_AliasCode) {
return false;
}
}
if (mem_info.GetState() != mem_state) {
return false;
}
if (mem_info.GetPermission() == KMemoryPermission_UserRead) {
break;
}
base_address = mem_info.GetEndAddress();
}
/* Check that first value is 0. */
u32 val;
if (!ReadValue(std::addressof(val), process, base_address)) {
return false;
}
if (val != 0) {
return false;
}
/* Read the name length. */
if (!ReadValue(std::addressof(val), process, base_address + sizeof(u32))) {
return false;
}
if (!(0 < val && val < dst_size)) {
return false;
}
const size_t name_len = val;
/* Read the name, one character at a time. */
for (size_t i = 0; i < name_len; ++i) {
if (!ReadValue(dst + i, process, base_address + 2 * sizeof(u32) + i)) {
return false;
}
if (!(0 < dst[i] && dst[i] <= 0x7F)) {
return false;
}
}
/* NULL-terminate. */
dst[name_len] = 0;
return true;
}
void PrintAddress(uintptr_t address) {
MESOSPHERE_RELEASE_LOG(" %p\n", reinterpret_cast<void *>(address));
}
void PrintAddressWithModuleName(uintptr_t address, bool has_module_name, const char *module_name, uintptr_t base_address) {
if (has_module_name) {
MESOSPHERE_RELEASE_LOG(" %p [%10s + %8lx]\n", reinterpret_cast<void *>(address), module_name, address - base_address);
} else {
MESOSPHERE_RELEASE_LOG(" %p [%10lx + %8lx]\n", reinterpret_cast<void *>(address), base_address, address - base_address);
}
}
void PrintAddressWithSymbol(uintptr_t address, bool has_module_name, const char *module_name, uintptr_t base_address, const char *symbol_name, uintptr_t func_address) {
if (has_module_name) {
MESOSPHERE_RELEASE_LOG(" %p [%10s + %8lx] (%s + %lx)\n", reinterpret_cast<void *>(address), module_name, address - base_address, symbol_name, address - func_address);
} else {
MESOSPHERE_RELEASE_LOG(" %p [%10lx + %8lx] (%s + %lx)\n", reinterpret_cast<void *>(address), base_address, address - base_address, symbol_name, address - func_address);
}
}
void PrintCodeAddress(KProcess *process, uintptr_t address, bool is_lr = true) {
/* Prepare to parse + print the address. */
uintptr_t test_address = is_lr ? address - sizeof(u32) : address;
uintptr_t base_address = address;
uintptr_t dyn_address = 0;
uintptr_t sym_tab = 0;
uintptr_t str_tab = 0;
size_t num_sym = 0;
u64 temp_64;
u32 temp_32;
/* Locate the start of .text. */
KMemoryInfo mem_info;
ams::svc::PageInfo page_info;
KMemoryState mem_state = KMemoryState_None;
while (true) {
if (R_FAILED(process->GetPageTable().QueryInfo(std::addressof(mem_info), std::addressof(page_info), base_address))) {
return PrintAddress(address);
}
if (mem_state == KMemoryState_None) {
mem_state = mem_info.GetState();
if (mem_state != KMemoryState_Code && mem_state != KMemoryState_AliasCode) {
return PrintAddress(address);
}
} else if (mem_info.GetState() != mem_state) {
return PrintAddress(address);
}
if (mem_info.GetPermission() != KMemoryPermission_UserReadExecute) {
return PrintAddress(address);
}
base_address = mem_info.GetAddress();
if (R_FAILED(process->GetPageTable().QueryInfo(std::addressof(mem_info), std::addressof(page_info), base_address - 1))) {
return PrintAddress(address);
}
if (mem_info.GetState() != mem_state) {
break;
}
if (mem_info.GetPermission() != KMemoryPermission_UserReadExecute) {
break;
}
}
/* Get the module name. */
char module_name[0x20];
const bool has_module_name = GetModuleName(module_name, sizeof(module_name), process, base_address);
/* If the process is 32-bit, just print the module. */
if (!process->Is64Bit()) {
return PrintAddressWithModuleName(address, has_module_name, module_name, base_address);
}
/* Locate .dyn using rocrt::ModuleHeader. */
{
/* Determine the ModuleHeader offset. */
u32 mod_offset;
if (!ReadValue(std::addressof(mod_offset), process, base_address + sizeof(u32))) {
return PrintAddressWithModuleName(address, has_module_name, module_name, base_address);
}
/* Read the signature. */
constexpr u32 SignatureFieldOffset = AMS_OFFSETOF(rocrt::ModuleHeader, signature);
if (!ReadValue(std::addressof(temp_32), process, base_address + mod_offset + SignatureFieldOffset)) {
return PrintAddressWithModuleName(address, has_module_name, module_name, base_address);
}
/* Check that the module signature is expected. */
if (temp_32 != rocrt::ModuleHeaderVersion) { /* MOD0 */
return PrintAddressWithModuleName(address, has_module_name, module_name, base_address);
}
/* Determine the dynamic offset. */
constexpr u32 DynamicFieldOffset = AMS_OFFSETOF(rocrt::ModuleHeader, dynamic_offset);
if (!ReadValue(std::addressof(temp_32), process, base_address + mod_offset + DynamicFieldOffset)) {
return PrintAddressWithModuleName(address, has_module_name, module_name, base_address);
}
dyn_address = base_address + mod_offset + temp_32;
}
/* Locate tables inside .dyn. */
for (size_t ofs = 0; /* ... */; ofs += 0x10) {
/* Read the DynamicTag. */
if (!ReadValue(std::addressof(temp_64), process, dyn_address + ofs)) {
return PrintAddressWithModuleName(address, has_module_name, module_name, base_address);
}
if (temp_64 == 0) {
/* We're done parsing .dyn. */
break;
} else if (temp_64 == 4) {
/* We found DT_HASH */
if (!ReadValue(std::addressof(temp_64), process, dyn_address + ofs + sizeof(u64))) {
return PrintAddressWithModuleName(address, has_module_name, module_name, base_address);
}
/* Read nchain, to get the number of symbols. */
if (!ReadValue(std::addressof(temp_32), process, base_address + temp_64 + sizeof(u32))) {
return PrintAddressWithModuleName(address, has_module_name, module_name, base_address);
}
num_sym = temp_32;
} else if (temp_64 == 5) {
/* We found DT_STRTAB */
if (!ReadValue(std::addressof(temp_64), process, dyn_address + ofs + sizeof(u64))) {
return PrintAddressWithModuleName(address, has_module_name, module_name, base_address);
}
str_tab = base_address + temp_64;
} else if (temp_64 == 6) {
/* We found DT_SYMTAB */
if (!ReadValue(std::addressof(temp_64), process, dyn_address + ofs + sizeof(u64))) {
return PrintAddressWithModuleName(address, has_module_name, module_name, base_address);
}
sym_tab = base_address + temp_64;
}
}
/* Check that we found all the tables. */
if (!(sym_tab != 0 && str_tab != 0 && num_sym != 0)) {
return PrintAddressWithModuleName(address, has_module_name, module_name, base_address);
}
/* Try to locate an appropriate symbol. */
for (size_t i = 0; i < num_sym; ++i) {
/* Read the symbol from userspace. */
struct {
u32 st_name;
u8 st_info;
u8 st_other;
u16 st_shndx;
u64 st_value;
u64 st_size;
} sym;
{
u64 x[sizeof(sym) / sizeof(u64)];
for (size_t j = 0; j < util::size(x); ++j) {
if (!ReadValue(x + j, process, sym_tab + sizeof(sym) * i + sizeof(u64) * j)) {
return PrintAddressWithModuleName(address, has_module_name, module_name, base_address);
}
}
std::memcpy(std::addressof(sym), x, sizeof(sym));
}
/* Check the symbol is valid/STT_FUNC. */
if (sym.st_shndx == 0 || ((sym.st_shndx & 0xFF00) == 0xFF00)) {
continue;
}
if ((sym.st_info & 0xF) != 2) {
continue;
}
/* Check the address. */
const uintptr_t func_start = base_address + sym.st_value;
if (func_start <= test_address && test_address < func_start + sym.st_size) {
/* Read the symbol name. */
const uintptr_t sym_address = str_tab + sym.st_name;
char sym_name[0x80];
sym_name[util::size(sym_name) - 1] = 0;
for (size_t j = 0; j < util::size(sym_name) - 1; ++j) {
if (!ReadValue(sym_name + j, process, sym_address + j)) {
return PrintAddressWithModuleName(address, has_module_name, module_name, base_address);
}
if (sym_name[j] == 0) {
break;
}
}
/* Print the symbol. */
return PrintAddressWithSymbol(address, has_module_name, module_name, base_address, sym_name, func_start);
}
}
/* Fall back to printing the module. */
return PrintAddressWithModuleName(address, has_module_name, module_name, base_address);
}
}
#endif
void KDebug::PrintBacktrace(KThread *thread) {
#if defined(MESOSPHERE_BUILD_FOR_DEBUGGING)
{
/* Treat no thread as current thread. */
if (thread == nullptr) {
thread = GetCurrentThreadPointer();
}
/* Get the exception context. */
KExceptionContext *e_ctx = GetExceptionContext(thread);
/* Get the owner process. */
if (auto *process = thread->GetOwnerProcess(); process != nullptr) {
/* Lock the owner process. */
KScopedLightLock state_lk(process->GetStateLock());
KScopedLightLock list_lk(process->GetListLock());
/* Suspend all the process's threads. */
{
KScopedSchedulerLock sl;
auto end = process->GetThreadList().end();
for (auto it = process->GetThreadList().begin(); it != end; ++it) {
if (std::addressof(*it) != GetCurrentThreadPointer()) {
it->RequestSuspend(KThread::SuspendType_Backtrace);
}
}
}
/* Print the backtrace. */
MESOSPHERE_RELEASE_LOG("User Backtrace\n");
if ((e_ctx->psr & 0x10) == 0) {
/* 64-bit thread. */
PrintCodeAddress(process, e_ctx->pc, false);
PrintCodeAddress(process, e_ctx->x[30]);
/* Walk the stack frames. */
uintptr_t fp = static_cast<uintptr_t>(e_ctx->x[29]);
for (auto i = 0; i < 0x20 && fp != 0 && util::IsAligned(fp, 0x10); ++i) {
/* Read the next frame. */
struct {
u64 fp;
u64 lr;
} stack_frame;
{
KMemoryInfo mem_info;
ams::svc::PageInfo page_info;
KPhysicalAddress phys_addr;
if (R_FAILED(process->GetPageTable().QueryInfo(std::addressof(mem_info), std::addressof(page_info), fp))) {
break;
}
if ((mem_info.GetState() & KMemoryState_FlagReferenceCounted) == 0) {
break;
}
if ((mem_info.GetAttribute() & KMemoryAttribute_Uncached) != 0) {
break;
}
if ((mem_info.GetPermission() & KMemoryPermission_UserRead) != KMemoryPermission_UserRead) {
break;
}
if (!process->GetPageTable().GetPhysicalAddress(std::addressof(phys_addr), fp)) {
break;
}
if (!IsHeapPhysicalAddress(phys_addr)) {
break;
}
u64 *frame_ptr = GetPointer<u64>(process->GetPageTable().GetHeapVirtualAddress(phys_addr));
stack_frame.fp = frame_ptr[0];
stack_frame.lr = frame_ptr[1];
}
/* Print and advance. */
PrintCodeAddress(process, stack_frame.lr);
fp = stack_frame.fp;
}
} else {
/* 32-bit thread. */
PrintCodeAddress(process, e_ctx->pc, false);
PrintCodeAddress(process, e_ctx->x[14]);
/* Walk the stack frames. */
uintptr_t fp = static_cast<uintptr_t>(e_ctx->x[11]);
for (auto i = 0; i < 0x20 && fp != 0 && util::IsAligned(fp, 4); ++i) {
/* Read the next frame. */
struct {
u32 fp;
u32 lr;
} stack_frame;
{
KMemoryInfo mem_info;
ams::svc::PageInfo page_info;
KPhysicalAddress phys_addr;
/* Read FP */
if (R_FAILED(process->GetPageTable().QueryInfo(std::addressof(mem_info), std::addressof(page_info), fp))) {
break;
}
if ((mem_info.GetState() & KMemoryState_FlagReferenceCounted) == 0) {
break;
}
if ((mem_info.GetAttribute() & KMemoryAttribute_Uncached) != 0) {
break;
}
if ((mem_info.GetPermission() & KMemoryPermission_UserRead) != KMemoryPermission_UserRead) {
break;
}
if (!process->GetPageTable().GetPhysicalAddress(std::addressof(phys_addr), fp)) {
break;
}
if (!IsHeapPhysicalAddress(phys_addr)) {
break;
}
stack_frame.fp = *GetPointer<u32>(process->GetPageTable().GetHeapVirtualAddress(phys_addr));
/* Read LR. */
uintptr_t lr_ptr = (e_ctx->x[13] <= stack_frame.fp && stack_frame.fp < e_ctx->x[13] + PageSize) ? fp + 4 : fp - 4;
if (R_FAILED(process->GetPageTable().QueryInfo(std::addressof(mem_info), std::addressof(page_info), lr_ptr))) {
break;
}
if ((mem_info.GetState() & KMemoryState_FlagReferenceCounted) == 0) {
break;
}
if ((mem_info.GetAttribute() & KMemoryAttribute_Uncached) != 0) {
break;
}
if ((mem_info.GetPermission() & KMemoryPermission_UserRead) != KMemoryPermission_UserRead) {
break;
}
if (!process->GetPageTable().GetPhysicalAddress(std::addressof(phys_addr), lr_ptr)) {
break;
}
if (!IsHeapPhysicalAddress(phys_addr)) {
break;
}
stack_frame.lr = *GetPointer<u32>(process->GetPageTable().GetHeapVirtualAddress(phys_addr));
}
/* Print and advance. */
PrintCodeAddress(process, stack_frame.lr);
fp = stack_frame.fp;
}
}
/* Resume the threads that we suspended. */
{
KScopedSchedulerLock sl;
auto end = process->GetThreadList().end();
for (auto it = process->GetThreadList().begin(); it != end; ++it) {
if (std::addressof(*it) != GetCurrentThreadPointer()) {
it->Resume(KThread::SuspendType_Backtrace);
}
}
}
}
}
#else
MESOSPHERE_UNUSED(thread);
#endif
}
}

55
Source/Atmosphere/libraries/libmesosphere/source/arch/arm64/kern_k_hardware_timer.cpp Normal file
View File

@@ -0,0 +1,55 @@
/*
* Copyright (c) Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <mesosphere.hpp>
namespace ams::kern::arch::arm64 {
void KHardwareTimer::Initialize() {
/* Setup the global timer for the core. */
InitializeGlobalTimer();
/* Set maximum time. */
m_maximum_time = static_cast<s64>(std::min<u64>(std::numeric_limits<s64>::max(), cpu::CounterTimerPhysicalTimerCompareValueRegisterAccessor().GetCompareValue()));
/* Bind the interrupt task for this core. */
Kernel::GetInterruptManager().BindHandler(this, KInterruptName_NonSecurePhysicalTimer, GetCurrentCoreId(), KInterruptController::PriorityLevel_Timer, true, true);
}
void KHardwareTimer::Finalize() {
/* Stop the hardware timer. */
StopTimer();
}
void KHardwareTimer::DoTask() {
/* Handle the interrupt. */
{
KScopedSchedulerLock slk;
KScopedSpinLock lk(this->GetLock());
/* Disable the timer interrupt while we handle this. */
DisableInterrupt();
if (const s64 next_time = this->DoInterruptTaskImpl(GetTick()); 0 < next_time && next_time <= m_maximum_time) {
/* We have a next time, so we should set the time to interrupt and turn the interrupt on. */
SetCompareValue(next_time);
EnableInterrupt();
}
}
/* Clear the timer interrupt. */
Kernel::GetInterruptManager().ClearInterrupt(KInterruptName_NonSecurePhysicalTimer, GetCurrentCoreId());
}
}

19
Source/Atmosphere/libraries/libmesosphere/source/arch/arm64/kern_k_interrupt_controller.board.generic.cpp Normal file
View File

@@ -0,0 +1,19 @@
/*
* Copyright (c) Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <mesosphere.hpp>
/* Include the common implementation. */
#include "../arm/kern_generic_interrupt_controller.inc"

389
Source/Atmosphere/libraries/libmesosphere/source/arch/arm64/kern_k_interrupt_manager.cpp Normal file
View File

@@ -0,0 +1,389 @@
/*
* Copyright (c) Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <mesosphere.hpp>
namespace ams::kern::arch::arm64 {
void KInterruptManager::Initialize(s32 core_id) {
m_interrupt_controller.Initialize(core_id);
}
void KInterruptManager::Finalize(s32 core_id) {
m_interrupt_controller.Finalize(core_id);
}
void KInterruptManager::Save(s32 core_id) {
/* Verify core id. */
MESOSPHERE_ASSERT(core_id == GetCurrentCoreId());
/* Ensure all cores get to this point before continuing. */
cpu::SynchronizeAllCores();
/* If on core 0, save the global interrupts. */
if (core_id == 0) {
MESOSPHERE_ABORT_UNLESS(!m_global_state_saved);
m_interrupt_controller.SaveGlobal(std::addressof(m_global_state));
m_global_state_saved = true;
}
/* Ensure all cores get to this point before continuing. */
cpu::SynchronizeAllCores();
/* Save all local interrupts. */
MESOSPHERE_ABORT_UNLESS(!m_local_state_saved[core_id]);
m_interrupt_controller.SaveCoreLocal(std::addressof(m_local_states[core_id]));
m_local_state_saved[core_id] = true;
/* Ensure all cores get to this point before continuing. */
cpu::SynchronizeAllCores();
/* Finalize all cores other than core 0. */
if (core_id != 0) {
this->Finalize(core_id);
}
/* Ensure all cores get to this point before continuing. */
cpu::SynchronizeAllCores();
/* Finalize core 0. */
if (core_id == 0) {
this->Finalize(core_id);
}
}
void KInterruptManager::Restore(s32 core_id) {
/* Verify core id. */
MESOSPHERE_ASSERT(core_id == GetCurrentCoreId());
/* Ensure all cores get to this point before continuing. */
cpu::SynchronizeAllCores();
/* Initialize core 0. */
if (core_id == 0) {
this->Initialize(core_id);
}
/* Ensure all cores get to this point before continuing. */
cpu::SynchronizeAllCores();
/* Initialize all cores other than core 0. */
if (core_id != 0) {
this->Initialize(core_id);
}
/* Ensure all cores get to this point before continuing. */
cpu::SynchronizeAllCores();
/* Restore all local interrupts. */
MESOSPHERE_ASSERT(m_local_state_saved[core_id]);
m_interrupt_controller.RestoreCoreLocal(std::addressof(m_local_states[core_id]));
m_local_state_saved[core_id] = false;
/* Ensure all cores get to this point before continuing. */
cpu::SynchronizeAllCores();
/* If on core 0, restore the global interrupts. */
if (core_id == 0) {
MESOSPHERE_ASSERT(m_global_state_saved);
m_interrupt_controller.RestoreGlobal(std::addressof(m_global_state));
m_global_state_saved = false;
}
/* Ensure all cores get to this point before continuing. */
cpu::SynchronizeAllCores();
}
bool KInterruptManager::OnHandleInterrupt() {
/* Get the interrupt id. */
const u32 raw_irq = m_interrupt_controller.GetIrq();
const s32 irq = KInterruptController::ConvertRawIrq(raw_irq);
/* Trace the interrupt. */
MESOSPHERE_KTRACE_INTERRUPT(irq);
/* If the IRQ is spurious, we don't need to reschedule. */
if (irq < 0) {
return false;
}
KInterruptTask *task = nullptr;
if (KInterruptController::IsLocal(irq)) {
/* Get local interrupt entry. */
auto &entry = GetLocalInterruptEntry(irq);
if (entry.handler != nullptr) {
/* Set manual clear needed if relevant. */
if (entry.manually_cleared) {
m_interrupt_controller.SetPriorityLevel(irq, KInterruptController::PriorityLevel_Low);
entry.needs_clear = true;
}
/* Set the handler. */
task = entry.handler->OnInterrupt(irq);
} else {
MESOSPHERE_LOG("Core%d: Unhandled local interrupt %d\n", GetCurrentCoreId(), irq);
}
} else if (KInterruptController::IsGlobal(irq)) {
KScopedSpinLock lk(this->GetGlobalInterruptLock());
/* Get global interrupt entry. */
auto &entry = GetGlobalInterruptEntry(irq);
if (entry.handler != nullptr) {
/* Set manual clear needed if relevant. */
if (entry.manually_cleared) {
m_interrupt_controller.Disable(irq);
entry.needs_clear = true;
}
/* Set the handler. */
task = entry.handler->OnInterrupt(irq);
} else {
MESOSPHERE_LOG("Core%d: Unhandled global interrupt %d\n", GetCurrentCoreId(), irq);
}
} else {
MESOSPHERE_LOG("Invalid interrupt %d\n", irq);
}
/* Acknowledge the interrupt. */
m_interrupt_controller.EndOfInterrupt(raw_irq);
/* If we found no task, then we don't need to reschedule. */
if (task == nullptr) {
return false;
}
/* If the task isn't the dummy task, we should add it to the queue. */
if (task != GetDummyInterruptTask()) {
Kernel::GetInterruptTaskManager().EnqueueTask(task);
}
return true;
}
void KInterruptManager::HandleInterrupt(bool user_mode) {
/* On interrupt, call OnHandleInterrupt() to determine if we need rescheduling and handle. */
const bool needs_scheduling = Kernel::GetInterruptManager().OnHandleInterrupt();
/* If we need scheduling, */
if (needs_scheduling) {
if (user_mode) {
/* If the interrupt occurred in the middle of a userland cache maintenance operation, ensure memory consistency before rescheduling. */
if (GetCurrentThread().IsInUserCacheMaintenanceOperation()) {
cpu::DataSynchronizationBarrier();
}
/* If the user disable count is set, we may need to pin the current thread. */
if (GetCurrentThread().GetUserDisableCount() != 0 && GetCurrentProcess().GetPinnedThread(GetCurrentCoreId()) == nullptr) {
KScopedSchedulerLock sl;
/* Pin the current thread. */
GetCurrentProcess().PinCurrentThread();
/* Set the interrupt flag for the thread. */
GetCurrentThread().SetInterruptFlag();
/* Request interrupt scheduling. */
Kernel::GetScheduler().RequestScheduleOnInterrupt();
} else {
/* Request interrupt scheduling. */
Kernel::GetScheduler().RequestScheduleOnInterrupt();
}
} else {
/* If the interrupt occurred in the middle of a cache maintenance operation, ensure memory consistency before rescheduling. */
if (GetCurrentThread().IsInCacheMaintenanceOperation()) {
cpu::DataSynchronizationBarrier();
} else if (GetCurrentThread().IsInTlbMaintenanceOperation()) {
/* Otherwise, if we're in the middle of a tlb maintenance operation, ensure inner shareable memory consistency before rescheduling. */
cpu::DataSynchronizationBarrierInnerShareable();
}
/* Request interrupt scheduling. */
Kernel::GetScheduler().RequestScheduleOnInterrupt();
}
}
/* If user mode, check if the thread needs termination. */
/* If it does, we can take advantage of this to terminate it. */
if (user_mode) {
KThread *cur_thread = GetCurrentThreadPointer();
if (cur_thread->IsTerminationRequested()) {
EnableInterrupts();
cur_thread->Exit();
}
}
}
Result KInterruptManager::BindHandler(KInterruptHandler *handler, s32 irq, s32 core_id, s32 priority, bool manual_clear, bool level) {
MESOSPHERE_UNUSED(core_id);
R_UNLESS(KInterruptController::IsGlobal(irq) || KInterruptController::IsLocal(irq), svc::ResultOutOfRange());
if (KInterruptController::IsGlobal(irq)) {
KScopedInterruptDisable di;
KScopedSpinLock lk(this->GetGlobalInterruptLock());
R_RETURN(this->BindGlobal(handler, irq, core_id, priority, manual_clear, level));
} else {
MESOSPHERE_ASSERT(core_id == GetCurrentCoreId());
KScopedInterruptDisable di;
R_RETURN(this->BindLocal(handler, irq, priority, manual_clear));
}
}
Result KInterruptManager::UnbindHandler(s32 irq, s32 core_id) {
MESOSPHERE_UNUSED(core_id);
R_UNLESS(KInterruptController::IsGlobal(irq) || KInterruptController::IsLocal(irq), svc::ResultOutOfRange());
if (KInterruptController::IsGlobal(irq)) {
KScopedInterruptDisable di;
KScopedSpinLock lk(this->GetGlobalInterruptLock());
R_RETURN(this->UnbindGlobal(irq));
} else {
MESOSPHERE_ASSERT(core_id == GetCurrentCoreId());
KScopedInterruptDisable di;
R_RETURN(this->UnbindLocal(irq));
}
}
Result KInterruptManager::ClearInterrupt(s32 irq, s32 core_id) {
MESOSPHERE_UNUSED(core_id);
R_UNLESS(KInterruptController::IsGlobal(irq) || KInterruptController::IsLocal(irq), svc::ResultOutOfRange());
if (KInterruptController::IsGlobal(irq)) {
KScopedInterruptDisable di;
KScopedSpinLock lk(this->GetGlobalInterruptLock());
R_RETURN(this->ClearGlobal(irq));
} else {
MESOSPHERE_ASSERT(core_id == GetCurrentCoreId());
KScopedInterruptDisable di;
R_RETURN(this->ClearLocal(irq));
}
}
Result KInterruptManager::BindGlobal(KInterruptHandler *handler, s32 irq, s32 core_id, s32 priority, bool manual_clear, bool level) {
/* Ensure the priority level is valid. */
R_UNLESS(KInterruptController::PriorityLevel_High <= priority, svc::ResultOutOfRange());
R_UNLESS(priority <= KInterruptController::PriorityLevel_Low, svc::ResultOutOfRange());
/* Ensure we aren't already bound. */
auto &entry = GetGlobalInterruptEntry(irq);
R_UNLESS(entry.handler == nullptr, svc::ResultBusy());
/* Set entry fields. */
entry.needs_clear = false;
entry.manually_cleared = manual_clear;
entry.handler = handler;
/* Configure the interrupt as level or edge. */
if (level) {
m_interrupt_controller.SetLevel(irq);
} else {
m_interrupt_controller.SetEdge(irq);
}
/* Configure the interrupt. */
m_interrupt_controller.Clear(irq);
m_interrupt_controller.SetTarget(irq, core_id);
m_interrupt_controller.SetPriorityLevel(irq, priority);
m_interrupt_controller.Enable(irq);
R_SUCCEED();
}
Result KInterruptManager::BindLocal(KInterruptHandler *handler, s32 irq, s32 priority, bool manual_clear) {
/* Ensure the priority level is valid. */
R_UNLESS(KInterruptController::PriorityLevel_High <= priority, svc::ResultOutOfRange());
R_UNLESS(priority <= KInterruptController::PriorityLevel_Low, svc::ResultOutOfRange());
/* Ensure we aren't already bound. */
auto &entry = this->GetLocalInterruptEntry(irq);
R_UNLESS(entry.handler == nullptr, svc::ResultBusy());
/* Set entry fields. */
entry.needs_clear = false;
entry.manually_cleared = manual_clear;
entry.handler = handler;
entry.priority = static_cast<u8>(priority);
/* Configure the interrupt. */
m_interrupt_controller.Clear(irq);
m_interrupt_controller.SetPriorityLevel(irq, priority);
m_interrupt_controller.Enable(irq);
R_SUCCEED();
}
Result KInterruptManager::UnbindGlobal(s32 irq) {
for (size_t core_id = 0; core_id < cpu::NumCores; core_id++) {
m_interrupt_controller.ClearTarget(irq, static_cast<s32>(core_id));
}
m_interrupt_controller.SetPriorityLevel(irq, KInterruptController::PriorityLevel_Low);
m_interrupt_controller.Disable(irq);
GetGlobalInterruptEntry(irq).handler = nullptr;
R_SUCCEED();
}
Result KInterruptManager::UnbindLocal(s32 irq) {
auto &entry = this->GetLocalInterruptEntry(irq);
R_UNLESS(entry.handler != nullptr, svc::ResultInvalidState());
m_interrupt_controller.SetPriorityLevel(irq, KInterruptController::PriorityLevel_Low);
m_interrupt_controller.Disable(irq);
entry.handler = nullptr;
R_SUCCEED();
}
Result KInterruptManager::ClearGlobal(s32 irq) {
/* We can't clear an entry with no handler. */
auto &entry = GetGlobalInterruptEntry(irq);
R_UNLESS(entry.handler != nullptr, svc::ResultInvalidState());
/* If auto-cleared, we can succeed immediately. */
R_SUCCEED_IF(!entry.manually_cleared);
R_SUCCEED_IF(!entry.needs_clear);
/* Clear and enable. */
entry.needs_clear = false;
m_interrupt_controller.Enable(irq);
R_SUCCEED();
}
Result KInterruptManager::ClearLocal(s32 irq) {
/* We can't clear an entry with no handler. */
auto &entry = this->GetLocalInterruptEntry(irq);
R_UNLESS(entry.handler != nullptr, svc::ResultInvalidState());
/* If auto-cleared, we can succeed immediately. */
R_SUCCEED_IF(!entry.manually_cleared);
R_SUCCEED_IF(!entry.needs_clear);
/* Clear and set priority. */
entry.needs_clear = false;
m_interrupt_controller.SetPriorityLevel(irq, entry.priority);
R_SUCCEED();
}
}

1044
Source/Atmosphere/libraries/libmesosphere/source/arch/arm64/kern_k_page_table.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

540
Source/Atmosphere/libraries/libmesosphere/source/arch/arm64/kern_k_page_table_impl.cpp Normal file
View File

@@ -0,0 +1,540 @@
/*
* Copyright (c) Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <mesosphere.hpp>
namespace ams::kern::arch::arm64 {
void KPageTableImpl::InitializeForKernel(void *tb, KVirtualAddress start, KVirtualAddress end) {
m_table = static_cast<L1PageTableEntry *>(tb);
m_is_kernel = true;
m_num_entries = util::AlignUp(end - start, L1BlockSize) / L1BlockSize;
/* Page table entries created by KInitialPageTable need to be iterated and modified to ensure KPageTable invariants. */
PageTableEntry *level_entries[EntryLevel_Count] = { nullptr, nullptr, m_table };
u32 level = EntryLevel_L1;
while (level != EntryLevel_L1 || (level_entries[EntryLevel_L1] - static_cast<PageTableEntry *>(m_table)) < m_num_entries) {
/* Get the pte; it must never have the validity-extension flag set. */
auto *pte = level_entries[level];
MESOSPHERE_ASSERT((pte->GetSoftwareReservedBits() & PageTableEntry::SoftwareReservedBit_Valid) == 0);
/* While we're a table, recurse, fixing up the reference counts. */
while (level > EntryLevel_L3 && pte->IsMappedTable()) {
/* Count how many references are in the table. */
auto *table = GetPointer<PageTableEntry>(GetPageTableVirtualAddress(pte->GetTable()));
size_t ref_count = 0;
for (size_t i = 0; i < BlocksPerTable; ++i) {
if (table[i].IsMapped()) {
++ref_count;
}
}
/* Set the reference count for our new page, adding one additional uncloseable reference; kernel pages must never be unreferenced. */
pte->SetTableReferenceCount(ref_count + 1).SetValid();
/* Iterate downwards. */
level -= 1;
level_entries[level] = table;
pte = level_entries[level];
/* Check that the entry isn't unexpected. */
MESOSPHERE_ASSERT((pte->GetSoftwareReservedBits() & PageTableEntry::SoftwareReservedBit_Valid) == 0);
}
/* We're dealing with some block. If it's mapped, set it valid. */
if (pte->IsMapped()) {
pte->SetValid();
}
/* Advance. */
while (true) {
/* Advance to the next entry at the current level. */
if (!util::IsAligned(reinterpret_cast<uintptr_t>(++level_entries[level]), PageSize)) {
break;
}
/* If we're at the end of a level, advance upwards. */
level_entries[level++] = nullptr;
if (level > EntryLevel_L1) {
return;
}
}
}
}
void KPageTableImpl::InitializeForProcess(void *tb, KVirtualAddress start, KVirtualAddress end) {
m_table = static_cast<L1PageTableEntry *>(tb);
m_is_kernel = false;
m_num_entries = util::AlignUp(end - start, L1BlockSize) / L1BlockSize;
}
L1PageTableEntry *KPageTableImpl::Finalize() {
return m_table;
}
bool KPageTableImpl::BeginTraversal(TraversalEntry *out_entry, TraversalContext *out_context, KProcessAddress address) const {
/* Setup invalid defaults. */
*out_entry = {};
*out_context = {};
/* Validate that we can read the actual entry. */
const size_t l0_index = GetL0Index(address);
const size_t l1_index = GetL1Index(address);
if (m_is_kernel) {
/* Kernel entries must be accessed via TTBR1. */
if ((l0_index != MaxPageTableEntries - 1) || (l1_index < MaxPageTableEntries - m_num_entries)) {
return false;
}
} else {
/* User entries must be accessed with TTBR0. */
if ((l0_index != 0) || l1_index >= m_num_entries) {
return false;
}
}
/* Get the L1 entry, and check if it's a table. */
out_context->level_entries[EntryLevel_L1] = this->GetL1Entry(address);
if (out_context->level_entries[EntryLevel_L1]->IsMappedTable()) {
/* Get the L2 entry, and check if it's a table. */
out_context->level_entries[EntryLevel_L2] = this->GetL2EntryFromTable(GetPageTableVirtualAddress(out_context->level_entries[EntryLevel_L1]->GetTable()), address);
if (out_context->level_entries[EntryLevel_L2]->IsMappedTable()) {
/* Get the L3 entry. */
out_context->level_entries[EntryLevel_L3] = this->GetL3EntryFromTable(GetPageTableVirtualAddress(out_context->level_entries[EntryLevel_L2]->GetTable()), address);
/* It's either a page or not. */
out_context->level = EntryLevel_L3;
} else {
/* Not a L2 table, so possibly an L2 block. */
out_context->level = EntryLevel_L2;
}
} else {
/* Not a L1 table, so possibly an L1 block. */
out_context->level = EntryLevel_L1;
}
/* Determine other fields. */
const auto *pte = out_context->level_entries[out_context->level];
out_context->is_contiguous = pte->IsContiguous();
out_entry->sw_reserved_bits = pte->GetSoftwareReservedBits();
out_entry->attr = 0;
out_entry->phys_addr = this->GetBlock(pte, out_context->level) + this->GetOffset(address, out_context->level);
out_entry->block_size = static_cast<size_t>(1) << (PageBits + LevelBits * out_context->level + 4 * out_context->is_contiguous);
return out_context->level == EntryLevel_L3 ? pte->IsPage() : pte->IsBlock();
}
bool KPageTableImpl::ContinueTraversal(TraversalEntry *out_entry, TraversalContext *context) const {
/* Advance entry. */
auto *cur_pte = context->level_entries[context->level];
auto *next_pte = reinterpret_cast<PageTableEntry *>(context->is_contiguous ? util::AlignDown(reinterpret_cast<uintptr_t>(cur_pte), BlocksPerContiguousBlock * sizeof(PageTableEntry)) + BlocksPerContiguousBlock * sizeof(PageTableEntry) : reinterpret_cast<uintptr_t>(cur_pte) + sizeof(PageTableEntry));
/* Set the pte. */
context->level_entries[context->level] = next_pte;
/* Advance appropriately. */
while (context->level < EntryLevel_L1 && util::IsAligned(reinterpret_cast<uintptr_t>(context->level_entries[context->level]), PageSize)) {
/* Advance the above table by one entry. */
context->level_entries[context->level + 1]++;
context->level = static_cast<EntryLevel>(util::ToUnderlying(context->level) + 1);
}
/* Check if we've hit the end of the L1 table. */
if (context->level == EntryLevel_L1) {
if (context->level_entries[EntryLevel_L1] - static_cast<const PageTableEntry *>(m_table) >= m_num_entries) {
*context = {};
*out_entry = {};
return false;
}
}
/* We may have advanced to a new table, and if we have we should descend. */
while (context->level > EntryLevel_L3 && context->level_entries[context->level]->IsMappedTable()) {
context->level_entries[context->level - 1] = GetPointer<PageTableEntry>(GetPageTableVirtualAddress(context->level_entries[context->level]->GetTable()));
context->level = static_cast<EntryLevel>(util::ToUnderlying(context->level) - 1);
}
const auto *pte = context->level_entries[context->level];
context->is_contiguous = pte->IsContiguous();
out_entry->sw_reserved_bits = pte->GetSoftwareReservedBits();
out_entry->attr = 0;
out_entry->phys_addr = this->GetBlock(pte, context->level);
out_entry->block_size = static_cast<size_t>(1) << (PageBits + LevelBits * context->level + 4 * context->is_contiguous);
return context->level == EntryLevel_L3 ? pte->IsPage() : pte->IsBlock();
}
bool KPageTableImpl::GetPhysicalAddress(KPhysicalAddress *out, KProcessAddress address) const {
/* Validate that we can read the actual entry. */
const size_t l0_index = GetL0Index(address);
const size_t l1_index = GetL1Index(address);
if (m_is_kernel) {
/* Kernel entries must be accessed via TTBR1. */
if ((l0_index != MaxPageTableEntries - 1) || (l1_index < MaxPageTableEntries - m_num_entries)) {
return false;
}
} else {
/* User entries must be accessed with TTBR0. */
if ((l0_index != 0) || l1_index >= m_num_entries) {
return false;
}
}
/* Get the L1 entry, and check if it's a table. */
const PageTableEntry *pte = this->GetL1Entry(address);
EntryLevel level = EntryLevel_L1;
if (pte->IsMappedTable()) {
/* Get the L2 entry, and check if it's a table. */
pte = this->GetL2EntryFromTable(GetPageTableVirtualAddress(pte->GetTable()), address);
level = EntryLevel_L2;
if (pte->IsMappedTable()) {
pte = this->GetL3EntryFromTable(GetPageTableVirtualAddress(pte->GetTable()), address);
level = EntryLevel_L3;
}
}
const bool is_block = level == EntryLevel_L3 ? pte->IsPage() : pte->IsBlock();
if (is_block) {
*out = this->GetBlock(pte, level) + this->GetOffset(address, level);
} else {
*out = Null<KPhysicalAddress>;
}
return is_block;
}
bool KPageTableImpl::MergePages(KVirtualAddress *out, TraversalContext *context, EntryUpdatedCallback on_entry_updated, const void *pt) {
/* We want to upgrade the pages by one step. */
if (context->is_contiguous) {
/* We can't merge an L1 table. */
if (context->level == EntryLevel_L1) {
return false;
}
/* We want to upgrade a contiguous mapping in a table to a block. */
PageTableEntry *pte = reinterpret_cast<PageTableEntry *>(util::AlignDown(reinterpret_cast<uintptr_t>(context->level_entries[context->level]), BlocksPerTable * sizeof(PageTableEntry)));
const KPhysicalAddress phys_addr = util::AlignDown(GetBlock(pte, context->level), GetBlockSize(static_cast<EntryLevel>(context->level + 1), false));
/* First, check that all entries are valid for us to merge. */
const u64 entry_template = pte->GetEntryTemplateForMerge();
for (size_t i = 0; i < BlocksPerTable; ++i) {
if (!pte[i].IsForMerge(entry_template | GetInteger(phys_addr + (i << (PageBits + LevelBits * context->level))) | PageTableEntry::ContigType_Contiguous | pte->GetTestTableMask())) {
return false;
}
if (i > 0 && pte[i].IsHeadOrHeadAndBodyMergeDisabled()) {
return false;
}
if (i < BlocksPerTable - 1 && pte[i].IsTailMergeDisabled()) {
return false;
}
}
/* The entries are valid for us to merge, so merge them. */
const auto *head_pte = pte;
const auto *tail_pte = pte + BlocksPerTable - 1;
const auto sw_reserved_bits = PageTableEntry::EncodeSoftwareReservedBits(head_pte->IsHeadMergeDisabled(), head_pte->IsHeadAndBodyMergeDisabled(), tail_pte->IsTailMergeDisabled());
*context->level_entries[context->level + 1] = PageTableEntry(PageTableEntry::BlockTag{}, phys_addr, PageTableEntry(entry_template), sw_reserved_bits, false, false);
on_entry_updated(pt);
/* Update our context. */
context->is_contiguous = false;
context->level = static_cast<EntryLevel>(util::ToUnderlying(context->level) + 1);
/* Set the output to the table we just freed. */
*out = KVirtualAddress(pte);
} else {
/* We want to upgrade a non-contiguous mapping to a contiguous mapping. */
PageTableEntry *pte = reinterpret_cast<PageTableEntry *>(util::AlignDown(reinterpret_cast<uintptr_t>(context->level_entries[context->level]), BlocksPerContiguousBlock * sizeof(PageTableEntry)));
const KPhysicalAddress phys_addr = util::AlignDown(GetBlock(pte, context->level), GetBlockSize(context->level, true));
/* First, check that all entries are valid for us to merge. */
const u64 entry_template = pte->GetEntryTemplateForMerge();
for (size_t i = 0; i < BlocksPerContiguousBlock; ++i) {
if (!pte[i].IsForMerge(entry_template | GetInteger(phys_addr + (i << (PageBits + LevelBits * context->level))) | pte->GetTestTableMask())) {
return false;
}
if (i > 0 && pte[i].IsHeadOrHeadAndBodyMergeDisabled()) {
return false;
}
if (i < BlocksPerContiguousBlock - 1 && pte[i].IsTailMergeDisabled()) {
return false;
}
}
/* The entries are valid for us to merge, so merge them. */
const auto *head_pte = pte;
const auto *tail_pte = pte + BlocksPerContiguousBlock - 1;
const auto sw_reserved_bits = PageTableEntry::EncodeSoftwareReservedBits(head_pte->IsHeadMergeDisabled(), head_pte->IsHeadAndBodyMergeDisabled(), tail_pte->IsTailMergeDisabled());
for (size_t i = 0; i < BlocksPerContiguousBlock; ++i) {
pte[i] = PageTableEntry(PageTableEntry::BlockTag{}, phys_addr + (i << (PageBits + LevelBits * context->level)), PageTableEntry(entry_template), sw_reserved_bits, true, context->level == EntryLevel_L3);
}
on_entry_updated(pt);
/* Update our context. */
context->level_entries[context->level] = pte;
context->is_contiguous = true;
}
return true;
}
void KPageTableImpl::SeparatePages(TraversalEntry *entry, TraversalContext *context, KProcessAddress address, PageTableEntry *pte, EntryUpdatedCallback on_entry_updated, const void *pt) const {
/* We want to downgrade the pages by one step. */
if (context->is_contiguous) {
/* We want to downgrade a contiguous mapping to a non-contiguous mapping. */
pte = reinterpret_cast<PageTableEntry *>(util::AlignDown(reinterpret_cast<uintptr_t>(context->level_entries[context->level]), BlocksPerContiguousBlock * sizeof(PageTableEntry)));
auto * const first = pte;
const KPhysicalAddress block = this->GetBlock(first, context->level);
for (size_t i = 0; i < BlocksPerContiguousBlock; ++i) {
pte[i] = PageTableEntry(PageTableEntry::BlockTag{}, block + (i << (PageBits + LevelBits * context->level)), PageTableEntry(first->GetEntryTemplateForSeparateContiguous(i)), PageTableEntry::SeparateContiguousTag{});
}
on_entry_updated(pt);
context->is_contiguous = false;
context->level_entries[context->level] = pte + (this->GetLevelIndex(address, context->level) & (BlocksPerContiguousBlock - 1));
} else {
/* We want to downgrade a block into a table. */
auto * const first = context->level_entries[context->level];
const KPhysicalAddress block = this->GetBlock(first, context->level);
for (size_t i = 0; i < BlocksPerTable; ++i) {
pte[i] = PageTableEntry(PageTableEntry::BlockTag{}, block + (i << (PageBits + LevelBits * (context->level - 1))), PageTableEntry(first->GetEntryTemplateForSeparate(i)), PageTableEntry::SoftwareReservedBit_None, true, context->level - 1 == EntryLevel_L3);
}
context->is_contiguous = true;
context->level = static_cast<EntryLevel>(util::ToUnderlying(context->level) - 1);
/* Wait for pending stores to complete. */
cpu::DataSynchronizationBarrierInnerShareableStore();
/* Update the block entry to be a table entry. */
*context->level_entries[context->level + 1] = PageTableEntry(PageTableEntry::TableTag{}, KPageTable::GetPageTablePhysicalAddress(KVirtualAddress(pte)), m_is_kernel, true, BlocksPerTable);
on_entry_updated(pt);
context->level_entries[context->level] = pte + this->GetLevelIndex(address, context->level);
}
entry->sw_reserved_bits = context->level_entries[context->level]->GetSoftwareReservedBits();
entry->attr = 0;
entry->phys_addr = this->GetBlock(context->level_entries[context->level], context->level) + this->GetOffset(address, context->level);
entry->block_size = static_cast<size_t>(1) << (PageBits + LevelBits * context->level + 4 * context->is_contiguous);
}
void KPageTableImpl::Dump(uintptr_t start, size_t size) const {
/* If zero size, there's nothing to dump. */
if (size == 0) {
return;
}
/* Define extents. */
const uintptr_t end = start + size;
const uintptr_t last = end - 1;
/* Define tracking variables. */
bool unmapped = false;
uintptr_t unmapped_start = 0;
/* Walk the table. */
uintptr_t cur = start;
while (cur < end) {
/* Validate that we can read the actual entry. */
const size_t l0_index = GetL0Index(cur);
const size_t l1_index = GetL1Index(cur);
if (m_is_kernel) {
/* Kernel entries must be accessed via TTBR1. */
if ((l0_index != MaxPageTableEntries - 1) || (l1_index < MaxPageTableEntries - m_num_entries)) {
return;
}
} else {
/* User entries must be accessed with TTBR0. */
if ((l0_index != 0) || l1_index >= m_num_entries) {
return;
}
}
/* Try to get from l1 table. */
const L1PageTableEntry *l1_entry = this->GetL1Entry(cur);
if (l1_entry->IsBlock()) {
/* Update. */
cur = util::AlignDown(cur, L1BlockSize);
if (unmapped) {
unmapped = false;
MESOSPHERE_RELEASE_LOG("%016lx - %016lx: not mapped\n", unmapped_start, cur - 1);
}
/* Print. */
MESOSPHERE_RELEASE_LOG("%016lx: %016lx PA=%p SZ=1G Mapped=%d UXN=%d PXN=%d Cont=%d nG=%d AF=%d SH=%x RO=%d UA=%d NS=%d AttrIndx=%d NoMerge=%d,%d,%d\n", cur,
*reinterpret_cast<const u64 *>(l1_entry),
reinterpret_cast<void *>(GetInteger(l1_entry->GetBlock())),
l1_entry->IsMapped(),
l1_entry->IsUserExecuteNever(),
l1_entry->IsPrivilegedExecuteNever(),
l1_entry->IsContiguous(),
!l1_entry->IsGlobal(),
static_cast<int>(l1_entry->GetAccessFlagInteger()),
static_cast<unsigned int>(l1_entry->GetShareableInteger()),
l1_entry->IsReadOnly(),
l1_entry->IsUserAccessible(),
l1_entry->IsNonSecure(),
static_cast<int>(l1_entry->GetPageAttributeInteger()),
l1_entry->IsHeadMergeDisabled(),
l1_entry->IsHeadAndBodyMergeDisabled(),
l1_entry->IsTailMergeDisabled());
/* Advance. */
cur += L1BlockSize;
continue;
} else if (!l1_entry->IsTable()) {
/* Update. */
cur = util::AlignDown(cur, L1BlockSize);
if (!unmapped) {
unmapped_start = cur;
unmapped = true;
}
/* Advance. */
cur += L1BlockSize;
continue;
}
/* Try to get from l2 table. */
const L2PageTableEntry *l2_entry = this->GetL2Entry(l1_entry, cur);
if (l2_entry->IsBlock()) {
/* Update. */
cur = util::AlignDown(cur, L2BlockSize);
if (unmapped) {
unmapped = false;
MESOSPHERE_RELEASE_LOG("%016lx - %016lx: not mapped\n", unmapped_start, cur - 1);
}
/* Print. */
MESOSPHERE_RELEASE_LOG("%016lx: %016lx PA=%p SZ=2M Mapped=%d UXN=%d PXN=%d Cont=%d nG=%d AF=%d SH=%x RO=%d UA=%d NS=%d AttrIndx=%d NoMerge=%d,%d,%d\n", cur,
*reinterpret_cast<const u64 *>(l2_entry),
reinterpret_cast<void *>(GetInteger(l2_entry->GetBlock())),
l2_entry->IsMapped(),
l2_entry->IsUserExecuteNever(),
l2_entry->IsPrivilegedExecuteNever(),
l2_entry->IsContiguous(),
!l2_entry->IsGlobal(),
static_cast<int>(l2_entry->GetAccessFlagInteger()),
static_cast<unsigned int>(l2_entry->GetShareableInteger()),
l2_entry->IsReadOnly(),
l2_entry->IsUserAccessible(),
l2_entry->IsNonSecure(),
static_cast<int>(l2_entry->GetPageAttributeInteger()),
l2_entry->IsHeadMergeDisabled(),
l2_entry->IsHeadAndBodyMergeDisabled(),
l2_entry->IsTailMergeDisabled());
/* Advance. */
cur += L2BlockSize;
continue;
} else if (!l2_entry->IsTable()) {
/* Update. */
cur = util::AlignDown(cur, L2BlockSize);
if (!unmapped) {
unmapped_start = cur;
unmapped = true;
}
/* Advance. */
cur += L2BlockSize;
continue;
}
/* Try to get from l3 table. */
const L3PageTableEntry *l3_entry = this->GetL3Entry(l2_entry, cur);
if (l3_entry->IsBlock()) {
/* Update. */
cur = util::AlignDown(cur, L3BlockSize);
if (unmapped) {
unmapped = false;
MESOSPHERE_RELEASE_LOG("%016lx - %016lx: not mapped\n", unmapped_start, cur - 1);
}
/* Print. */
MESOSPHERE_RELEASE_LOG("%016lx: %016lx PA=%p SZ=4K Mapped=%d UXN=%d PXN=%d Cont=%d nG=%d AF=%d SH=%x RO=%d UA=%d NS=%d AttrIndx=%d NoMerge=%d,%d,%d\n", cur,
*reinterpret_cast<const u64 *>(l3_entry),
reinterpret_cast<void *>(GetInteger(l3_entry->GetBlock())),
l3_entry->IsMapped(),
l3_entry->IsUserExecuteNever(),
l3_entry->IsPrivilegedExecuteNever(),
l3_entry->IsContiguous(),
!l3_entry->IsGlobal(),
static_cast<int>(l3_entry->GetAccessFlagInteger()),
static_cast<unsigned int>(l3_entry->GetShareableInteger()),
l3_entry->IsReadOnly(),
l3_entry->IsUserAccessible(),
l3_entry->IsNonSecure(),
static_cast<int>(l3_entry->GetPageAttributeInteger()),
l3_entry->IsHeadMergeDisabled(),
l3_entry->IsHeadAndBodyMergeDisabled(),
l3_entry->IsTailMergeDisabled());
/* Advance. */
cur += L3BlockSize;
continue;
} else {
/* Update. */
cur = util::AlignDown(cur, L3BlockSize);
if (!unmapped) {
unmapped_start = cur;
unmapped = true;
}
/* Advance. */
cur += L3BlockSize;
continue;
}
}
/* Print the last unmapped range if necessary. */
if (unmapped) {
MESOSPHERE_RELEASE_LOG("%016lx - %016lx: not mapped\n", unmapped_start, last);
}
}
size_t KPageTableImpl::CountPageTables() const {
size_t num_tables = 0;
#if defined(MESOSPHERE_BUILD_FOR_DEBUGGING)
{
++num_tables;
for (size_t l1_index = 0; l1_index < m_num_entries; ++l1_index) {
auto &l1_entry = m_table[l1_index];
if (l1_entry.IsTable()) {
++num_tables;
for (size_t l2_index = 0; l2_index < MaxPageTableEntries; ++l2_index) {
auto *l2_entry = GetPointer<L2PageTableEntry>(GetTableEntry(KMemoryLayout::GetLinearVirtualAddress(l1_entry.GetTable()), l2_index));
if (l2_entry->IsTable()) {
++num_tables;
}
}
}
}
}
#endif
return num_tables;
}
}

38
Source/Atmosphere/libraries/libmesosphere/source/arch/arm64/kern_k_supervisor_page_table.cpp Normal file
View File

@@ -0,0 +1,38 @@
/*
* Copyright (c) Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <mesosphere.hpp>
namespace ams::kern::arch::arm64 {
void KSupervisorPageTable::Initialize(s32 core_id) {
/* Verify that sctlr_el1 has the wxn bit set. */
MESOSPHERE_ABORT_UNLESS(cpu::SystemControlRegisterAccessor().GetWxn());
/* Invalidate the entire TLB. */
cpu::InvalidateEntireTlb();
/* If core 0, initialize our base page table. */
if (core_id == 0) {
/* TODO: constexpr defines. */
const u64 ttbr1 = cpu::GetTtbr1El1() & 0xFFFFFFFFFFFFul;
const u64 kernel_vaddr_start = 0xFFFFFF8000000000ul;
const u64 kernel_vaddr_end = 0xFFFFFFFFFFE00000ul;
void *table = GetVoidPointer(KPageTableBase::GetLinearMappedVirtualAddress(ttbr1));
m_page_table.InitializeForKernel(table, kernel_vaddr_start, kernel_vaddr_end);
}
}
}

283
Source/Atmosphere/libraries/libmesosphere/source/arch/arm64/kern_k_thread_context.cpp Normal file
View File

@@ -0,0 +1,283 @@
/*
* Copyright (c) Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <mesosphere.hpp>
namespace ams::kern::arch::arm64 {
/* These are implemented elsewhere (asm). */
void UserModeThreadStarter();
void SupervisorModeThreadStarter();
void InvokeSupervisorModeThread(uintptr_t argument, uintptr_t entrypoint) {
/* Invoke the function. */
using SupervisorModeFunctionType = void (*)(uintptr_t);
reinterpret_cast<SupervisorModeFunctionType>(entrypoint)(argument);
/* Wait forever. */
AMS_INFINITE_LOOP();
}
void OnThreadStart() {
MESOSPHERE_ASSERT(!KInterruptManager::AreInterruptsEnabled());
/* Send KDebug event for this thread's creation. */
{
KScopedInterruptEnable ei;
const uintptr_t params[2] = { GetCurrentThread().GetId(), GetInteger(GetCurrentThread().GetThreadLocalRegionAddress()) };
KDebug::OnDebugEvent(ams::svc::DebugEvent_CreateThread, params, util::size(params));
}
/* Handle any pending dpc. */
while (GetCurrentThread().HasDpc()) {
KDpcManager::HandleDpc();
}
/* Clear our status as in an exception handler */
GetCurrentThread().ClearInExceptionHandler();
}
namespace {
ALWAYS_INLINE bool IsFpuEnabled() {
return cpu::ArchitecturalFeatureAccessControlRegisterAccessor().IsFpEnabled();
}
ALWAYS_INLINE void EnableFpu() {
cpu::ArchitecturalFeatureAccessControlRegisterAccessor().SetFpEnabled(true).Store();
cpu::InstructionMemoryBarrier();
}
uintptr_t SetupStackForUserModeThreadStarter(KVirtualAddress pc, KVirtualAddress k_sp, KVirtualAddress u_sp, uintptr_t arg, const bool is_64_bit) {
/* NOTE: Stack layout on entry looks like following: */
/* SP */
/* | */
/* v */
/* | KExceptionContext (size 0x120) | KThread::StackParameters (size 0x130) | */
KExceptionContext *ctx = GetPointer<KExceptionContext>(k_sp) - 1;
/* Clear context. */
std::memset(ctx, 0, sizeof(*ctx));
/* Set PC and argument. */
ctx->pc = GetInteger(pc) & ~(UINT64_C(1));
ctx->x[0] = arg;
/* Set PSR. */
if (is_64_bit) {
ctx->psr = 0;
} else {
constexpr u64 PsrArmValue = 0x00;
constexpr u64 PsrThumbValue = 0x20;
ctx->psr = ((pc & 1) == 0 ? PsrArmValue : PsrThumbValue) | (0x10);
MESOSPHERE_LOG("Creating User 32-Thread, %016lx\n", GetInteger(pc));
}
/* Set CFI-value. */
if (is_64_bit) {
ctx->x[18] = KSystemControl::GenerateRandomU64() | 1;
}
/* Set stack pointer. */
if (is_64_bit) {
ctx->sp = GetInteger(u_sp);
} else {
ctx->x[13] = GetInteger(u_sp);
}
return reinterpret_cast<uintptr_t>(ctx);
}
uintptr_t SetupStackForSupervisorModeThreadStarter(KVirtualAddress pc, KVirtualAddress sp, uintptr_t arg) {
/* NOTE: Stack layout on entry looks like following: */
/* SP */
/* | */
/* v */
/* | u64 argument | u64 entrypoint | KThread::StackParameters (size 0x140) | */
static_assert(sizeof(KThread::StackParameters) == 0x140);
u64 *stack = GetPointer<u64>(sp);
*(--stack) = GetInteger(pc);
*(--stack) = arg;
return reinterpret_cast<uintptr_t>(stack);
}
}
Result KThreadContext::Initialize(KVirtualAddress u_pc, KVirtualAddress k_sp, KVirtualAddress u_sp, uintptr_t arg, bool is_user, bool is_64_bit, bool is_main) {
MESOSPHERE_ASSERT(k_sp != Null<KVirtualAddress>);
/* Ensure that the stack pointers are aligned. */
k_sp = util::AlignDown(GetInteger(k_sp), 16);
u_sp = util::AlignDown(GetInteger(u_sp), 16);
/* Determine LR and SP. */
if (is_user) {
/* Usermode thread. */
m_lr = reinterpret_cast<uintptr_t>(::ams::kern::arch::arm64::UserModeThreadStarter);
m_sp = SetupStackForUserModeThreadStarter(u_pc, k_sp, u_sp, arg, is_64_bit);
} else {
/* Kernel thread. */
MESOSPHERE_ASSERT(is_64_bit);
if (is_main) {
/* Main thread. */
m_lr = GetInteger(u_pc);
m_sp = GetInteger(k_sp);
} else {
/* Generic Kernel thread. */
m_lr = reinterpret_cast<uintptr_t>(::ams::kern::arch::arm64::SupervisorModeThreadStarter);
m_sp = SetupStackForSupervisorModeThreadStarter(u_pc, k_sp, arg);
}
}
/* Clear callee-saved registers. */
for (size_t i = 0; i < util::size(m_callee_saved.registers); i++) {
m_callee_saved.registers[i] = 0;
}
/* Clear FPU state. */
m_fpcr = 0;
m_fpsr = 0;
for (size_t i = 0; i < util::size(m_callee_saved_fpu.fpu64.v); ++i) {
m_callee_saved_fpu.fpu64.v[i] = 0;
}
/* Lock the context, if we're a main thread. */
m_locked = is_main;
R_SUCCEED();
}
void KThreadContext::SetArguments(uintptr_t arg0, uintptr_t arg1) {
u64 *stack = reinterpret_cast<u64 *>(m_sp);
stack[0] = arg0;
stack[1] = arg1;
}
void KThreadContext::CloneFpuStatus() {
u64 pcr, psr;
cpu::InstructionMemoryBarrier();
KScopedInterruptDisable di;
if (IsFpuEnabled()) {
__asm__ __volatile__("mrs %[pcr], fpcr" : [pcr]"=r"(pcr) :: "memory");
__asm__ __volatile__("mrs %[psr], fpsr" : [psr]"=r"(psr) :: "memory");
} else {
pcr = GetCurrentThread().GetContext().GetFpcr();
psr = GetCurrentThread().GetContext().GetFpsr();
}
this->SetFpcr(pcr);
this->SetFpsr(psr);
}
void GetUserContext(ams::svc::ThreadContext *out, const KThread *thread) {
MESOSPHERE_ASSERT(KScheduler::IsSchedulerLockedByCurrentThread());
MESOSPHERE_ASSERT(thread->IsSuspended());
MESOSPHERE_ASSERT(thread->GetOwnerProcess() != nullptr);
/* Get the contexts. */
const KExceptionContext *e_ctx = GetExceptionContext(thread);
const KThreadContext *t_ctx = std::addressof(thread->GetContext());
if (thread->GetOwnerProcess()->Is64Bit()) {
/* Set special registers. */
out->fp = e_ctx->x[29];
out->lr = e_ctx->x[30];
out->sp = e_ctx->sp;
out->pc = e_ctx->pc;
out->pstate = e_ctx->psr & cpu::El0Aarch64PsrMask;
/* Get the thread's general purpose registers. */
if (thread->IsCallingSvc()) {
for (size_t i = 19; i < 29; ++i) {
out->r[i] = e_ctx->x[i];
}
if (e_ctx->write == 0) {
out->pc -= sizeof(u32);
}
} else {
for (size_t i = 0; i < 29; ++i) {
out->r[i] = e_ctx->x[i];
}
}
/* Copy tpidr. */
out->tpidr = e_ctx->tpidr;
/* Copy fpu registers. */
static_assert(util::size(ams::svc::ThreadContext{}.v) == KThreadContext::NumFpuRegisters);
static_assert(KThreadContext::NumCallerSavedFpuRegisters == KThreadContext::NumCalleeSavedFpuRegisters * 3);
static_assert(KThreadContext::NumFpuRegisters == KThreadContext::NumCallerSavedFpuRegisters + KThreadContext::NumCalleeSavedFpuRegisters);
const auto &caller_save_fpu = thread->GetCallerSaveFpuRegisters().fpu64;
const auto &callee_save_fpu = t_ctx->GetCalleeSaveFpuRegisters().fpu64;
if (!thread->IsCallingSvc() || thread->IsInUsermodeExceptionHandler()) {
KThreadContext::GetFpuRegisters(out->v, caller_save_fpu, callee_save_fpu);
} else {
for (size_t i = 0; i < KThreadContext::NumCalleeSavedFpuRegisters; ++i) {
out->v[(KThreadContext::NumCallerSavedFpuRegisters / 3) + i] = caller_save_fpu.v[i];
}
}
} else {
/* Set special registers. */
out->pc = static_cast<u32>(e_ctx->pc);
out->pstate = e_ctx->psr & cpu::El0Aarch32PsrMask;
/* Get the thread's general purpose registers. */
for (size_t i = 0; i < 15; ++i) {
out->r[i] = static_cast<u32>(e_ctx->x[i]);
}
/* Adjust PC, if the thread is calling svc. */
if (thread->IsCallingSvc()) {
if (e_ctx->write == 0) {
/* Adjust by 2 if thumb mode, 4 if arm mode. */
out->pc -= ((e_ctx->psr & 0x20) == 0) ? sizeof(u32) : sizeof(u16);
}
}
/* Copy tpidr. */
out->tpidr = static_cast<u32>(e_ctx->tpidr);
/* Copy fpu registers. */
static_assert(util::size(ams::svc::ThreadContext{}.v) == KThreadContext::NumFpuRegisters);
static_assert(KThreadContext::NumCallerSavedFpuRegisters == KThreadContext::NumCalleeSavedFpuRegisters * 3);
static_assert(KThreadContext::NumFpuRegisters == KThreadContext::NumCallerSavedFpuRegisters + KThreadContext::NumCalleeSavedFpuRegisters);
const auto &caller_save_fpu = thread->GetCallerSaveFpuRegisters().fpu32;
const auto &callee_save_fpu = t_ctx->GetCalleeSaveFpuRegisters().fpu32;
if (!thread->IsCallingSvc() || thread->IsInUsermodeExceptionHandler()) {
KThreadContext::GetFpuRegisters(out->v, caller_save_fpu, callee_save_fpu);
} else {
for (size_t i = 0; i < KThreadContext::NumCalleeSavedFpuRegisters / 2; ++i) {
out->v[((KThreadContext::NumCallerSavedFpuRegisters / 3) / 2) + i] = caller_save_fpu.v[i];
}
}
}
/* Copy fpcr/fpsr. */
out->fpcr = t_ctx->GetFpcr();
out->fpsr = t_ctx->GetFpsr();
}
void KThreadContext::OnThreadTerminating(const KThread *thread) {
MESOSPHERE_UNUSED(thread);
/* ... */
}
}

89
Source/Atmosphere/libraries/libmesosphere/source/arch/arm64/kern_panic_asm.s Normal file
View File

@@ -0,0 +1,89 @@
/*
* Copyright (c) Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <mesosphere/kern_build_config.hpp>
#include <mesosphere/kern_select_assembly_offsets.h>
#if defined(MESOSPHERE_ENABLE_PANIC_REGISTER_DUMP)
#define MESOSPHERE_GENERATE_PANIC_EXCEPTION_CONTEXT \
/* Save x0/x1 to stack. */ \
stp x0, x1, [sp, #-16]!; \
\
/* Get the exception context for the core. */ \
adr x0, _ZN3ams4kern26g_panic_exception_contextsE; \
mrs x1, mpidr_el1; \
and x1, x1, #0xFF; \
lsl x1, x1, #0x8; \
add x0, x0, x1; \
lsr x1, x1, #0x3; \
add x0, x0, x1; \
\
/* Save x0/x1/sp to the context. */ \
ldr x1, [sp, #(8 * 0)]; \
str x1, [x0, #(EXCEPTION_CONTEXT_X0)]; \
ldr x1, [sp, #(8 * 1)]; \
str x1, [x0, #(EXCEPTION_CONTEXT_X1)]; \
\
/* Save all other registers to the context. */ \
stp x2, x3, [x0, #(EXCEPTION_CONTEXT_X2_X3)]; \
stp x4, x5, [x0, #(EXCEPTION_CONTEXT_X4_X5)]; \
stp x6, x7, [x0, #(EXCEPTION_CONTEXT_X6_X7)]; \
stp x8, x9, [x0, #(EXCEPTION_CONTEXT_X8_X9)]; \
stp x10, x11, [x0, #(EXCEPTION_CONTEXT_X10_X11)]; \
stp x12, x13, [x0, #(EXCEPTION_CONTEXT_X12_X13)]; \
stp x14, x15, [x0, #(EXCEPTION_CONTEXT_X14_X15)]; \
stp x16, x17, [x0, #(EXCEPTION_CONTEXT_X16_X17)]; \
stp x18, x19, [x0, #(EXCEPTION_CONTEXT_X18_X19)]; \
stp x20, x21, [x0, #(EXCEPTION_CONTEXT_X20_X21)]; \
stp x22, x23, [x0, #(EXCEPTION_CONTEXT_X22_X23)]; \
stp x24, x25, [x0, #(EXCEPTION_CONTEXT_X24_X25)]; \
stp x26, x27, [x0, #(EXCEPTION_CONTEXT_X26_X27)]; \
stp x28, x29, [x0, #(EXCEPTION_CONTEXT_X28_X29)]; \
\
add x1, sp, #16; \
stp x30, x1, [x0, #(EXCEPTION_CONTEXT_X30_SP)]; \
\
/* Restore x0/x1. */ \
ldp x0, x1, [sp], #16;
#else
#define MESOSPHERE_GENERATE_PANIC_EXCEPTION_CONTEXT
#endif
/* ams::kern::Panic(const char *file, int line, const char *format, ...) */
.section .text._ZN3ams4kern5PanicEPKciS2_z, "ax", %progbits
.global _ZN3ams4kern5PanicEPKciS2_z
.type _ZN3ams4kern5PanicEPKciS2_z, %function
_ZN3ams4kern5PanicEPKciS2_z:
/* Generate the exception context. */
MESOSPHERE_GENERATE_PANIC_EXCEPTION_CONTEXT
/* Jump to the architecturally-common implementation function. */
b _ZN3ams4kern9PanicImplEPKciS2_z
/* ams::kern::Panic() */
.section .text._ZN3ams4kern5PanicEv, "ax", %progbits
.global _ZN3ams4kern5PanicEv
.type _ZN3ams4kern5PanicEv, %function
_ZN3ams4kern5PanicEv:
/* Generate the exception context. */
MESOSPHERE_GENERATE_PANIC_EXCEPTION_CONTEXT
/* Jump to the architecturally-common implementation function. */
b _ZN3ams4kern9PanicImplEv

800
Source/Atmosphere/libraries/libmesosphere/source/arch/arm64/kern_userspace_memory_access_asm.s Normal file
View File

@@ -0,0 +1,800 @@
/*
* Copyright (c) Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
/* ams::kern::arch::arm64::UserspaceAccessFunctionAreaBegin() */
.section .text._ZN3ams4kern4arch5arm6432UserspaceAccessFunctionAreaBeginEv, "ax", %progbits
.global _ZN3ams4kern4arch5arm6432UserspaceAccessFunctionAreaBeginEv
.type _ZN3ams4kern4arch5arm6432UserspaceAccessFunctionAreaBeginEv, %function
.balign 0x10
_ZN3ams4kern4arch5arm6432UserspaceAccessFunctionAreaBeginEv:
/* NOTE: This is not a real function, and only exists as a label for safety. */
/* ================ All Userspace Access Functions after this line. ================ */
/* ams::kern::arch::arm64::UserspaceAccess::Impl::CopyMemoryFromUser(void *dst, const void *src, size_t size) */
.section .text._ZN3ams4kern4arch5arm6415UserspaceAccess4Impl18CopyMemoryFromUserEPvPKvm, "ax", %progbits
.global _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl18CopyMemoryFromUserEPvPKvm
.type _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl18CopyMemoryFromUserEPvPKvm, %function
.balign 0x10
_ZN3ams4kern4arch5arm6415UserspaceAccess4Impl18CopyMemoryFromUserEPvPKvm:
/* Check if there's anything to copy. */
cmp x2, #0
b.eq 2f
/* Keep track of the last address. */
add x3, x1, x2
1: /* We're copying memory byte-by-byte. */
ldtrb w2, [x1]
strb w2, [x0], #1
add x1, x1, #1
cmp x1, x3
b.ne 1b
2: /* We're done. */
mov x0, #1
ret
/* ams::kern::arch::arm64::UserspaceAccess::Impl::CopyMemoryFromUserAligned32Bit(void *dst, const void *src, size_t size) */
.section .text._ZN3ams4kern4arch5arm6415UserspaceAccess4Impl30CopyMemoryFromUserAligned32BitEPvPKvm, "ax", %progbits
.global _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl30CopyMemoryFromUserAligned32BitEPvPKvm
.type _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl30CopyMemoryFromUserAligned32BitEPvPKvm, %function
.balign 0x10
_ZN3ams4kern4arch5arm6415UserspaceAccess4Impl30CopyMemoryFromUserAligned32BitEPvPKvm:
/* Check if there are 0x40 bytes to copy */
cmp x2, #0x3F
b.ls 1f
ldtr x4, [x1, #0x00]
ldtr x5, [x1, #0x08]
ldtr x6, [x1, #0x10]
ldtr x7, [x1, #0x18]
ldtr x8, [x1, #0x20]
ldtr x9, [x1, #0x28]
ldtr x10, [x1, #0x30]
ldtr x11, [x1, #0x38]
stp x4, x5, [x0, #0x00]
stp x6, x7, [x0, #0x10]
stp x8, x9, [x0, #0x20]
stp x10, x11, [x0, #0x30]
add x0, x0, #0x40
add x1, x1, #0x40
sub x2, x2, #0x40
b _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl30CopyMemoryFromUserAligned32BitEPvPKvm
1: /* We have less than 0x40 bytes to copy. */
cmp x2, #0
b.eq 2f
ldtr w4, [x1]
str w4, [x0], #4
add x1, x1, #4
sub x2, x2, #4
b 1b
2: /* We're done. */
mov x0, #1
ret
/* ams::kern::arch::arm64::UserspaceAccess::Impl::CopyMemoryFromUserAligned64Bit(void *dst, const void *src, size_t size) */
.section .text._ZN3ams4kern4arch5arm6415UserspaceAccess4Impl30CopyMemoryFromUserAligned64BitEPvPKvm, "ax", %progbits
.global _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl30CopyMemoryFromUserAligned64BitEPvPKvm
.type _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl30CopyMemoryFromUserAligned64BitEPvPKvm, %function
.balign 0x10
_ZN3ams4kern4arch5arm6415UserspaceAccess4Impl30CopyMemoryFromUserAligned64BitEPvPKvm:
/* Check if there are 0x40 bytes to copy */
cmp x2, #0x3F
b.ls 1f
ldtr x4, [x1, #0x00]
ldtr x5, [x1, #0x08]
ldtr x6, [x1, #0x10]
ldtr x7, [x1, #0x18]
ldtr x8, [x1, #0x20]
ldtr x9, [x1, #0x28]
ldtr x10, [x1, #0x30]
ldtr x11, [x1, #0x38]
stp x4, x5, [x0, #0x00]
stp x6, x7, [x0, #0x10]
stp x8, x9, [x0, #0x20]
stp x10, x11, [x0, #0x30]
add x0, x0, #0x40
add x1, x1, #0x40
sub x2, x2, #0x40
b _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl30CopyMemoryFromUserAligned64BitEPvPKvm
1: /* We have less than 0x40 bytes to copy. */
cmp x2, #0
b.eq 2f
ldtr x4, [x1]
str x4, [x0], #8
add x1, x1, #8
sub x2, x2, #8
b 1b
2: /* We're done. */
mov x0, #1
ret
/* ams::kern::arch::arm64::UserspaceAccess::Impl::CopyMemoryFromUserSize64Bit(void *dst, const void *src) */
.section .text._ZN3ams4kern4arch5arm6415UserspaceAccess4Impl27CopyMemoryFromUserSize64BitEPvPKv, "ax", %progbits
.global _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl27CopyMemoryFromUserSize64BitEPvPKv
.type _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl27CopyMemoryFromUserSize64BitEPvPKv, %function
.balign 0x10
_ZN3ams4kern4arch5arm6415UserspaceAccess4Impl27CopyMemoryFromUserSize64BitEPvPKv:
/* Just load and store a u64. */
ldtr x2, [x1]
str x2, [x0]
/* We're done. */
mov x0, #1
ret
/* ams::kern::arch::arm64::UserspaceAccess::Impl::CopyMemoryFromUserSize32Bit(void *dst, const void *src) */
.section .text._ZN3ams4kern4arch5arm6415UserspaceAccess4Impl27CopyMemoryFromUserSize32BitEPvPKv, "ax", %progbits
.global _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl27CopyMemoryFromUserSize32BitEPvPKv
.type _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl27CopyMemoryFromUserSize32BitEPvPKv, %function
.balign 0x10
_ZN3ams4kern4arch5arm6415UserspaceAccess4Impl27CopyMemoryFromUserSize32BitEPvPKv:
/* Just load and store a u32. */
ldtr w2, [x1]
str w2, [x0]
/* We're done. */
mov x0, #1
ret
/* ams::kern::arch::arm64::UserspaceAccess::Impl::CopyMemoryFromUserSize32BitWithSupervisorAccess(void *dst, const void *src) */
.section .text._ZN3ams4kern4arch5arm6415UserspaceAccess4Impl47CopyMemoryFromUserSize32BitWithSupervisorAccessEPvPKv, "ax", %progbits
.global _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl47CopyMemoryFromUserSize32BitWithSupervisorAccessEPvPKv
.type _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl47CopyMemoryFromUserSize32BitWithSupervisorAccessEPvPKv, %function
.balign 0x10
_ZN3ams4kern4arch5arm6415UserspaceAccess4Impl47CopyMemoryFromUserSize32BitWithSupervisorAccessEPvPKv:
/* Just load and store a u32. */
/* NOTE: This is done with supervisor access permissions. */
ldr w2, [x1]
str w2, [x0]
/* We're done. */
mov x0, #1
ret
/* ams::kern::arch::arm64::UserspaceAccess::Impl::CopyStringFromUser(void *dst, const void *src, size_t size) */
.section .text._ZN3ams4kern4arch5arm6415UserspaceAccess4Impl18CopyStringFromUserEPvPKvm, "ax", %progbits
.global _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl18CopyStringFromUserEPvPKvm
.type _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl18CopyStringFromUserEPvPKvm, %function
.balign 0x10
_ZN3ams4kern4arch5arm6415UserspaceAccess4Impl18CopyStringFromUserEPvPKvm:
/* Check if there's anything to copy. */
cmp x2, #0
b.eq 3f
/* Keep track of the start address and last address. */
mov x4, x1
add x3, x1, x2
1: /* We're copying memory byte-by-byte. */
ldtrb w2, [x1]
strb w2, [x0], #1
add x1, x1, #1
/* If we read a null terminator, we're done. */
cmp w2, #0
b.eq 2f
/* Check if we're done. */
cmp x1, x3
b.ne 1b
2: /* We're done, and we copied some amount of data from the string. */
sub x0, x1, x4
ret
3: /* We're done, and there was no string data. */
mov x0, #0
ret
/* ams::kern::arch::arm64::UserspaceAccess::Impl::CopyMemoryToUser(void *dst, const void *src, size_t size) */
.section .text._ZN3ams4kern4arch5arm6415UserspaceAccess4Impl16CopyMemoryToUserEPvPKvm, "ax", %progbits
.global _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl16CopyMemoryToUserEPvPKvm
.type _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl16CopyMemoryToUserEPvPKvm, %function
.balign 0x10
_ZN3ams4kern4arch5arm6415UserspaceAccess4Impl16CopyMemoryToUserEPvPKvm:
/* Check if there's anything to copy. */
cmp x2, #0
b.eq 2f
/* Keep track of the last address. */
add x3, x1, x2
1: /* We're copying memory byte-by-byte. */
ldrb w2, [x1], #1
sttrb w2, [x0]
add x0, x0, #1
cmp x1, x3
b.ne 1b
2: /* We're done. */
mov x0, #1
ret
/* ams::kern::arch::arm64::UserspaceAccess::Impl::CopyMemoryToUserAligned32Bit(void *dst, const void *src, size_t size) */
.section .text._ZN3ams4kern4arch5arm6415UserspaceAccess4Impl28CopyMemoryToUserAligned32BitEPvPKvm, "ax", %progbits
.global _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl28CopyMemoryToUserAligned32BitEPvPKvm
.type _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl28CopyMemoryToUserAligned32BitEPvPKvm, %function
.balign 0x10
_ZN3ams4kern4arch5arm6415UserspaceAccess4Impl28CopyMemoryToUserAligned32BitEPvPKvm:
/* Check if there are 0x40 bytes to copy */
cmp x2, #0x3F
b.ls 1f
ldp x4, x5, [x1, #0x00]
ldp x6, x7, [x1, #0x10]
ldp x8, x9, [x1, #0x20]
ldp x10, x11, [x1, #0x30]
sttr x4, [x0, #0x00]
sttr x5, [x0, #0x08]
sttr x6, [x0, #0x10]
sttr x7, [x0, #0x18]
sttr x8, [x0, #0x20]
sttr x9, [x0, #0x28]
sttr x10, [x0, #0x30]
sttr x11, [x0, #0x38]
add x0, x0, #0x40
add x1, x1, #0x40
sub x2, x2, #0x40
b _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl28CopyMemoryToUserAligned32BitEPvPKvm
1: /* We have less than 0x40 bytes to copy. */
cmp x2, #0
b.eq 2f
ldr w4, [x1], #4
sttr w4, [x0]
add x0, x0, #4
sub x2, x2, #4
b 1b
2: /* We're done. */
mov x0, #1
ret
/* ams::kern::arch::arm64::UserspaceAccess::Impl::CopyMemoryToUserAligned64Bit(void *dst, const void *src, size_t size) */
.section .text._ZN3ams4kern4arch5arm6415UserspaceAccess4Impl28CopyMemoryToUserAligned64BitEPvPKvm, "ax", %progbits
.global _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl28CopyMemoryToUserAligned64BitEPvPKvm
.type _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl28CopyMemoryToUserAligned64BitEPvPKvm, %function
.balign 0x10
_ZN3ams4kern4arch5arm6415UserspaceAccess4Impl28CopyMemoryToUserAligned64BitEPvPKvm:
/* Check if there are 0x40 bytes to copy */
cmp x2, #0x3F
b.ls 1f
ldp x4, x5, [x1, #0x00]
ldp x6, x7, [x1, #0x10]
ldp x8, x9, [x1, #0x20]
ldp x10, x11, [x1, #0x30]
sttr x4, [x0, #0x00]
sttr x5, [x0, #0x08]
sttr x6, [x0, #0x10]
sttr x7, [x0, #0x18]
sttr x8, [x0, #0x20]
sttr x9, [x0, #0x28]
sttr x10, [x0, #0x30]
sttr x11, [x0, #0x38]
add x0, x0, #0x40
add x1, x1, #0x40
sub x2, x2, #0x40
b _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl28CopyMemoryToUserAligned64BitEPvPKvm
1: /* We have less than 0x40 bytes to copy. */
cmp x2, #0
b.eq 2f
ldr x4, [x1], #8
sttr x4, [x0]
add x0, x0, #8
sub x2, x2, #8
b 1b
2: /* We're done. */
mov x0, #1
ret
/* ams::kern::arch::arm64::UserspaceAccess::Impl::CopyMemoryToUserSize32Bit(void *dst, const void *src) */
.section .text._ZN3ams4kern4arch5arm6415UserspaceAccess4Impl25CopyMemoryToUserSize32BitEPvPKv, "ax", %progbits
.global _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl25CopyMemoryToUserSize32BitEPvPKv
.type _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl25CopyMemoryToUserSize32BitEPvPKv, %function
.balign 0x10
_ZN3ams4kern4arch5arm6415UserspaceAccess4Impl25CopyMemoryToUserSize32BitEPvPKv:
/* Just load and store a u32. */
ldr w2, [x1]
sttr w2, [x0]
/* We're done. */
mov x0, #1
ret
/* ams::kern::arch::arm64::UserspaceAccess::Impl::CopyStringToUser(void *dst, const void *src, size_t size) */
.section .text._ZN3ams4kern4arch5arm6415UserspaceAccess4Impl16CopyStringToUserEPvPKvm, "ax", %progbits
.global _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl16CopyStringToUserEPvPKvm
.type _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl16CopyStringToUserEPvPKvm, %function
.balign 0x10
_ZN3ams4kern4arch5arm6415UserspaceAccess4Impl16CopyStringToUserEPvPKvm:
/* Check if there's anything to copy. */
cmp x2, #0
b.eq 3f
/* Keep track of the start address and last address. */
mov x4, x1
add x3, x1, x2
1: /* We're copying memory byte-by-byte. */
ldrb w2, [x1], #1
sttrb w2, [x0]
add x0, x0, #1
/* If we read a null terminator, we're done. */
cmp w2, #0
b.eq 2f
/* Check if we're done. */
cmp x1, x3
b.ne 1b
2: /* We're done, and we copied some amount of data from the string. */
sub x0, x1, x4
ret
3: /* We're done, and there was no string data. */
mov x0, #0
ret
/* ams::kern::arch::arm64::UserspaceAccess::Impl::UpdateLockAtomic(u32 *out, u32 *address, u32 if_zero, u32 new_orr_mask) */
.section .text._ZN3ams4kern4arch5arm6415UserspaceAccess4Impl16UpdateLockAtomicEPjS5_jj, "ax", %progbits
.global _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl16UpdateLockAtomicEPjS5_jj
.type _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl16UpdateLockAtomicEPjS5_jj, %function
.balign 0x10
_ZN3ams4kern4arch5arm6415UserspaceAccess4Impl16UpdateLockAtomicEPjS5_jj:
/* Load the value from the address. */
ldaxr w4, [x1]
/* Orr in the new mask. */
orr w5, w4, w3
/* If the value is zero, use the if_zero value, otherwise use the newly orr'd value. */
cmp w4, wzr
csel w5, w2, w5, eq
/* Try to store. */
stlxr w6, w5, [x1]
/* If we failed to store, try again. */
cbnz w6, _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl16UpdateLockAtomicEPjS5_jj
/* We're done. */
str w4, [x0]
mov x0, #1
ret
/* ams::kern::arch::arm64::UserspaceAccess::Impl::UpdateIfEqualAtomic(s32 *out, s32 *address, s32 compare_value, s32 new_value) */
.section .text._ZN3ams4kern4arch5arm6415UserspaceAccess4Impl19UpdateIfEqualAtomicEPiS5_ii, "ax", %progbits
.global _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl19UpdateIfEqualAtomicEPiS5_ii
.type _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl19UpdateIfEqualAtomicEPiS5_ii, %function
.balign 0x10
_ZN3ams4kern4arch5arm6415UserspaceAccess4Impl19UpdateIfEqualAtomicEPiS5_ii:
/* Load the value from the address. */
ldaxr w4, [x1]
/* Compare it to the desired one. */
cmp w4, w2
/* If equal, we want to try to write the new value. */
b.eq 1f
/* Otherwise, clear our exclusive hold and finish. */
clrex
b 2f
1: /* Try to store. */
stlxr w5, w3, [x1]
/* If we failed to store, try again. */
cbnz w5, _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl19UpdateIfEqualAtomicEPiS5_ii
2: /* We're done. */
str w4, [x0]
mov x0, #1
ret
/* ams::kern::arch::arm64::UserspaceAccess::Impl::DecrementIfLessThanAtomic(s32 *out, s32 *address, s32 compare) */
.section .text._ZN3ams4kern4arch5arm6415UserspaceAccess4Impl25DecrementIfLessThanAtomicEPiS5_i, "ax", %progbits
.global _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl25DecrementIfLessThanAtomicEPiS5_i
.type _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl25DecrementIfLessThanAtomicEPiS5_i, %function
.balign 0x10
_ZN3ams4kern4arch5arm6415UserspaceAccess4Impl25DecrementIfLessThanAtomicEPiS5_i:
/* Load the value from the address. */
ldaxr w3, [x1]
/* Compare it to the desired one. */
cmp w3, w2
/* If less than, we want to try to decrement. */
b.lt 1f
/* Otherwise, clear our exclusive hold and finish. */
clrex
b 2f
1: /* Decrement and try to store. */
sub w4, w3, #1
stlxr w5, w4, [x1]
/* If we failed to store, try again. */
cbnz w5, _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl25DecrementIfLessThanAtomicEPiS5_i
2: /* We're done. */
str w3, [x0]
mov x0, #1
ret
/* ams::kern::arch::arm64::UserspaceAccess::Impl::StoreDataCache(uintptr_t start, uintptr_t end) */
.section .text._ZN3ams4kern4arch5arm6415UserspaceAccess4Impl14StoreDataCacheEmm, "ax", %progbits
.global _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl14StoreDataCacheEmm
.type _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl14StoreDataCacheEmm, %function
.balign 0x10
_ZN3ams4kern4arch5arm6415UserspaceAccess4Impl14StoreDataCacheEmm:
/* Check if we have any work to do. */
cmp x1, x0
b.eq 2f
1: /* Loop, storing each cache line. */
dc cvac, x0
add x0, x0, #0x40
cmp x1, x0
b.ne 1b
2: /* We're done! */
mov x0, #1
ret
/* ams::kern::arch::arm64::UserspaceAccess::Impl::FlushDataCache(uintptr_t start, uintptr_t end) */
.section .text._ZN3ams4kern4arch5arm6415UserspaceAccess4Impl14FlushDataCacheEmm, "ax", %progbits
.global _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl14FlushDataCacheEmm
.type _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl14FlushDataCacheEmm, %function
.balign 0x10
_ZN3ams4kern4arch5arm6415UserspaceAccess4Impl14FlushDataCacheEmm:
/* Check if we have any work to do. */
cmp x1, x0
b.eq 2f
1: /* Loop, flushing each cache line. */
dc civac, x0
add x0, x0, #0x40
cmp x1, x0
b.ne 1b
2: /* We're done! */
mov x0, #1
ret
/* ams::kern::arch::arm64::UserspaceAccess::Impl::InvalidateDataCache(uintptr_t start, uintptr_t end) */
.section .text._ZN3ams4kern4arch5arm6415UserspaceAccess4Impl19InvalidateDataCacheEmm, "ax", %progbits
.global _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl19InvalidateDataCacheEmm
.type _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl19InvalidateDataCacheEmm, %function
.balign 0x10
_ZN3ams4kern4arch5arm6415UserspaceAccess4Impl19InvalidateDataCacheEmm:
/* Check if we have any work to do. */
cmp x1, x0
b.eq 2f
1: /* Loop, invalidating each cache line. */
dc ivac, x0
add x0, x0, #0x40
cmp x1, x0
b.ne 1b
2: /* We're done! */
mov x0, #1
ret
/* ams::kern::arch::arm64::UserspaceAccess::Impl::ReadIoMemory32Bit(void *dst, const void *src, size_t size) */
.section .text._ZN3ams4kern4arch5arm6415UserspaceAccess4Impl17ReadIoMemory32BitEPvPKvm, "ax", %progbits
.global _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl17ReadIoMemory32BitEPvPKvm
.type _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl17ReadIoMemory32BitEPvPKvm, %function
.balign 0x10
_ZN3ams4kern4arch5arm6415UserspaceAccess4Impl17ReadIoMemory32BitEPvPKvm:
/* Check if we have any work to do. */
cmp x2, #0
b.eq 3f
/* Save variables in temporary registers. */
mov x4, x0
mov x5, x1
mov x6, x2
add x7, x5, x6
/* Save our return address. */
mov x8, x30
/* Prepare return address for read failure. */
adr x10, 4f
1: /* Set our return address so that on read failure we continue as though we read -1. */
mov x30, x10
/* Read the word from io. */
ldtr w9, [x5]
dsb sy
nop
2: /* Restore our return address. */
mov x30, x8
/* Write the value we read. */
sttr w9, [x4]
/* Advance. */
add x4, x4, #4
add x5, x5, #4
cmp x5, x7
b.ne 1b
3: /* We're done! */
mov x0, #1
ret
4: /* We failed to read a value, so continue as though we read -1. */
mov w9, #0xFFFFFFFF
b 2b
/* ams::kern::arch::arm64::UserspaceAccess::Impl::ReadIoMemory16Bit(void *dst, const void *src, size_t size) */
.section .text._ZN3ams4kern4arch5arm6415UserspaceAccess4Impl17ReadIoMemory16BitEPvPKvm, "ax", %progbits
.global _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl17ReadIoMemory16BitEPvPKvm
.type _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl17ReadIoMemory16BitEPvPKvm, %function
.balign 0x10
_ZN3ams4kern4arch5arm6415UserspaceAccess4Impl17ReadIoMemory16BitEPvPKvm:
/* Check if we have any work to do. */
cmp x2, #0
b.eq 3f
/* Save variables in temporary registers. */
mov x4, x0
mov x5, x1
mov x6, x2
add x7, x5, x6
/* Save our return address. */
mov x8, x30
/* Prepare return address for read failure. */
adr x10, 4f
1: /* Set our return address so that on read failure we continue as though we read -1. */
mov x30, x10
/* Read the word from io. */
ldtrh w9, [x5]
dsb sy
nop
2: /* Restore our return address. */
mov x30, x8
/* Write the value we read. */
sttrh w9, [x4]
/* Advance. */
add x4, x4, #2
add x5, x5, #2
cmp x5, x7
b.ne 1b
3: /* We're done! */
mov x0, #1
ret
4: /* We failed to read a value, so continue as though we read -1. */
mov w9, #0xFFFFFFFF
b 2b
/* ams::kern::arch::arm64::UserspaceAccess::Impl::ReadIoMemory8Bit(void *dst, const void *src, size_t size) */
.section .text._ZN3ams4kern4arch5arm6415UserspaceAccess4Impl16ReadIoMemory8BitEPvPKvm, "ax", %progbits
.global _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl16ReadIoMemory8BitEPvPKvm
.type _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl16ReadIoMemory8BitEPvPKvm, %function
.balign 0x10
_ZN3ams4kern4arch5arm6415UserspaceAccess4Impl16ReadIoMemory8BitEPvPKvm:
/* Check if we have any work to do. */
cmp x2, #0
b.eq 3f
/* Save variables in temporary registers. */
mov x4, x0
mov x5, x1
mov x6, x2
add x7, x5, x6
/* Save our return address. */
mov x8, x30
/* Prepare return address for read failure. */
adr x10, 4f
1: /* Set our return address so that on read failure we continue as though we read -1. */
mov x30, x10
/* Read the word from io. */
ldtrb w9, [x5]
dsb sy
nop
2: /* Restore our return address. */
mov x30, x8
/* Write the value we read. */
sttrb w9, [x4]
/* Advance. */
add x4, x4, #1
add x5, x5, #1
cmp x5, x7
b.ne 1b
3: /* We're done! */
mov x0, #1
ret
4: /* We failed to read a value, so continue as though we read -1. */
mov w9, #0xFFFFFFFF
b 2b
/* ams::kern::arch::arm64::UserspaceAccess::Impl::WriteIoMemory32Bit(void *dst, const void *src, size_t size) */
.section .text._ZN3ams4kern4arch5arm6415UserspaceAccess4Impl18WriteIoMemory32BitEPvPKvm, "ax", %progbits
.global _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl18WriteIoMemory32BitEPvPKvm
.type _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl18WriteIoMemory32BitEPvPKvm, %function
.balign 0x10
_ZN3ams4kern4arch5arm6415UserspaceAccess4Impl18WriteIoMemory32BitEPvPKvm:
/* Check if we have any work to do. */
cmp x2, #0
b.eq 3f
/* Save variables in temporary registers. */
mov x4, x0
mov x5, x1
mov x6, x2
add x7, x5, x6
/* Save our return address. */
mov x8, x30
/* Prepare return address for failure. */
adr x10, 2f
1: /* Read the word from normal memory. */
ldtr w9, [x5]
/* Set our return address so that on failure we continue. */
mov x30, x10
/* Write the word to io. */
sttr w9, [x5]
dsb sy
2: /* Continue. */
mov x30, x8
/* Advance. */
add x4, x4, #4
add x5, x5, #4
cmp x5, x7
b.ne 1b
3: /* We're done! */
mov x0, #1
ret
/* ams::kern::arch::arm64::UserspaceAccess::Impl::WriteIoMemory16Bit(void *dst, const void *src, size_t size) */
.section .text._ZN3ams4kern4arch5arm6415UserspaceAccess4Impl18WriteIoMemory16BitEPvPKvm, "ax", %progbits
.global _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl18WriteIoMemory16BitEPvPKvm
.type _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl18WriteIoMemory16BitEPvPKvm, %function
.balign 0x10
_ZN3ams4kern4arch5arm6415UserspaceAccess4Impl18WriteIoMemory16BitEPvPKvm:
/* Check if we have any work to do. */
cmp x2, #0
b.eq 3f
/* Save variables in temporary registers. */
mov x4, x0
mov x5, x1
mov x6, x2
add x7, x5, x6
/* Save our return address. */
mov x8, x30
/* Prepare return address for failure. */
adr x10, 2f
1: /* Read the word from normal memory. */
ldtrh w9, [x5]
/* Set our return address so that on failure we continue. */
mov x30, x10
/* Write the word to io. */
sttrh w9, [x5]
dsb sy
2: /* Continue. */
mov x30, x8
/* Advance. */
add x4, x4, #2
add x5, x5, #2
cmp x5, x7
b.ne 1b
3: /* We're done! */
mov x0, #1
ret
/* ams::kern::arch::arm64::UserspaceAccess::Impl::WriteIoMemory8Bit(void *dst, const void *src, size_t size) */
.section .text._ZN3ams4kern4arch5arm6415UserspaceAccess4Impl17WriteIoMemory8BitEPvPKvm, "ax", %progbits
.global _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl17WriteIoMemory8BitEPvPKvm
.type _ZN3ams4kern4arch5arm6415UserspaceAccess4Impl17WriteIoMemory8BitEPvPKvm, %function
.balign 0x10
_ZN3ams4kern4arch5arm6415UserspaceAccess4Impl17WriteIoMemory8BitEPvPKvm:
/* Check if we have any work to do. */
cmp x2, #0
b.eq 3f
/* Save variables in temporary registers. */
mov x4, x0
mov x5, x1
mov x6, x2
add x7, x5, x6
/* Save our return address. */
mov x8, x30
/* Prepare return address for failure. */
adr x10, 2f
1: /* Read the word from normal memory. */
ldtrb w9, [x5]
/* Set our return address so that on failure we continue. */
mov x30, x10
/* Write the word to io. */
sttrb w9, [x5]
dsb sy
2: /* Continue. */
mov x30, x8
/* Advance. */
add x4, x4, #1
add x5, x5, #1
cmp x5, x7
b.ne 1b
3: /* We're done! */
mov x0, #1
ret
/* ================ All Userspace Access Functions before this line. ================ */
/* ams::kern::arch::arm64::UserspaceAccessFunctionAreaEnd() */
.section .text._ZN3ams4kern4arch5arm6430UserspaceAccessFunctionAreaEndEv, "ax", %progbits
.global _ZN3ams4kern4arch5arm6430UserspaceAccessFunctionAreaEndEv
.type _ZN3ams4kern4arch5arm6430UserspaceAccessFunctionAreaEndEv, %function
.balign 0x10
_ZN3ams4kern4arch5arm6430UserspaceAccessFunctionAreaEndEv:
/* NOTE: This is not a real function, and only exists as a label for safety. */

67
Source/Atmosphere/libraries/libmesosphere/source/arch/arm64/svc/kern_svc_address_arbiter_asm.s Normal file
View File

@@ -0,0 +1,67 @@
/*
* Copyright (c) Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
/* ams::kern::svc::CallWaitForAddress64From32() */
.section .text._ZN3ams4kern3svc26CallWaitForAddress64From32Ev, "ax", %progbits
.global _ZN3ams4kern3svc26CallWaitForAddress64From32Ev
.type _ZN3ams4kern3svc26CallWaitForAddress64From32Ev, %function
_ZN3ams4kern3svc26CallWaitForAddress64From32Ev:
/* Save LR + callee-save registers. */
str x30, [sp, #-16]!
stp x6, x7, [sp, #-16]!
/* Gather the arguments into correct registers. */
/* NOTE: This has to be manually implemented via asm, */
/* in order to avoid breaking ABI with pre-19.0.0. */
orr x2, x2, x5, lsl#32
orr x3, x3, x4, lsl#32
/* Invoke the svc handler. */
bl _ZN3ams4kern3svc22WaitForAddress64From32ENS_3svc7AddressENS2_15ArbitrationTypeEll
/* Clean up registers. */
mov x1, xzr
mov x2, xzr
mov x3, xzr
mov x4, xzr
mov x5, xzr
ldp x6, x7, [sp], #0x10
ldr x30, [sp], #0x10
ret
/* ams::kern::svc::CallWaitForAddress64() */
.section .text._ZN3ams4kern3svc20CallWaitForAddress64Ev, "ax", %progbits
.global _ZN3ams4kern3svc20CallWaitForAddress64Ev
.type _ZN3ams4kern3svc20CallWaitForAddress64Ev, %function
_ZN3ams4kern3svc20CallWaitForAddress64Ev:
/* Save LR + FP. */
stp x29, x30, [sp, #-16]!
/* Invoke the svc handler. */
bl _ZN3ams4kern3svc22WaitForAddress64From32ENS_3svc7AddressENS2_15ArbitrationTypeEll
/* Clean up registers. */
mov x1, xzr
mov x2, xzr
mov x3, xzr
mov x4, xzr
mov x5, xzr
mov x6, xzr
mov x7, xzr
ldp x29, x30, [sp], #0x10
ret

32
Source/Atmosphere/libraries/libmesosphere/source/arch/arm64/svc/kern_svc_call_secure_monitor_asm.s Normal file
View File

@@ -0,0 +1,32 @@
/*
* Copyright (c) Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
/* ams::kern::svc::CallCallSecureMonitor64From32() */
.section .text._ZN3ams4kern3svc29CallCallSecureMonitor64From32Ev, "ax", %progbits
.global _ZN3ams4kern3svc29CallCallSecureMonitor64From32Ev
.type _ZN3ams4kern3svc29CallCallSecureMonitor64From32Ev, %function
_ZN3ams4kern3svc29CallCallSecureMonitor64From32Ev:
/* Secure Monitor 64-from-32 ABI is not supported. */
mov x0, xzr
mov x1, xzr
mov x2, xzr
mov x3, xzr
mov x4, xzr
mov x5, xzr
mov x6, xzr
mov x7, xzr
ret

133
Source/Atmosphere/libraries/libmesosphere/source/arch/arm64/svc/kern_svc_exception_asm.s Normal file
View File

@@ -0,0 +1,133 @@
/*
* Copyright (c) Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <mesosphere/kern_select_assembly_macros.h>
/* ams::kern::svc::CallReturnFromException64(Result result) */
.section .text._ZN3ams4kern3svc25CallReturnFromException64Ev, "ax", %progbits
.global _ZN3ams4kern3svc25CallReturnFromException64Ev
.type _ZN3ams4kern3svc25CallReturnFromException64Ev, %function
_ZN3ams4kern3svc25CallReturnFromException64Ev:
/* Save registers the SVC entry handler didn't. */
stp x12, x13, [sp, #(EXCEPTION_CONTEXT_X12_X13)]
stp x14, x15, [sp, #(EXCEPTION_CONTEXT_X14_X15)]
stp x16, x17, [sp, #(EXCEPTION_CONTEXT_X16_X17)]
str x19, [sp, #(EXCEPTION_CONTEXT_X19)]
stp x20, x21, [sp, #(EXCEPTION_CONTEXT_X20_X21)]
stp x22, x23, [sp, #(EXCEPTION_CONTEXT_X22_X23)]
stp x24, x25, [sp, #(EXCEPTION_CONTEXT_X24_X25)]
stp x26, x27, [sp, #(EXCEPTION_CONTEXT_X26_X27)]
stp x28, x29, [sp, #(EXCEPTION_CONTEXT_X28_X29)]
/* Call ams::kern::arch::arm64::ReturnFromException(result). */
bl _ZN3ams4kern4arch5arm6419ReturnFromExceptionENS_6ResultE
0: /* We should never reach this point. */
b 0b
/* ams::kern::svc::CallReturnFromException64From32(Result result) */
.section .text._ZN3ams4kern3svc31CallReturnFromException64From32Ev, "ax", %progbits
.global _ZN3ams4kern3svc31CallReturnFromException64From32Ev
.type _ZN3ams4kern3svc31CallReturnFromException64From32Ev, %function
_ZN3ams4kern3svc31CallReturnFromException64From32Ev:
/* Save registers the SVC entry handler didn't. */
/* ... */
/* Call ams::kern::arch::arm64::ReturnFromException(result). */
bl _ZN3ams4kern4arch5arm6419ReturnFromExceptionENS_6ResultE
0: /* We should never reach this point. */
b 0b
/* ams::kern::svc::RestoreContext(uintptr_t sp) */
.section .text._ZN3ams4kern3svc14RestoreContextEm, "ax", %progbits
.global _ZN3ams4kern3svc14RestoreContextEm
.type _ZN3ams4kern3svc14RestoreContextEm, %function
_ZN3ams4kern3svc14RestoreContextEm:
/* Set the stack pointer, set daif. */
mov sp, x0
msr daifset, #2
0: /* We should handle DPC. */
/* Check the dpc flags. */
ldrb w8, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_DPC_FLAGS)]
cbz w8, 1f
/* We have DPC to do! */
/* Save registers and call ams::kern::KDpcManager::HandleDpc(). */
sub sp, sp, #0x40
stp x0, x1, [sp, #(8 * 0)]
stp x2, x3, [sp, #(8 * 2)]
stp x4, x5, [sp, #(8 * 4)]
stp x6, x7, [sp, #(8 * 6)]
bl _ZN3ams4kern11KDpcManager9HandleDpcEv
ldp x0, x1, [sp, #(8 * 0)]
ldp x2, x3, [sp, #(8 * 2)]
ldp x4, x5, [sp, #(8 * 4)]
ldp x6, x7, [sp, #(8 * 6)]
add sp, sp, #0x40
b 0b
1: /* We're done with DPC, and should return from the svc. */
/* Get our exception flags. */
ldrb w9, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
/* Clear in-svc, in-user-exception, and needs-fpu-restore flags. */
and w10, w9, #(~(THREAD_EXCEPTION_FLAG_IS_FPU_CONTEXT_RESTORE_NEEDED))
and w10, w10, #(~(THREAD_EXCEPTION_FLAG_IS_CALLING_SVC))
and w10, w10, #(~(THREAD_EXCEPTION_FLAG_IS_IN_USERMODE_EXCEPTION_HANDLER))
strb w10, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
/* If we don't need to restore the fpu, skip restoring it. */
tbz w9, #(THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_FPU_CONTEXT_RESTORE_NEEDED), 3f
/* Enable and restore the fpu. */
ENABLE_AND_RESTORE_FPU(x10, x8, x9, w8, w9, 2, 3)
/* Restore registers. */
ldp x30, x8, [sp, #(EXCEPTION_CONTEXT_X30_SP)]
ldp x9, x10, [sp, #(EXCEPTION_CONTEXT_PC_PSR)]
ldr x11, [sp, #(EXCEPTION_CONTEXT_TPIDR)]
#if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP)
/* Since we're returning from an exception, set SPSR.SS so that we advance an instruction if single-stepping. */
orr x10, x10, #(1 << 21)
#endif
msr sp_el0, x8
msr elr_el1, x9
msr spsr_el1, x10
msr tpidr_el0, x11
ldp x0, x1, [sp, #(EXCEPTION_CONTEXT_X0_X1)]
ldp x2, x3, [sp, #(EXCEPTION_CONTEXT_X2_X3)]
ldp x4, x5, [sp, #(EXCEPTION_CONTEXT_X4_X5)]
ldp x6, x7, [sp, #(EXCEPTION_CONTEXT_X6_X7)]
ldp x8, x9, [sp, #(EXCEPTION_CONTEXT_X8_X9)]
ldp x10, x11, [sp, #(EXCEPTION_CONTEXT_X10_X11)]
ldp x12, x13, [sp, #(EXCEPTION_CONTEXT_X12_X13)]
ldp x14, x15, [sp, #(EXCEPTION_CONTEXT_X14_X15)]
ldp x16, x17, [sp, #(EXCEPTION_CONTEXT_X16_X17)]
ldp x18, x19, [sp, #(EXCEPTION_CONTEXT_X18_X19)]
ldp x20, x21, [sp, #(EXCEPTION_CONTEXT_X20_X21)]
ldp x22, x23, [sp, #(EXCEPTION_CONTEXT_X22_X23)]
ldp x24, x25, [sp, #(EXCEPTION_CONTEXT_X24_X25)]
ldp x26, x27, [sp, #(EXCEPTION_CONTEXT_X26_X27)]
ldp x28, x29, [sp, #(EXCEPTION_CONTEXT_X28_X29)]
/* Return. */
add sp, sp, #(EXCEPTION_CONTEXT_SIZE)
ERET_WITH_SPECULATION_BARRIER

28
Source/Atmosphere/libraries/libmesosphere/source/arch/arm64/svc/kern_svc_handlers.cpp Normal file
View File

@@ -0,0 +1,28 @@
/*
* Copyright (c) Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <mesosphere.hpp>
namespace ams::kern::svc {
void TraceSvcEntry(const u64 *data) {
MESOSPHERE_KTRACE_SVC_ENTRY(GetCurrentThread().GetSvcId(), data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7]);
}
void TraceSvcExit(const u64 *data) {
MESOSPHERE_KTRACE_SVC_EXIT(GetCurrentThread().GetSvcId(), data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7]);
}
}

552
Source/Atmosphere/libraries/libmesosphere/source/arch/arm64/svc/kern_svc_handlers_asm.s Normal file
View File

@@ -0,0 +1,552 @@
/*
* Copyright (c) Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <mesosphere/kern_build_config.hpp>
#include <mesosphere/kern_select_assembly_macros.h>
/* ams::kern::arch::arm64::SvcHandler64() */
.section .text._ZN3ams4kern4arch5arm6412SvcHandler64Ev, "ax", %progbits
.global _ZN3ams4kern4arch5arm6412SvcHandler64Ev
.type _ZN3ams4kern4arch5arm6412SvcHandler64Ev, %function
_ZN3ams4kern4arch5arm6412SvcHandler64Ev:
/* Create a KExceptionContext for the exception. */
sub sp, sp, #(EXCEPTION_CONTEXT_SIZE)
/* Save registers needed for ReturnFromException */
stp x9, x10, [sp, #(EXCEPTION_CONTEXT_X9_X10)]
str x11, [sp, #(EXCEPTION_CONTEXT_X11)]
str x18, [sp, #(EXCEPTION_CONTEXT_X18)]
mrs x8, sp_el0
mrs x9, elr_el1
mrs x10, spsr_el1
mrs x11, tpidr_el0
ldr x18, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_CUR_THREAD)]
/* Save callee-saved registers. */
stp x19, x20, [sp, #(EXCEPTION_CONTEXT_X19_X20)]
stp x21, x22, [sp, #(EXCEPTION_CONTEXT_X21_X22)]
stp x23, x24, [sp, #(EXCEPTION_CONTEXT_X23_X24)]
stp x25, x26, [sp, #(EXCEPTION_CONTEXT_X25_X26)]
stp x27, x28, [sp, #(EXCEPTION_CONTEXT_X27_X28)]
/* Save miscellaneous registers. */
stp x0, x1, [sp, #(EXCEPTION_CONTEXT_X0_X1)]
stp x2, x3, [sp, #(EXCEPTION_CONTEXT_X2_X3)]
stp x4, x5, [sp, #(EXCEPTION_CONTEXT_X4_X5)]
stp x6, x7, [sp, #(EXCEPTION_CONTEXT_X6_X7)]
stp x29, x30, [sp, #(EXCEPTION_CONTEXT_X29_X30)]
stp x8, x9, [sp, #(EXCEPTION_CONTEXT_SP_PC)]
stp x10, x11, [sp, #(EXCEPTION_CONTEXT_PSR_TPIDR)]
/* Check if the SVC index is out of range. */
mrs x8, esr_el1
and x8, x8, #0xFF
cmp x8, #(AMS_KERN_NUM_SUPERVISOR_CALLS)
b.ge 3f
/* Check the specific SVC permission bit for allowal. */
mov x9, sp
add x9, x9, x8, lsr#3
ldrb w9, [x9, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_SVC_PERMISSION)]
and x10, x8, #0x7
lsr x10, x9, x10
tst x10, #1
b.eq 3f
/* Check if our disable count allows us to call SVCs. */
mrs x10, tpidrro_el0
add x10, x10, #(THREAD_LOCAL_REGION_DISABLE_COUNT)
ldtrh w10, [x10]
cbz w10, 1f
/* It might not, so check the stack params to see if we must not allow the SVC. */
ldrb w10, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_IS_PINNED)]
cbz w10, 3f
1: /* We can call the SVC. */
adr x10, _ZN3ams4kern3svc10SvcTable64E
ldr x11, [x10, x8, lsl#3]
cbz x11, 3f
/* Note that we're calling the SVC. */
ldrb w9, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
orr w9, w9, #(THREAD_EXCEPTION_FLAG_IS_CALLING_SVC)
strb w9, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
strb w8, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_CURRENT_SVC_ID)]
/* If we should, trace the svc entry. */
#if defined(MESOSPHERE_BUILD_FOR_TRACING)
sub sp, sp, #0x50
stp x0, x1, [sp, #(8 * 0)]
stp x2, x3, [sp, #(8 * 2)]
stp x4, x5, [sp, #(8 * 4)]
stp x6, x7, [sp, #(8 * 6)]
str x11, [sp, #(8 * 8)]
mov x0, sp
bl _ZN3ams4kern3svc13TraceSvcEntryEPKm
ldp x0, x1, [sp, #(8 * 0)]
ldp x2, x3, [sp, #(8 * 2)]
ldp x4, x5, [sp, #(8 * 4)]
ldp x6, x7, [sp, #(8 * 6)]
ldr x11, [sp, #(8 * 8)]
add sp, sp, #0x50
#endif
/* Invoke the SVC handler. */
msr daifclr, #2
blr x11
msr daifset, #2
2: /* We completed the SVC, and we should handle DPC. */
/* Check the dpc flags. */
ldrb w8, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_DPC_FLAGS)]
cbz w8, 5f
/* We have DPC to do! */
/* Save registers and call ams::kern::KDpcManager::HandleDpc(). */
sub sp, sp, #0x40
stp x0, x1, [sp, #(8 * 0)]
stp x2, x3, [sp, #(8 * 2)]
stp x4, x5, [sp, #(8 * 4)]
stp x6, x7, [sp, #(8 * 6)]
bl _ZN3ams4kern11KDpcManager9HandleDpcEv
ldp x0, x1, [sp, #(8 * 0)]
ldp x2, x3, [sp, #(8 * 2)]
ldp x4, x5, [sp, #(8 * 4)]
ldp x6, x7, [sp, #(8 * 6)]
add sp, sp, #0x40
b 2b
3: /* Invalid SVC. */
/* Setup the context to call into HandleException. */
stp x0, x1, [sp, #(EXCEPTION_CONTEXT_X0_X1)]
stp x2, x3, [sp, #(EXCEPTION_CONTEXT_X2_X3)]
stp x4, x5, [sp, #(EXCEPTION_CONTEXT_X4_X5)]
stp x6, x7, [sp, #(EXCEPTION_CONTEXT_X6_X7)]
stp xzr, xzr, [sp, #(EXCEPTION_CONTEXT_X8_X9)]
stp xzr, xzr, [sp, #(EXCEPTION_CONTEXT_X10_X11)]
stp xzr, xzr, [sp, #(EXCEPTION_CONTEXT_X12_X13)]
stp xzr, xzr, [sp, #(EXCEPTION_CONTEXT_X14_X15)]
stp xzr, xzr, [sp, #(EXCEPTION_CONTEXT_X16_X17)]
str x19, [sp, #(EXCEPTION_CONTEXT_X19)]
stp x20, x21, [sp, #(EXCEPTION_CONTEXT_X20_X21)]
stp x22, x23, [sp, #(EXCEPTION_CONTEXT_X22_X23)]
stp x24, x25, [sp, #(EXCEPTION_CONTEXT_X24_X25)]
stp x26, x27, [sp, #(EXCEPTION_CONTEXT_X26_X27)]
stp x28, x29, [sp, #(EXCEPTION_CONTEXT_X28_X29)]
/* Call ams::kern::arch::arm64::HandleException(ams::kern::arch::arm64::KExceptionContext *) */
mov x0, sp
bl _ZN3ams4kern4arch5arm6415HandleExceptionEPNS2_17KExceptionContextE
/* If we don't need to restore the fpu, skip restoring it. */
ldrb w9, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
tbz w9, #(THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_FPU_CONTEXT_RESTORE_NEEDED), 4f
/* Clear the needs-fpu-restore flag. */
and w9, w9, #(~THREAD_EXCEPTION_FLAG_IS_FPU_CONTEXT_RESTORE_NEEDED)
strb w9, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
/* Enable and restore the fpu. */
ENABLE_AND_RESTORE_FPU64(x10, x8, x9, w8, w9)
4: /* Restore registers. */
ldp x30, x8, [sp, #(EXCEPTION_CONTEXT_X30_SP)]
ldp x9, x10, [sp, #(EXCEPTION_CONTEXT_PC_PSR)]
ldr x11, [sp, #(EXCEPTION_CONTEXT_TPIDR)]
#if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP)
/* Since we're returning from an SVC, make sure SPSR.SS is cleared so that if we're single-stepping we break instantly on the instruction after the SVC. */
bic x10, x10, #(1 << 21)
#endif
msr sp_el0, x8
msr elr_el1, x9
msr spsr_el1, x10
msr tpidr_el0, x11
ldp x0, x1, [sp, #(EXCEPTION_CONTEXT_X0_X1)]
ldp x2, x3, [sp, #(EXCEPTION_CONTEXT_X2_X3)]
ldp x4, x5, [sp, #(EXCEPTION_CONTEXT_X4_X5)]
ldp x6, x7, [sp, #(EXCEPTION_CONTEXT_X6_X7)]
ldp x8, x9, [sp, #(EXCEPTION_CONTEXT_X8_X9)]
ldp x10, x11, [sp, #(EXCEPTION_CONTEXT_X10_X11)]
ldp x12, x13, [sp, #(EXCEPTION_CONTEXT_X12_X13)]
ldp x14, x15, [sp, #(EXCEPTION_CONTEXT_X14_X15)]
ldp x16, x17, [sp, #(EXCEPTION_CONTEXT_X16_X17)]
ldp x18, x19, [sp, #(EXCEPTION_CONTEXT_X18_X19)]
ldp x20, x21, [sp, #(EXCEPTION_CONTEXT_X20_X21)]
ldp x22, x23, [sp, #(EXCEPTION_CONTEXT_X22_X23)]
ldp x24, x25, [sp, #(EXCEPTION_CONTEXT_X24_X25)]
ldp x26, x27, [sp, #(EXCEPTION_CONTEXT_X26_X27)]
ldp x28, x29, [sp, #(EXCEPTION_CONTEXT_X28_X29)]
/* Return. */
add sp, sp, #(EXCEPTION_CONTEXT_SIZE)
ERET_WITH_SPECULATION_BARRIER
5: /* Return from SVC. */
/* If we should, trace the svc exit. */
#if defined(MESOSPHERE_BUILD_FOR_TRACING)
sub sp, sp, #0x40
stp x0, x1, [sp, #(8 * 0)]
stp x2, x3, [sp, #(8 * 2)]
stp x4, x5, [sp, #(8 * 4)]
stp x6, x7, [sp, #(8 * 6)]
mov x0, sp
bl _ZN3ams4kern3svc12TraceSvcExitEPKm
ldp x0, x1, [sp, #(8 * 0)]
ldp x2, x3, [sp, #(8 * 2)]
ldp x4, x5, [sp, #(8 * 4)]
ldp x6, x7, [sp, #(8 * 6)]
add sp, sp, #0x40
#endif
/* Get our exception flags. */
ldrb w9, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
/* Clear in-svc and needs-fpu-restore flags. */
and w8, w9, #(~(THREAD_EXCEPTION_FLAG_IS_FPU_CONTEXT_RESTORE_NEEDED))
and w8, w8, #(~(THREAD_EXCEPTION_FLAG_IS_CALLING_SVC))
strb w8, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
/* If we don't need to restore the fpu, skip restoring it. */
tbz w9, #(THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_FPU_CONTEXT_RESTORE_NEEDED), 7f
/* If we need to restore the fpu, check if we need to do a full restore. */
tbnz w9, #(THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_IN_USERMODE_EXCEPTION_HANDLER), 6f
/* Enable the fpu. */
ENABLE_FPU(x8)
/* Get the thread context and restore fpsr/fpcr. */
GET_THREAD_CONTEXT_AND_RESTORE_FPCR_FPSR(x10, x8, x9, w8, w9)
/* Restore callee-saved registers to 64-bit fpu. */
RESTORE_FPU64_CALLEE_SAVE_REGISTERS(x10)
/* Clear caller-saved registers to 64-bit fpu. */
movi v0.2d, #0
movi v1.2d, #0
movi v2.2d, #0
movi v3.2d, #0
movi v4.2d, #0
movi v5.2d, #0
movi v6.2d, #0
movi v7.2d, #0
movi v16.2d, #0
movi v17.2d, #0
movi v18.2d, #0
movi v19.2d, #0
movi v20.2d, #0
movi v21.2d, #0
movi v22.2d, #0
movi v23.2d, #0
movi v24.2d, #0
movi v25.2d, #0
movi v26.2d, #0
movi v27.2d, #0
movi v28.2d, #0
movi v29.2d, #0
movi v30.2d, #0
movi v31.2d, #0
b 7f
6: /* We need to do a full fpu restore. */
ENABLE_AND_RESTORE_FPU64(x10, x8, x9, w8, w9)
7: /* Restore registers. */
ldp x30, x8, [sp, #(EXCEPTION_CONTEXT_X30_SP)]
ldp x9, x10, [sp, #(EXCEPTION_CONTEXT_PC_PSR)]
ldr x11, [sp, #(EXCEPTION_CONTEXT_TPIDR)]
ldr x18, [sp, #(EXCEPTION_CONTEXT_X18)]
#if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP)
/* Since we're returning from an SVC, make sure SPSR.SS is cleared so that if we're single-stepping we break instantly on the instruction after the SVC. */
bic x10, x10, #(1 << 21)
#endif
msr sp_el0, x8
msr elr_el1, x9
msr spsr_el1, x10
msr tpidr_el0, x11
/* Clear registers. */
mov x8, xzr
mov x9, xzr
mov x10, xzr
mov x11, xzr
mov x12, xzr
mov x13, xzr
mov x14, xzr
mov x15, xzr
mov x16, xzr
mov x17, xzr
/* Return. */
add sp, sp, #(EXCEPTION_CONTEXT_SIZE)
ERET_WITH_SPECULATION_BARRIER
/* ams::kern::arch::arm64::SvcHandler32() */
.section .text._ZN3ams4kern4arch5arm6412SvcHandler32Ev, "ax", %progbits
.global _ZN3ams4kern4arch5arm6412SvcHandler32Ev
.type _ZN3ams4kern4arch5arm6412SvcHandler32Ev, %function
_ZN3ams4kern4arch5arm6412SvcHandler32Ev:
/* Ensure that our registers are 32-bit. */
mov w0, w0
mov w1, w1
mov w2, w2
mov w3, w3
mov w4, w4
mov w5, w5
mov w6, w6
mov w7, w7
/* Create a KExceptionContext for the exception. */
sub sp, sp, #(EXCEPTION_CONTEXT_SIZE)
/* Save system registers */
mrs x17, elr_el1
mrs x20, spsr_el1
mrs x19, tpidr_el0
ldr x18, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_CUR_THREAD)]
stp x17, x20, [sp, #(EXCEPTION_CONTEXT_PC_PSR)]
str x19, [sp, #(EXCEPTION_CONTEXT_TPIDR)]
/* Save registers. */
stp x0, x1, [sp, #(EXCEPTION_CONTEXT_X0_X1)]
stp x2, x3, [sp, #(EXCEPTION_CONTEXT_X2_X3)]
stp x4, x5, [sp, #(EXCEPTION_CONTEXT_X4_X5)]
stp x6, x7, [sp, #(EXCEPTION_CONTEXT_X6_X7)]
stp x8, x9, [sp, #(EXCEPTION_CONTEXT_X8_X9)]
stp x10, x11, [sp, #(EXCEPTION_CONTEXT_X10_X11)]
stp x12, x13, [sp, #(EXCEPTION_CONTEXT_X12_X13)]
stp x14, xzr, [sp, #(EXCEPTION_CONTEXT_X14_X15)]
/* Check if the SVC index is out of range. */
mrs x8, esr_el1
and x8, x8, #0xFF
cmp x8, #(AMS_KERN_NUM_SUPERVISOR_CALLS)
b.ge 3f
/* Check the specific SVC permission bit for allowal. */
mov x9, sp
add x9, x9, x8, lsr#3
ldrb w9, [x9, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_SVC_PERMISSION)]
and x10, x8, #0x7
lsr x10, x9, x10
tst x10, #1
b.eq 3f
/* Check if our disable count allows us to call SVCs. */
mrs x10, tpidrro_el0
add x10, x10, #(THREAD_LOCAL_REGION_DISABLE_COUNT)
ldtrh w10, [x10]
cbz w10, 1f
/* It might not, so check the stack params to see if we must not allow the SVC. */
ldrb w10, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_IS_PINNED)]
cbz w10, 3f
1: /* We can call the SVC. */
adr x10, _ZN3ams4kern3svc16SvcTable64From32E
ldr x11, [x10, x8, lsl#3]
cbz x11, 3f
/* Note that we're calling the SVC. */
ldrb w9, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
orr w9, w9, #(THREAD_EXCEPTION_FLAG_IS_CALLING_SVC)
strb w9, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
strb w8, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_CURRENT_SVC_ID)]
/* If we should, trace the svc entry. */
#if defined(MESOSPHERE_BUILD_FOR_TRACING)
sub sp, sp, #0x50
stp x0, x1, [sp, #(8 * 0)]
stp x2, x3, [sp, #(8 * 2)]
stp x4, x5, [sp, #(8 * 4)]
stp x6, x7, [sp, #(8 * 6)]
str x11, [sp, #(8 * 8)]
mov x0, sp
bl _ZN3ams4kern3svc13TraceSvcEntryEPKm
ldp x0, x1, [sp, #(8 * 0)]
ldp x2, x3, [sp, #(8 * 2)]
ldp x4, x5, [sp, #(8 * 4)]
ldp x6, x7, [sp, #(8 * 6)]
ldr x11, [sp, #(8 * 8)]
add sp, sp, #0x50
#endif
/* Invoke the SVC handler. */
msr daifclr, #2
blr x11
msr daifset, #2
2: /* We completed the SVC, and we should handle DPC. */
/* Check the dpc flags. */
ldrb w8, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_DPC_FLAGS)]
cbz w8, 5f
/* We have DPC to do! */
/* Save registers and call ams::kern::KDpcManager::HandleDpc(). */
sub sp, sp, #0x20
stp w0, w1, [sp, #(4 * 0)]
stp w2, w3, [sp, #(4 * 2)]
stp w4, w5, [sp, #(4 * 4)]
stp w6, w7, [sp, #(4 * 6)]
bl _ZN3ams4kern11KDpcManager9HandleDpcEv
ldp w0, w1, [sp, #(4 * 0)]
ldp w2, w3, [sp, #(4 * 2)]
ldp w4, w5, [sp, #(4 * 4)]
ldp w6, w7, [sp, #(4 * 6)]
add sp, sp, #0x20
b 2b
3: /* Invalid SVC. */
/* Setup the context to call into HandleException. */
stp x0, x1, [sp, #(EXCEPTION_CONTEXT_X0_X1)]
stp x2, x3, [sp, #(EXCEPTION_CONTEXT_X2_X3)]
stp x4, x5, [sp, #(EXCEPTION_CONTEXT_X4_X5)]
stp x6, x7, [sp, #(EXCEPTION_CONTEXT_X6_X7)]
stp xzr, xzr, [sp, #(EXCEPTION_CONTEXT_X16_X17)]
stp xzr, xzr, [sp, #(EXCEPTION_CONTEXT_X18_X19)]
stp xzr, xzr, [sp, #(EXCEPTION_CONTEXT_X20_X21)]
stp xzr, xzr, [sp, #(EXCEPTION_CONTEXT_X22_X23)]
stp xzr, xzr, [sp, #(EXCEPTION_CONTEXT_X24_X25)]
stp xzr, xzr, [sp, #(EXCEPTION_CONTEXT_X26_X27)]
stp xzr, xzr, [sp, #(EXCEPTION_CONTEXT_X28_X29)]
stp xzr, xzr, [sp, #(EXCEPTION_CONTEXT_X30_SP)]
/* Call ams::kern::arch::arm64::HandleException(ams::kern::arch::arm64::KExceptionContext *) */
mov x0, sp
bl _ZN3ams4kern4arch5arm6415HandleExceptionEPNS2_17KExceptionContextE
/* If we don't need to restore the fpu, skip restoring it. */
ldrb w9, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
tbz w9, #(THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_FPU_CONTEXT_RESTORE_NEEDED), 4f
/* Clear the needs-fpu-restore flag. */
and w9, w9, #(~THREAD_EXCEPTION_FLAG_IS_FPU_CONTEXT_RESTORE_NEEDED)
strb w9, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
/* Enable and restore the fpu. */
ENABLE_AND_RESTORE_FPU32(x10, x8, x9, w8, w9)
4: /* Restore registers. */
ldp x9, x10, [sp, #(EXCEPTION_CONTEXT_PC_PSR)]
ldr x11, [sp, #(EXCEPTION_CONTEXT_TPIDR)]
#if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP)
/* Since we're returning from an SVC, make sure SPSR.SS is cleared so that if we're single-stepping we break instantly on the instruction after the SVC. */
bic x10, x10, #(1 << 21)
#endif
msr elr_el1, x9
msr spsr_el1, x10
msr tpidr_el0, x11
ldp x0, x1, [sp, #(EXCEPTION_CONTEXT_X0_X1)]
ldp x2, x3, [sp, #(EXCEPTION_CONTEXT_X2_X3)]
ldp x4, x5, [sp, #(EXCEPTION_CONTEXT_X4_X5)]
ldp x6, x7, [sp, #(EXCEPTION_CONTEXT_X6_X7)]
ldp x8, x9, [sp, #(EXCEPTION_CONTEXT_X8_X9)]
ldp x10, x11, [sp, #(EXCEPTION_CONTEXT_X10_X11)]
ldp x12, x13, [sp, #(EXCEPTION_CONTEXT_X12_X13)]
ldp x14, x15, [sp, #(EXCEPTION_CONTEXT_X14_X15)]
/* Return. */
add sp, sp, #(EXCEPTION_CONTEXT_SIZE)
ERET_WITH_SPECULATION_BARRIER
5: /* Return from SVC. */
/* If we should, trace the svc exit. */
#if defined(MESOSPHERE_BUILD_FOR_TRACING)
sub sp, sp, #0x40
stp x0, x1, [sp, #(8 * 0)]
stp x2, x3, [sp, #(8 * 2)]
stp x4, x5, [sp, #(8 * 4)]
stp x6, x7, [sp, #(8 * 6)]
mov x0, sp
bl _ZN3ams4kern3svc12TraceSvcExitEPKm
ldp x0, x1, [sp, #(8 * 0)]
ldp x2, x3, [sp, #(8 * 2)]
ldp x4, x5, [sp, #(8 * 4)]
ldp x6, x7, [sp, #(8 * 6)]
add sp, sp, #0x40
#endif
/* Get our exception flags. */
ldrb w9, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
/* Clear in-svc and needs-fpu-restore flags. */
and w8, w9, #(~(THREAD_EXCEPTION_FLAG_IS_FPU_CONTEXT_RESTORE_NEEDED))
and w8, w8, #(~(THREAD_EXCEPTION_FLAG_IS_CALLING_SVC))
strb w8, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
/* If we don't need to restore the fpu, skip restoring it. */
tbz w9, #(THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_FPU_CONTEXT_RESTORE_NEEDED), 7f
/* If we need to restore the fpu, check if we need to do a full restore. */
tbnz w9, #(THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_IN_USERMODE_EXCEPTION_HANDLER), 6f
/* Enable the fpu. */
ENABLE_FPU(x8)
/* Get the thread context and restore fpsr/fpcr. */
GET_THREAD_CONTEXT_AND_RESTORE_FPCR_FPSR(x10, x8, x9, w8, w9)
/* Restore callee-saved registers to 32-bit fpu. */
RESTORE_FPU32_CALLEE_SAVE_REGISTERS(x10)
/* Clear caller-saved registers to 32-bit fpu. */
movi v0.2d, #0
movi v1.2d, #0
movi v2.2d, #0
movi v3.2d, #0
movi v8.2d, #0
movi v9.2d, #0
movi v10.2d, #0
movi v11.2d, #0
movi v12.2d, #0
movi v13.2d, #0
movi v14.2d, #0
movi v15.2d, #0
b 7f
6: /* We need to do a full fpu restore. */
ENABLE_AND_RESTORE_FPU32(x10, x8, x9, w8, w9)
7: /* Restore registers. */
ldp x9, x10, [sp, #(EXCEPTION_CONTEXT_PC_PSR)]
ldr x11, [sp, #(EXCEPTION_CONTEXT_TPIDR)]
#if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP)
/* Since we're returning from an SVC, make sure SPSR.SS is cleared so that if we're single-stepping we break instantly on the instruction after the SVC. */
bic x10, x10, #(1 << 21)
#endif
msr elr_el1, x9
msr spsr_el1, x10
msr tpidr_el0, x11
ldp x8, x9, [sp, #(EXCEPTION_CONTEXT_X8_X9)]
ldp x10, x11, [sp, #(EXCEPTION_CONTEXT_X10_X11)]
ldp x12, x13, [sp, #(EXCEPTION_CONTEXT_X12_X13)]
ldp x14, xzr, [sp, #(EXCEPTION_CONTEXT_X14_X15)]
/* Return. */
add sp, sp, #(EXCEPTION_CONTEXT_SIZE)
ERET_WITH_SPECULATION_BARRIER

139
Source/Atmosphere/libraries/libmesosphere/source/arch/arm64/svc/kern_svc_light_ipc_asm.s Normal file
View File

@@ -0,0 +1,139 @@
/*
* Copyright (c) Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
/* ams::kern::svc::CallSendSyncRequestLight64() */
.section .text._ZN3ams4kern3svc26CallSendSyncRequestLight64Ev, "ax", %progbits
.global _ZN3ams4kern3svc26CallSendSyncRequestLight64Ev
.type _ZN3ams4kern3svc26CallSendSyncRequestLight64Ev, %function
_ZN3ams4kern3svc26CallSendSyncRequestLight64Ev:
/* Allocate space for the light ipc data. */
sub sp, sp, #(4 * 8)
/* Store the light ipc data. */
stp w1, w2, [sp, #(4 * 0)]
stp w3, w4, [sp, #(4 * 2)]
stp w5, w6, [sp, #(4 * 4)]
str w7, [sp, #(4 * 6)]
/* Invoke the svc handler. */
mov x1, sp
stp x29, x30, [sp, #-16]!
bl _ZN3ams4kern3svc22SendSyncRequestLight64EjPj
ldp x29, x30, [sp], #16
/* Load the light ipc data. */
ldp w1, w2, [sp, #(4 * 0)]
ldp w3, w4, [sp, #(4 * 2)]
ldp w5, w6, [sp, #(4 * 4)]
ldr w7, [sp, #(4 * 6)]
/* Free the stack space for the light ipc data. */
add sp, sp, #(4 * 8)
ret
/* ams::kern::svc::CallSendSyncRequestLight64From32() */
.section .text._ZN3ams4kern3svc32CallSendSyncRequestLight64From32Ev, "ax", %progbits
.global _ZN3ams4kern3svc32CallSendSyncRequestLight64From32Ev
.type _ZN3ams4kern3svc32CallSendSyncRequestLight64From32Ev, %function
_ZN3ams4kern3svc32CallSendSyncRequestLight64From32Ev:
/* Allocate space for the light ipc data. */
sub sp, sp, #(4 * 8)
/* Store the light ipc data. */
stp w1, w2, [sp, #(4 * 0)]
stp w3, w4, [sp, #(4 * 2)]
stp w5, w6, [sp, #(4 * 4)]
str w7, [sp, #(4 * 6)]
/* Invoke the svc handler. */
mov x1, sp
stp x29, x30, [sp, #-16]!
bl _ZN3ams4kern3svc28SendSyncRequestLight64From32EjPj
ldp x29, x30, [sp], #16
/* Load the light ipc data. */
ldp w1, w2, [sp, #(4 * 0)]
ldp w3, w4, [sp, #(4 * 2)]
ldp w5, w6, [sp, #(4 * 4)]
ldr w7, [sp, #(4 * 6)]
/* Free the stack space for the light ipc data. */
add sp, sp, #(4 * 8)
ret
/* ams::kern::svc::CallReplyAndReceiveLight64() */
.section .text._ZN3ams4kern3svc26CallReplyAndReceiveLight64Ev, "ax", %progbits
.global _ZN3ams4kern3svc26CallReplyAndReceiveLight64Ev
.type _ZN3ams4kern3svc26CallReplyAndReceiveLight64Ev, %function
_ZN3ams4kern3svc26CallReplyAndReceiveLight64Ev:
/* Allocate space for the light ipc data. */
sub sp, sp, #(4 * 8)
/* Store the light ipc data. */
stp w1, w2, [sp, #(4 * 0)]
stp w3, w4, [sp, #(4 * 2)]
stp w5, w6, [sp, #(4 * 4)]
str w7, [sp, #(4 * 6)]
/* Invoke the svc handler. */
mov x1, sp
stp x29, x30, [sp, #-16]!
bl _ZN3ams4kern3svc22ReplyAndReceiveLight64EjPj
ldp x29, x30, [sp], #16
/* Load the light ipc data. */
ldp w1, w2, [sp, #(4 * 0)]
ldp w3, w4, [sp, #(4 * 2)]
ldp w5, w6, [sp, #(4 * 4)]
ldr w7, [sp, #(4 * 6)]
/* Free the stack space for the light ipc data. */
add sp, sp, #(4 * 8)
ret
/* ams::kern::svc::CallReplyAndReceiveLight64From32() */
.section .text._ZN3ams4kern3svc32CallReplyAndReceiveLight64From32Ev, "ax", %progbits
.global _ZN3ams4kern3svc32CallReplyAndReceiveLight64From32Ev
.type _ZN3ams4kern3svc32CallReplyAndReceiveLight64From32Ev, %function
_ZN3ams4kern3svc32CallReplyAndReceiveLight64From32Ev:
/* Allocate space for the light ipc data. */
sub sp, sp, #(4 * 8)
/* Store the light ipc data. */
stp w1, w2, [sp, #(4 * 0)]
stp w3, w4, [sp, #(4 * 2)]
stp w5, w6, [sp, #(4 * 4)]
str w7, [sp, #(4 * 6)]
/* Invoke the svc handler. */
mov x1, sp
stp x29, x30, [sp, #-16]!
bl _ZN3ams4kern3svc28ReplyAndReceiveLight64From32EjPj
ldp x29, x30, [sp], #16
/* Load the light ipc data. */
ldp w1, w2, [sp, #(4 * 0)]
ldp w3, w4, [sp, #(4 * 2)]
ldp w5, w6, [sp, #(4 * 4)]
ldr w7, [sp, #(4 * 6)]
/* Free the stack space for the light ipc data. */
add sp, sp, #(4 * 8)
ret

182
Source/Atmosphere/libraries/libmesosphere/source/arch/arm64/svc/kern_svc_tables.cpp Normal file
View File

@@ -0,0 +1,182 @@
/*
* Copyright (c) Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifdef MESOSPHERE_USE_STUBBED_SVC_TABLES
#include <mesosphere/kern_debug_log.hpp>
#endif
#include <mesosphere/svc/kern_svc_tables.hpp>
#include <vapours/svc/svc_codegen.hpp>
namespace ams::kern::svc {
/* Declare special prototypes for the light ipc handlers. */
void CallSendSyncRequestLight64();
void CallSendSyncRequestLight64From32();
void CallReplyAndReceiveLight64();
void CallReplyAndReceiveLight64From32();
/* Declare special prototypes for ReturnFromException. */
void CallReturnFromException64();
void CallReturnFromException64From32();
/* Declare special prototype for (unsupported) CallCallSecureMonitor64From32. */
void CallCallSecureMonitor64From32();
/* Declare special prototypes for WaitForAddress. */
void CallWaitForAddress64();
void CallWaitForAddress64From32();
namespace {
#ifndef MESOSPHERE_USE_STUBBED_SVC_TABLES
#define DECLARE_SVC_STRUCT(ID, RETURN_TYPE, NAME, ...) \
class NAME { \
private: \
using Impl = ::ams::svc::codegen::KernelSvcWrapper<::ams::kern::svc::NAME##64, ::ams::kern::svc::NAME##64From32>; \
public: \
static NOINLINE void Call64() { return Impl::Call64(); } \
static NOINLINE void Call64From32() { return Impl::Call64From32(); } \
};
#else
#define DECLARE_SVC_STRUCT(ID, RETURN_TYPE, NAME, ...) \
class NAME { \
public: \
static NOINLINE void Call64() { MESOSPHERE_PANIC("Stubbed Svc"#NAME"64 was called"); } \
static NOINLINE void Call64From32() { MESOSPHERE_PANIC("Stubbed Svc"#NAME"64From32 was called"); } \
};
#endif
/* Set omit-frame-pointer to prevent GCC from emitting MOV X29, SP instructions. */
#pragma GCC push_options
#pragma GCC optimize ("-O3")
#pragma GCC optimize ("omit-frame-pointer")
AMS_SVC_FOREACH_KERN_DEFINITION(DECLARE_SVC_STRUCT, _)
#pragma GCC pop_options
constexpr const std::array<SvcTableEntry, NumSupervisorCalls> SvcTable64From32Impl = [] {
std::array<SvcTableEntry, NumSupervisorCalls> table = {};
#define AMS_KERN_SVC_SET_TABLE_ENTRY(ID, RETURN_TYPE, NAME, ...) \
if (table[ID] == nullptr) { table[ID] = NAME::Call64From32; }
AMS_SVC_FOREACH_KERN_DEFINITION(AMS_KERN_SVC_SET_TABLE_ENTRY, _)
#undef AMS_KERN_SVC_SET_TABLE_ENTRY
table[svc::SvcId_SendSyncRequestLight] = CallSendSyncRequestLight64From32;
table[svc::SvcId_ReplyAndReceiveLight] = CallReplyAndReceiveLight64From32;
table[svc::SvcId_ReturnFromException] = CallReturnFromException64From32;
table[svc::SvcId_CallSecureMonitor] = CallCallSecureMonitor64From32;
table[svc::SvcId_WaitForAddress] = CallWaitForAddress64From32;
return table;
}();
constexpr const std::array<SvcTableEntry, NumSupervisorCalls> SvcTable64Impl = [] {
std::array<SvcTableEntry, NumSupervisorCalls> table = {};
#define AMS_KERN_SVC_SET_TABLE_ENTRY(ID, RETURN_TYPE, NAME, ...) \
if (table[ID] == nullptr) { table[ID] = NAME::Call64; }
AMS_SVC_FOREACH_KERN_DEFINITION(AMS_KERN_SVC_SET_TABLE_ENTRY, _)
#undef AMS_KERN_SVC_SET_TABLE_ENTRY
table[svc::SvcId_SendSyncRequestLight] = CallSendSyncRequestLight64;
table[svc::SvcId_ReplyAndReceiveLight] = CallReplyAndReceiveLight64;
table[svc::SvcId_ReturnFromException] = CallReturnFromException64;
table[svc::SvcId_WaitForAddress] = CallWaitForAddress64;
return table;
}();
constexpr bool IsValidSvcTable(const std::array<SvcTableEntry, NumSupervisorCalls> &table) {
for (size_t i = 0; i < NumSupervisorCalls; i++) {
if (table[i] != nullptr) {
return true;
}
}
return false;
}
static_assert(IsValidSvcTable(SvcTable64Impl));
static_assert(IsValidSvcTable(SvcTable64From32Impl));
}
constinit const std::array<SvcTableEntry, NumSupervisorCalls> SvcTable64 = SvcTable64Impl;
constinit const std::array<SvcTableEntry, NumSupervisorCalls> SvcTable64From32 = SvcTable64From32Impl;
void PatchSvcTableEntry(const SvcTableEntry *table, u32 id, SvcTableEntry entry);
namespace {
/* NOTE: Although the SVC tables are constants, our global constructor will run before .rodata is protected R--. */
class SvcTablePatcher {
private:
using SvcTable = std::array<SvcTableEntry, NumSupervisorCalls>;
private:
static SvcTablePatcher s_instance;
private:
ALWAYS_INLINE const SvcTableEntry *GetTableData(const SvcTable *table) {
if (table != nullptr) {
return table->data();
} else {
return nullptr;
}
}
NOINLINE void PatchTables(const SvcTableEntry *table_64, const SvcTableEntry *table_64_from_32) {
/* Get the target firmware. */
const auto target_fw = kern::GetTargetFirmware();
/* 10.0.0 broke the ABI for QueryIoMapping, and renamed it to QueryMemoryMapping. */
if (target_fw < TargetFirmware_10_0_0) {
if (table_64) { ::ams::kern::svc::PatchSvcTableEntry(table_64, svc::SvcId_QueryMemoryMapping, LegacyQueryIoMapping::Call64); }
if (table_64_from_32) { ::ams::kern::svc::PatchSvcTableEntry(table_64_from_32, svc::SvcId_QueryMemoryMapping, LegacyQueryIoMapping::Call64From32); }
}
/* 6.0.0 broke the ABI for GetFutureThreadInfo, and renamed it to GetDebugFutureThreadInfo. */
if (target_fw < TargetFirmware_6_0_0) {
static_assert(svc::SvcId_GetDebugFutureThreadInfo == svc::SvcId_LegacyGetFutureThreadInfo);
if (table_64) { ::ams::kern::svc::PatchSvcTableEntry(table_64, svc::SvcId_GetDebugFutureThreadInfo, LegacyGetFutureThreadInfo::Call64); }
if (table_64_from_32) { ::ams::kern::svc::PatchSvcTableEntry(table_64_from_32, svc::SvcId_GetDebugFutureThreadInfo, LegacyGetFutureThreadInfo::Call64From32); }
}
/* 3.0.0 broke the ABI for ContinueDebugEvent. */
if (target_fw < TargetFirmware_3_0_0) {
if (table_64) { ::ams::kern::svc::PatchSvcTableEntry(table_64, svc::SvcId_ContinueDebugEvent, LegacyContinueDebugEvent::Call64); }
if (table_64_from_32) { ::ams::kern::svc::PatchSvcTableEntry(table_64_from_32, svc::SvcId_ContinueDebugEvent, LegacyContinueDebugEvent::Call64From32); }
}
}
public:
SvcTablePatcher(const SvcTable *table_64, const SvcTable *table_64_from_32) {
PatchTables(GetTableData(table_64), GetTableData(table_64_from_32));
}
};
SvcTablePatcher SvcTablePatcher::s_instance(std::addressof(SvcTable64), std::addressof(SvcTable64From32));
}
}