OmniNX/Atmosphere
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

kern: update for new hw maintenance semantics

Browse Source
This commit is contained in:
Michael Scire
2022-03-22 21:33:43 -07:00
committed by SciresM
parent 6e17317d5d
commit 9d89835ff8
19 changed files with 386 additions and 244 deletions
Download Patch File Download Diff File Expand all files Collapse all files

136
libraries/libmesosphere/source/arch/arm64/kern_cpu.cpp
View File

@@ -176,7 +176,7 @@ namespace ams::kern::arch::arm64::cpu {
const u64 target_mask = m_target_cores.Load();
DataSynchronizationBarrier();
DataSynchronizationBarrierInnerShareable();
Kernel::GetInterruptManager().SendInterProcessorInterrupt(KInterruptName_CacheOperation, target_mask);
this->ProcessOperation();
@@ -213,32 +213,37 @@ namespace ams::kern::arch::arm64::cpu {
};
/* Instances of the interrupt handlers. */
KThreadTerminationInterruptHandler g_thread_termination_handler;
KCacheHelperInterruptHandler g_cache_operation_handler;
KPerformanceCounterInterruptHandler g_performance_counter_handler[cpu::NumCores];
constinit KThreadTerminationInterruptHandler g_thread_termination_handler;
constinit KCacheHelperInterruptHandler g_cache_operation_handler;
constinit KPerformanceCounterInterruptHandler g_performance_counter_handler[cpu::NumCores];
/* Expose this as a global, for asm to use. */
s32 g_all_core_sync_count;
constinit s32 g_all_core_sync_count;
template<bool Init, typename F>
template<typename F>
ALWAYS_INLINE void PerformCacheOperationBySetWayImpl(int level, F f) {
/* Used in multiple locations. */
const u64 level_sel_value = static_cast<u64>(level << 1);
/* Get the cache size id register value with interrupts disabled. */
u64 ccsidr_value;
if constexpr (Init) {
/* During init, we can just set the selection register directly. */
cpu::SetCsselrEl1(level_sel_value);
cpu::InstructionMemoryBarrier();
ccsidr_value = cpu::GetCcsidrEl1();
} else {
/* After init, we need to care about interrupts. */
{
/* Disable interrupts. */
KScopedInterruptDisable di;
/* Configure the cache select register for our level. */
cpu::SetCsselrEl1(level_sel_value);
/* Ensure our configuration takes before reading the cache size id register. */
cpu::InstructionMemoryBarrier();
/* Get the cache size id register. */
ccsidr_value = cpu::GetCcsidrEl1();
}
/* Ensure that no memory inconsistencies occur between cache management invocations. */
cpu::DataSynchronizationBarrier();
/* Get cache size id info. */
CacheSizeIdRegisterAccessor ccsidr_el1(ccsidr_value);
const int num_sets = ccsidr_el1.GetNumberOfSets();
@@ -266,13 +271,11 @@ namespace ams::kern::arch::arm64::cpu {
}
void StoreDataCacheBySetWay(int level) {
PerformCacheOperationBySetWayImpl<false>(level, StoreDataCacheLineBySetWayImpl);
cpu::DataSynchronizationBarrier();
PerformCacheOperationBySetWayImpl(level, StoreDataCacheLineBySetWayImpl);
}
void FlushDataCacheBySetWay(int level) {
PerformCacheOperationBySetWayImpl<false>(level, FlushDataCacheLineBySetWayImpl);
cpu::DataSynchronizationBarrier();
PerformCacheOperationBySetWayImpl(level, FlushDataCacheLineBySetWayImpl);
}
void KCacheHelperInterruptHandler::ProcessOperation() {
@@ -284,9 +287,11 @@ namespace ams::kern::arch::arm64::cpu {
break;
case Operation::StoreDataCache:
StoreDataCacheBySetWay(0);
cpu::DataSynchronizationBarrier();
break;
case Operation::FlushDataCache:
FlushDataCacheBySetWay(0);
cpu::DataSynchronizationBarrier();
break;
}
}
@@ -323,14 +328,6 @@ namespace ams::kern::arch::arm64::cpu {
R_SUCCEED();
}
ALWAYS_INLINE Result InvalidateInstructionCacheRange(uintptr_t start, uintptr_t end) {
MESOSPHERE_ASSERT(util::IsAligned(start, InstructionCacheLineSize));
MESOSPHERE_ASSERT(util::IsAligned(end, InstructionCacheLineSize));
R_UNLESS(UserspaceAccess::InvalidateInstructionCache(start, end), svc::ResultInvalidCurrentMemory());
EnsureInstructionConsistency();
R_SUCCEED();
}
ALWAYS_INLINE void InvalidateEntireInstructionCacheLocalImpl() {
__asm__ __volatile__("ic iallu" ::: "memory");
}
@@ -341,26 +338,12 @@ namespace ams::kern::arch::arm64::cpu {
}
void StoreEntireCacheForInit() {
/* Store local. */
{
CacheLineIdRegisterAccessor clidr_el1;
const int levels_of_unification = clidr_el1.GetLevelsOfUnification();
for (int level = 0; level != levels_of_unification; ++level) {
PerformCacheOperationBySetWayImpl<true>(level, StoreDataCacheLineBySetWayImpl);
}
}
/* Store shared. */
{
CacheLineIdRegisterAccessor clidr_el1;
const int levels_of_coherency = clidr_el1.GetLevelsOfCoherency();
const int levels_of_unification = clidr_el1.GetLevelsOfUnification();
for (int level = levels_of_unification; level <= levels_of_coherency; ++level) {
PerformCacheOperationBySetWayImpl<true>(level, StoreDataCacheLineBySetWayImpl);
}
void StoreCacheForInit(void *addr, size_t size) {
/* Store the data cache for the specified range. */
const uintptr_t start = util::AlignDown(reinterpret_cast<uintptr_t>(addr), DataCacheLineSize);
const uintptr_t end = start + size;
for (uintptr_t cur = start; cur < end; cur += DataCacheLineSize) {
__asm__ __volatile__("dc cvac, %[cur]" :: [cur]"r"(cur) : "memory");
}
/* Data synchronization barrier. */
@@ -370,36 +353,7 @@ namespace ams::kern::arch::arm64::cpu {
InvalidateEntireInstructionCacheLocalImpl();
/* Ensure local instruction consistency. */
DataSynchronizationBarrierInnerShareable();
InstructionMemoryBarrier();
}
void FlushEntireCacheForInit() {
/* Flush data cache. */
{
/* Get levels of coherence/unificaiton. */
CacheLineIdRegisterAccessor clidr_el1;
const int levels_of_coherency = clidr_el1.GetLevelsOfCoherency();
/* Store cache from L1 up to (level of coherence - 1). */
for (int level = 0; level < levels_of_coherency - 1; ++level) {
PerformCacheOperationBySetWayImpl<true>(level, StoreDataCacheLineBySetWayImpl);
DataSynchronizationBarrier();
}
/* Flush cache from (level of coherence - 1) down to L0. */
for (int level = levels_of_coherency; level > 0; --level) {
PerformCacheOperationBySetWayImpl<true>(level - 1, FlushDataCacheLineBySetWayImpl);
DataSynchronizationBarrier();
}
}
/* Invalidate instruction cache. */
InvalidateEntireInstructionCacheLocalImpl();
EnsureInstructionConsistency();
/* Invalidate entire TLB. */
InvalidateEntireTlb();
}
void FlushEntireDataCache() {
@@ -417,10 +371,17 @@ namespace ams::kern::arch::arm64::cpu {
for (int level = levels_of_coherency; level > 1; --level) {
FlushDataCacheBySetWay(level - 1);
}
/* Data synchronization barrier for full system. */
DataSynchronizationBarrier();
}
Result InvalidateDataCache(void *addr, size_t size) {
KScopedCoreMigrationDisable dm;
/* Mark ourselves as in a cache maintenance operation, and prevent re-ordering. */
__asm__ __volatile__("" ::: "memory");
GetCurrentThread().SetInCacheMaintenanceOperation();
ON_SCOPE_EXIT { GetCurrentThread().ClearInCacheMaintenanceOperation(); __asm__ __volatile__("" ::: "memory"); };
const uintptr_t start = reinterpret_cast<uintptr_t>(addr);
const uintptr_t end = start + size;
uintptr_t aligned_start = util::AlignDown(start, DataCacheLineSize);
@@ -444,7 +405,11 @@ namespace ams::kern::arch::arm64::cpu {
}
Result StoreDataCache(const void *addr, size_t size) {
KScopedCoreMigrationDisable dm;
/* Mark ourselves as in a cache maintenance operation, and prevent re-ordering. */
__asm__ __volatile__("" ::: "memory");
GetCurrentThread().SetInCacheMaintenanceOperation();
ON_SCOPE_EXIT { GetCurrentThread().ClearInCacheMaintenanceOperation(); __asm__ __volatile__("" ::: "memory"); };
const uintptr_t start = util::AlignDown(reinterpret_cast<uintptr_t>(addr), DataCacheLineSize);
const uintptr_t end = util::AlignUp( reinterpret_cast<uintptr_t>(addr) + size, DataCacheLineSize);
@@ -452,26 +417,17 @@ namespace ams::kern::arch::arm64::cpu {
}
Result FlushDataCache(const void *addr, size_t size) {
KScopedCoreMigrationDisable dm;
/* Mark ourselves as in a cache maintenance operation, and prevent re-ordering. */
__asm__ __volatile__("" ::: "memory");
GetCurrentThread().SetInCacheMaintenanceOperation();
ON_SCOPE_EXIT { GetCurrentThread().ClearInCacheMaintenanceOperation(); __asm__ __volatile__("" ::: "memory"); };
const uintptr_t start = util::AlignDown(reinterpret_cast<uintptr_t>(addr), DataCacheLineSize);
const uintptr_t end = util::AlignUp( reinterpret_cast<uintptr_t>(addr) + size, DataCacheLineSize);
R_RETURN(FlushDataCacheRange(start, end));
}
Result InvalidateInstructionCache(void *addr, size_t size) {
KScopedCoreMigrationDisable dm;
const uintptr_t start = util::AlignDown(reinterpret_cast<uintptr_t>(addr), InstructionCacheLineSize);
const uintptr_t end = util::AlignUp( reinterpret_cast<uintptr_t>(addr) + size, InstructionCacheLineSize);
R_TRY(InvalidateInstructionCacheRange(start, end));
/* Request the interrupt helper to perform an instruction memory barrier. */
g_cache_operation_handler.RequestOperation(KCacheHelperInterruptHandler::Operation::InstructionMemoryBarrier);
R_SUCCEED();
}
void InvalidateEntireInstructionCache() {
KScopedCoreMigrationDisable dm;