OmniNX/Atmosphere
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

kern: implement smmu init

Browse Source
This commit is contained in:
Michael Scire
2020-02-15 00:00:35 -08:00
parent 2c496e94d5
commit 30d6b359f9
21 changed files with 1368 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

92
libraries/libmesosphere/source/arch/arm64/kern_cpu.cpp
View File

@@ -23,6 +23,13 @@ namespace ams::kern::arch::arm64::cpu {
namespace {
class KScopedCoreMigrationDisable {
public:
ALWAYS_INLINE KScopedCoreMigrationDisable() { GetCurrentThread().DisableCoreMigration(); }
ALWAYS_INLINE ~KScopedCoreMigrationDisable() { GetCurrentThread().EnableCoreMigration(); }
};
/* Nintendo registers a handler for a SGI on thread termination, but does not handle anything. */
/* This is sufficient, because post-interrupt scheduling is all they really intend to occur. */
class KThreadTerminationInterruptHandler : public KInterruptHandler {
@@ -284,6 +291,38 @@ namespace ams::kern::arch::arm64::cpu {
__asm__ __volatile__("wfe" ::: "memory");
}
ALWAYS_INLINE Result InvalidateDataCacheRange(uintptr_t start, uintptr_t end) {
MESOSPHERE_ASSERT(util::IsAligned(start, DataCacheLineSize));
MESOSPHERE_ASSERT(util::IsAligned(end, DataCacheLineSize));
R_UNLESS(arm64::InvalidateDataCache(start, end), svc::ResultInvalidCurrentMemory());
DataSynchronizationBarrier();
return ResultSuccess();
}
ALWAYS_INLINE Result StoreDataCacheRange(uintptr_t start, uintptr_t end) {
MESOSPHERE_ASSERT(util::IsAligned(start, DataCacheLineSize));
MESOSPHERE_ASSERT(util::IsAligned(end, DataCacheLineSize));
R_UNLESS(arm64::StoreDataCache(start, end), svc::ResultInvalidCurrentMemory());
DataSynchronizationBarrier();
return ResultSuccess();
}
ALWAYS_INLINE Result FlushDataCacheRange(uintptr_t start, uintptr_t end) {
MESOSPHERE_ASSERT(util::IsAligned(start, DataCacheLineSize));
MESOSPHERE_ASSERT(util::IsAligned(end, DataCacheLineSize));
R_UNLESS(arm64::FlushDataCache(start, end), svc::ResultInvalidCurrentMemory());
DataSynchronizationBarrier();
return ResultSuccess();
}
ALWAYS_INLINE Result InvalidateInstructionCacheRange(uintptr_t start, uintptr_t end) {
MESOSPHERE_ASSERT(util::IsAligned(start, InstructionCacheLineSize));
MESOSPHERE_ASSERT(util::IsAligned(end, InstructionCacheLineSize));
R_UNLESS(arm64::InvalidateInstructionCache(start, end), svc::ResultInvalidCurrentMemory());
EnsureInstructionConsistency();
return ResultSuccess();
}
}
void FlushEntireDataCacheSharedForInit() {
@@ -294,6 +333,59 @@ namespace ams::kern::arch::arm64::cpu {
return PerformCacheOperationBySetWayLocal<true>(FlushDataCacheLineBySetWayImpl);
}
Result InvalidateDataCache(void *addr, size_t size) {
KScopedCoreMigrationDisable dm;
const uintptr_t start = reinterpret_cast<uintptr_t>(addr);
const uintptr_t end = start + size;
uintptr_t aligned_start = util::AlignDown(start, DataCacheLineSize);
uintptr_t aligned_end = util::AlignUp(end, DataCacheLineSize);
if (aligned_start != start) {
R_TRY(FlushDataCacheRange(aligned_start, aligned_start + DataCacheLineSize));
aligned_start += DataCacheLineSize;
}
if (aligned_start < aligned_end && (aligned_end != end)) {
aligned_end -= DataCacheLineSize;
R_TRY(FlushDataCacheRange(aligned_end, aligned_end + DataCacheLineSize));
}
if (aligned_start < aligned_end) {
R_TRY(InvalidateDataCacheRange(aligned_start, aligned_end));
}
return ResultSuccess();
}
Result StoreDataCache(const void *addr, size_t size) {
KScopedCoreMigrationDisable dm;
const uintptr_t start = util::AlignDown(reinterpret_cast<uintptr_t>(addr), DataCacheLineSize);
const uintptr_t end = util::AlignUp( reinterpret_cast<uintptr_t>(addr), DataCacheLineSize);
return StoreDataCacheRange(start, end);
}
Result FlushDataCache(const void *addr, size_t size) {
KScopedCoreMigrationDisable dm;
const uintptr_t start = util::AlignDown(reinterpret_cast<uintptr_t>(addr), DataCacheLineSize);
const uintptr_t end = util::AlignUp( reinterpret_cast<uintptr_t>(addr), DataCacheLineSize);
return FlushDataCacheRange(start, end);
}
Result InvalidateInstructionCache(void *addr, size_t size) {
KScopedCoreMigrationDisable dm;
const uintptr_t start = util::AlignDown(reinterpret_cast<uintptr_t>(addr), InstructionCacheLineSize);
const uintptr_t end = util::AlignUp( reinterpret_cast<uintptr_t>(addr), InstructionCacheLineSize);
R_TRY(InvalidateInstructionCacheRange(start, end));
/* Request the interrupt helper to invalidate, too. */
g_cache_operation_handler.RequestOperation(KCacheHelperInterruptHandler::Operation::InvalidateInstructionCache);
return ResultSuccess();
}
void InitializeInterruptThreads(s32 core_id) {
/* Initialize the cache operation handler. */
g_cache_operation_handler.Initialize(core_id);

109
libraries/libmesosphere/source/arch/arm64/kern_userspace_memory_access_asm.s Normal file
View File

@@ -0,0 +1,109 @@
/*
* Copyright (c) 2018-2020 Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
/* ams::kern::arch::arm64::UserspaceMemoryAccessFunctionAreaBegin() */
.section .text._ZN3ams4kern4arch5arm6438UserspaceMemoryAccessFunctionAreaBeginEv, "ax", %progbits
.global _ZN3ams4kern4arch5arm6438UserspaceMemoryAccessFunctionAreaBeginEv
.type _ZN3ams4kern4arch5arm6438UserspaceMemoryAccessFunctionAreaBeginEv, %function
_ZN3ams4kern4arch5arm6438UserspaceMemoryAccessFunctionAreaBeginEv:
/* NOTE: This is not a real function, and only exists as a label for safety. */
/* ================ All Userspace Memory Functions after this line. ================ */
/* ams::kern::arch::arm64::StoreDataCache(uintptr_t start, uintptr_t end) */
.section .text._ZN3ams4kern4arch5arm6414StoreDataCacheEmm, "ax", %progbits
.global _ZN3ams4kern4arch5arm6414StoreDataCacheEmm
.type _ZN3ams4kern4arch5arm6414StoreDataCacheEmm, %function
_ZN3ams4kern4arch5arm6414StoreDataCacheEmm:
/* Check if we have any work to do. */
cmp x1, x0
b.eq 2f
1: /* Loop, storing each cache line. */
dc cvac, x0
add x0, x0, #0x40
cmp x1, x0
b.ne 1b
2: /* We're done! */
mov x0, #1
ret
/* ams::kern::arch::arm64::FlushDataCache(uintptr_t start, uintptr_t end) */
.section .text._ZN3ams4kern4arch5arm6414FlushDataCacheEmm, "ax", %progbits
.global _ZN3ams4kern4arch5arm6414FlushDataCacheEmm
.type _ZN3ams4kern4arch5arm6414FlushDataCacheEmm, %function
_ZN3ams4kern4arch5arm6414FlushDataCacheEmm:
/* Check if we have any work to do. */
cmp x1, x0
b.eq 2f
1: /* Loop, flushing each cache line. */
dc civac, x0
add x0, x0, #0x40
cmp x1, x0
b.ne 1b
2: /* We're done! */
mov x0, #1
ret
/* ams::kern::arch::arm64::InvalidateDataCache(uintptr_t start, uintptr_t end) */
.section .text._ZN3ams4kern4arch5arm6419InvalidateDataCacheEmm, "ax", %progbits
.global _ZN3ams4kern4arch5arm6419InvalidateDataCacheEmm
.type _ZN3ams4kern4arch5arm6419InvalidateDataCacheEmm, %function
_ZN3ams4kern4arch5arm6419InvalidateDataCacheEmm:
/* Check if we have any work to do. */
cmp x1, x0
b.eq 2f
1: /* Loop, invalidating each cache line. */
dc ivac, x0
add x0, x0, #0x40
cmp x1, x0
b.ne 1b
2: /* We're done! */
mov x0, #1
ret
/* ams::kern::arch::arm64::InvalidateInstructionCache(uintptr_t start, uintptr_t end) */
.section .text._ZN3ams4kern4arch5arm6426InvalidateInstructionCacheEmm, "ax", %progbits
.global _ZN3ams4kern4arch5arm6426InvalidateInstructionCacheEmm
.type _ZN3ams4kern4arch5arm6426InvalidateInstructionCacheEmm, %function
_ZN3ams4kern4arch5arm6426InvalidateInstructionCacheEmm:
/* Check if we have any work to do. */
cmp x1, x0
b.eq 2f
1: /* Loop, invalidating each cache line. */
ic ivau, x0
add x0, x0, #0x40
cmp x1, x0
b.ne 1b
2: /* We're done! */
mov x0, #1
ret
/* ================ All Userspace Memory Functions before this line. ================ */
/* ams::kern::arch::arm64::UserspaceMemoryAccessFunctionAreaEnd() */
.section .text._ZN3ams4kern4arch5arm6436UserspaceMemoryAccessFunctionAreaEndEv, "ax", %progbits
.global _ZN3ams4kern4arch5arm6436UserspaceMemoryAccessFunctionAreaEndEv
.type _ZN3ams4kern4arch5arm6436UserspaceMemoryAccessFunctionAreaEndEv, %function
_ZN3ams4kern4arch5arm6436UserspaceMemoryAccessFunctionAreaEndEv:
/* NOTE: This is not a real function, and only exists as a label for safety. */