OmniNX/Atmosphere
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

kern: fuck the KPolice^H^H^H^H^H^HPageGroups

Browse Source
This commit is contained in:
Michael Scire
2021-04-07 17:07:01 -07:00
committed by SciresM
parent dc7862882f
commit 96937a611d
8 changed files with 493 additions and 231 deletions
Download Patch File Download Diff File Expand all files Collapse all files

280
libraries/libmesosphere/source/board/nintendo/nx/kern_k_device_page_table.cpp
View File

@@ -216,6 +216,12 @@ namespace ams::kern::board::nintendo::nx {
return (m_value & (1u << n));
}
template<Bit... Bits>
constexpr ALWAYS_INLINE u32 SelectBits() const {
constexpr u32 Mask = ((1u << Bits) | ...);
return m_value & Mask;
}
constexpr ALWAYS_INLINE bool GetBit(Bit n) const {
return this->SelectBit(n) != 0;
}
@@ -242,12 +248,14 @@ namespace ams::kern::board::nintendo::nx {
constexpr ALWAYS_INLINE bool IsNonSecure() const { return this->GetBit(Bit_NonSecure); }
constexpr ALWAYS_INLINE bool IsWriteable() const { return this->GetBit(Bit_Writeable); }
constexpr ALWAYS_INLINE bool IsReadable() const { return this->GetBit(Bit_Readable); }
constexpr ALWAYS_INLINE bool IsValid() const { return this->IsWriteable() || this->IsReadable(); }
constexpr ALWAYS_INLINE bool IsValid() const { return this->SelectBits<Bit_Readable, Bit_Writeable>(); }
constexpr ALWAYS_INLINE u32 GetAttributes() const { return this->SelectBit(Bit_NonSecure) | this->SelectBit(Bit_Writeable) | this->SelectBit(Bit_Readable); }
constexpr ALWAYS_INLINE u32 GetAttributes() const { return this->SelectBits<Bit_Readable, Bit_Writeable, Bit_NonSecure>(); }
constexpr ALWAYS_INLINE KPhysicalAddress GetPhysicalAddress() const { return (static_cast<u64>(m_value) << DevicePageBits) & PhysicalAddressMask; }
ALWAYS_INLINE void InvalidateAttributes() { this->SetValue(m_value & ~(0xCu << 28)); }
ALWAYS_INLINE void Invalidate() { this->SetValue(0); }
};
@@ -847,7 +855,7 @@ namespace ams::kern::board::nintendo::nx {
}
/* Forcibly unmap all pages. */
this->UnmapImpl(0, (1ul << DeviceVirtualAddressBits), true);
this->UnmapImpl(0, (1ul << DeviceVirtualAddressBits), false);
/* Release all asids. */
for (size_t i = 0; i < TableCount; ++i) {
@@ -1117,12 +1125,11 @@ namespace ams::kern::board::nintendo::nx {
return ResultSuccess();
}
Result KDevicePageTable::MapImpl(size_t *out_mapped_size, s32 &num_pt, s32 max_pt, const KPageGroup &pg, KDeviceVirtualAddress device_address, ams::svc::MemoryPermission device_perm) {
Result KDevicePageTable::MapImpl(size_t *out_mapped_size, s32 &num_pt, s32 max_pt, KProcessPageTable *page_table, KProcessAddress process_address, size_t size, KDeviceVirtualAddress device_address, ams::svc::MemoryPermission device_perm, bool is_aligned) {
/* Clear the output size. */
*out_mapped_size = 0;
/* Get the size, and validate the address. */
const u64 size = pg.GetNumPages() * PageSize;
MESOSPHERE_ASSERT((device_address & ~DeviceVirtualAddressMask) == 0);
MESOSPHERE_ASSERT(((device_address + size - 1) & ~DeviceVirtualAddressMask) == 0);
@@ -1130,28 +1137,33 @@ namespace ams::kern::board::nintendo::nx {
R_UNLESS(this->IsFree(device_address, size), svc::ResultInvalidCurrentMemory());
/* Ensure that if we fail, we unmap anything we mapped. */
auto unmap_guard = SCOPE_GUARD { this->UnmapImpl(device_address, size, true); };
auto unmap_guard = SCOPE_GUARD { this->UnmapImpl(device_address, size, false); };
/* Iterate, mapping device pages. */
KDeviceVirtualAddress cur_addr = device_address;
for (auto it = pg.begin(); it != pg.end(); ++it) {
/* Require that we be able to map the device page. */
R_UNLESS(IsHeapVirtualAddress(it->GetAddress()), svc::ResultInvalidCurrentMemory());
while (true) {
/* Get the current contiguous range. */
KPageTableBase::MemoryRange contig_range = {};
R_TRY(page_table->OpenMemoryRangeForMapDeviceAddressSpace(std::addressof(contig_range), process_address + *out_mapped_size, size - *out_mapped_size, ConvertToKMemoryPermission(device_perm), is_aligned));
/* Get the physical address for the page. */
const KPhysicalAddress phys_addr = GetHeapPhysicalAddress(it->GetAddress());
/* Ensure we close the range when we're done. */
ON_SCOPE_EXIT { contig_range.Close(); };
/* Map the device page. */
const u64 block_size = it->GetSize();
size_t mapped_size = 0;
R_TRY(this->MapDevicePage(std::addressof(mapped_size), num_pt, max_pt, phys_addr, block_size, cur_addr, device_perm));
R_TRY(this->MapDevicePage(std::addressof(mapped_size), num_pt, max_pt, GetHeapPhysicalAddress(contig_range.address), contig_range.size, cur_addr, device_perm));
/* Advance. */
cur_addr += block_size;
cur_addr += contig_range.size;
*out_mapped_size += mapped_size;
/* If we didn't map as much as we wanted, break. */
if (mapped_size < block_size) {
if (mapped_size < contig_range.size) {
break;
}
/* Similarly, if we're done, break. */
if (*out_mapped_size >= size) {
break;
}
}
@@ -1186,8 +1198,6 @@ namespace ams::kern::board::nintendo::nx {
/* Check if there's nothing mapped at l1. */
if (l1 == nullptr || !l1[l1_index].IsValid()) {
MESOSPHERE_ASSERT(force);
const size_t remaining_in_entry = (PageTableSize / sizeof(PageTableEntry)) - l2_index;
const size_t map_count = std::min<size_t>(remaining_in_entry, remaining / DevicePageSize);
@@ -1201,30 +1211,12 @@ namespace ams::kern::board::nintendo::nx {
const size_t remaining_in_entry = (PageTableSize / sizeof(PageTableEntry)) - l2_index;
const size_t map_count = std::min<size_t>(remaining_in_entry, remaining / DevicePageSize);
size_t num_closed = 0;
bool invalidated_tlb = false;
/* Invalidate the attributes of all entries. */
for (size_t i = 0; i < map_count; ++i) {
if (l2[l2_index + i].IsValid()) {
/* Get the physical address. */
const KPhysicalAddress phys_addr = l2[l2_index + i].GetPhysicalAddress();
MESOSPHERE_ASSERT(IsHeapPhysicalAddress(phys_addr));
/* Invalidate the entry. */
l2[l2_index + i].Invalidate();
l2[l2_index + i].InvalidateAttributes();
++num_closed;
/* Try to add the page to the group. */
if (R_FAILED(pg.AddBlock(GetHeapVirtualAddress(phys_addr), DevicePageSize / PageSize))) {
/* If we can't add it for deferred close, close it now. */
cpu::StoreDataCache(std::addressof(l2[l2_index + i]), sizeof(PageTableEntry));
InvalidatePtc(GetPageTablePhysicalAddress(KVirtualAddress(std::addressof(l2[l2_index + i]))));
SmmuSynchronizationBarrier();
/* Close the page's reference. */
mm.Close(GetHeapVirtualAddress(phys_addr), 1);
}
} else {
MESOSPHERE_ASSERT(force);
}
}
cpu::StoreDataCache(std::addressof(l2[l2_index]), map_count * sizeof(PageTableEntry));
@@ -1235,6 +1227,38 @@ namespace ams::kern::board::nintendo::nx {
}
SmmuSynchronizationBarrier();
/* Close the memory manager's references to the pages. */
{
KPhysicalAddress contig_phys_addr = Null<KPhysicalAddress>;
size_t contig_count = 0;
for (size_t i = 0; i < map_count; ++i) {
/* Get the physical address. */
const KPhysicalAddress phys_addr = l2[l2_index + i].GetPhysicalAddress();
MESOSPHERE_ASSERT(IsHeapPhysicalAddress(phys_addr));
/* Fully invalidate the entry. */
l2[l2_index + i].Invalidate();
if (contig_count == 0) {
/* Ensure that our address/count is valid. */
contig_phys_addr = phys_addr;
contig_count = contig_phys_addr != Null<KPhysicalAddress> ? 1 : 0;
} else if (phys_addr == Null<KPhysicalAddress> || phys_addr != (contig_phys_addr + (contig_count * DevicePageSize))) {
/* If we're no longer contiguous, close the range we've been building. */
mm.Close(GetHeapVirtualAddress(contig_phys_addr), (contig_count * DevicePageSize) / PageSize);
contig_phys_addr = phys_addr;
contig_count = contig_phys_addr != Null<KPhysicalAddress> ? 1 : 0;
} else {
++contig_count;
}
}
if (contig_count > 0) {
mm.Close(GetHeapVirtualAddress(contig_phys_addr), (contig_count * DevicePageSize) / PageSize);
}
}
/* Close the pages. */
if (ptm.Close(KVirtualAddress(l2), num_closed)) {
/* Invalidate the l1 entry. */
@@ -1243,22 +1267,12 @@ namespace ams::kern::board::nintendo::nx {
/* Synchronize. */
InvalidatePtc(GetPageTablePhysicalAddress(KVirtualAddress(std::addressof(l1[l1_index]))));
InvalidateTlbSection(m_table_asids[l0_index], address);
SmmuSynchronizationBarrier();
/* We invalidated the tlb. */
invalidated_tlb = true;
/* Free the l2 page. */
ptm.Free(KVirtualAddress(l2));
}
/* Invalidate the tlb if we haven't already. */
if (!invalidated_tlb) {
InvalidateTlbSection(m_table_asids[l0_index], address);
SmmuSynchronizationBarrier();
}
/* Advance. */
address += map_count * DevicePageSize;
remaining -= map_count * DevicePageSize;
@@ -1287,114 +1301,158 @@ namespace ams::kern::board::nintendo::nx {
remaining -= DeviceLargePageSize;
}
}
/* Close references to the pages in the group. */
pg.Close();
}
Result KDevicePageTable::MakePageGroup(KPageGroup *out, KDeviceVirtualAddress address, u64 size) const {
MESOSPHERE_ASSERT((address & ~DeviceVirtualAddressMask) == 0);
MESOSPHERE_ASSERT(((address + size - 1) & ~DeviceVirtualAddressMask) == 0);
bool KDevicePageTable::Compare(KProcessPageTable *page_table, KProcessAddress process_address, size_t size, KDeviceVirtualAddress device_address) const {
MESOSPHERE_ASSERT((device_address & ~DeviceVirtualAddressMask) == 0);
MESOSPHERE_ASSERT(((device_address + size - 1) & ~DeviceVirtualAddressMask) == 0);
/* We need to traverse the ranges that make up our mapping, to make sure they're all good. Start by getting a contiguous range. */
KPageTableBase::MemoryRange contig_range = {};
if (R_FAILED(page_table->OpenMemoryRangeForUnmapDeviceAddressSpace(std::addressof(contig_range), process_address, size))) {
return false;
}
/* Ensure that we close the range when we're done. */
bool range_open = true;
ON_SCOPE_EXIT { if (range_open) { contig_range.Close(); } };
/* Walk the directory. */
u64 remaining = size;
bool first = true;
u32 attr = 0;
while (remaining > 0) {
const size_t l0_index = (address / DeviceRegionSize);
const size_t l1_index = (address % DeviceRegionSize) / DeviceLargePageSize;
const size_t l2_index = (address % DeviceLargePageSize) / DevicePageSize;
KProcessAddress cur_process_address = process_address;
size_t remaining_size = size;
KPhysicalAddress cur_phys_address = GetHeapPhysicalAddress(contig_range.address);
size_t remaining_in_range = contig_range.size;
bool first = true;
u32 first_attr = 0;
while (remaining_size > 0) {
/* Convert the device address to a series of indices. */
const size_t l0_index = (device_address / DeviceRegionSize);
const size_t l1_index = (device_address % DeviceRegionSize) / DeviceLargePageSize;
const size_t l2_index = (device_address % DeviceLargePageSize) / DevicePageSize;
/* Get and validate l1. */
const PageDirectoryEntry *l1 = GetPointer<PageDirectoryEntry>(m_tables[l0_index]);
R_UNLESS(l1 != nullptr, svc::ResultInvalidCurrentMemory());
R_UNLESS(l1[l1_index].IsValid(), svc::ResultInvalidCurrentMemory());
if (!(l1 != nullptr && l1[l1_index].IsValid())) {
return false;
}
if (l1[l1_index].IsTable()) {
/* We're acting on an l2 entry. */
const PageTableEntry *l2 = GetPointer<PageTableEntry>(GetPageTableVirtualAddress(l1[l1_index].GetPhysicalAddress()));
/* Determine the number of pages to check. */
const size_t remaining_in_entry = (PageTableSize / sizeof(PageTableEntry)) - l2_index;
const size_t map_count = std::min<size_t>(remaining_in_entry, remaining / DevicePageSize);
const size_t map_count = std::min<size_t>(remaining_in_entry, remaining_size / DevicePageSize);
/* Check each page. */
for (size_t i = 0; i < map_count; ++i) {
/* Ensure the l2 entry is valid. */
R_UNLESS(l2[l2_index + i].IsValid(), svc::ResultInvalidCurrentMemory());
/* Get the physical address. */
const KPhysicalAddress phys_addr = l2[l2_index + i].GetPhysicalAddress();
MESOSPHERE_ASSERT(IsHeapPhysicalAddress(phys_addr));
/* Add to the group. */
R_TRY(out->AddBlock(GetHeapVirtualAddress(phys_addr), DevicePageSize / PageSize));
/* If this is our first entry, get the attribute. */
if (first) {
attr = l2[l2_index + i].GetAttributes();
first = false;
} else {
/* Validate the attributes match the first entry. */
R_UNLESS(l2[l2_index + i].GetAttributes() == attr, svc::ResultInvalidCurrentMemory());
if (!l2[l2_index + i].IsValid()) {
return false;
}
/* Check that the attributes match the first attributes we encountered. */
const u32 cur_attr = l2[l2_index + i].GetAttributes();
if (!first && cur_attr != first_attr) {
return false;
}
/* If there's nothing remaining in the range, refresh the range. */
if (remaining_in_range == 0) {
contig_range.Close();
range_open = false;
if (R_FAILED(page_table->OpenMemoryRangeForUnmapDeviceAddressSpace(std::addressof(contig_range), cur_process_address, remaining_size))) {
return false;
}
range_open = true;
cur_phys_address = GetHeapPhysicalAddress(contig_range.address);
remaining_in_range = contig_range.size;
}
/* Check that the physical address is expected. */
if (l2[l2_index + i].GetPhysicalAddress() != cur_phys_address) {
return false;
}
/* Advance. */
cur_phys_address += DevicePageSize;
cur_process_address += DevicePageSize;
remaining_size -= DevicePageSize;
remaining_in_range -= DevicePageSize;
first = false;
first_attr = cur_attr;
}
/* Advance. */
address += DevicePageSize * map_count;
remaining -= DevicePageSize * map_count;
/* Advance the device address. */
device_address += map_count * DevicePageSize;
} else {
/* We're acting on an l1 entry. */
R_UNLESS(l2_index == 0, svc::ResultInvalidCurrentMemory());
R_UNLESS(remaining >= DeviceLargePageSize, svc::ResultInvalidCurrentMemory());
if (!(l2_index == 0 && remaining_size >= DeviceLargePageSize)) {
return false;
}
/* Get the physical address. */
const KPhysicalAddress phys_addr = l1[l1_index].GetPhysicalAddress();
MESOSPHERE_ASSERT(IsHeapPhysicalAddress(phys_addr));
/* Check that the attributes match the first attributes we encountered. */
const u32 cur_attr = l1[l1_index].GetAttributes();
if (!first && cur_attr != first_attr) {
return false;
}
/* Add to the group. */
R_TRY(out->AddBlock(GetHeapVirtualAddress(phys_addr), DeviceLargePageSize / PageSize));
/* If there's nothing remaining in the range, refresh the range. */
if (remaining_in_range == 0) {
contig_range.Close();
/* If this is our first entry, get the attribute. */
if (first) {
attr = l1[l1_index].GetAttributes();
first = false;
} else {
/* Validate the attributes match the first entry. */
R_UNLESS(l1[l1_index].GetAttributes() == attr, svc::ResultInvalidCurrentMemory());
range_open = false;
if (R_FAILED(page_table->OpenMemoryRangeForUnmapDeviceAddressSpace(std::addressof(contig_range), cur_process_address, remaining_size))) {
return false;
}
range_open = true;
cur_phys_address = GetHeapPhysicalAddress(contig_range.address);
remaining_in_range = contig_range.size;
}
/* Check that the physical address is expected, and there's enough in the range. */
if (remaining_in_range < DeviceLargePageSize || l1[l1_index].GetPhysicalAddress() != cur_phys_address) {
return false;
}
/* Advance. */
address += DeviceLargePageSize;
remaining -= DeviceLargePageSize;
cur_phys_address += DeviceLargePageSize;
cur_process_address += DeviceLargePageSize;
remaining_size -= DeviceLargePageSize;
remaining_in_range -= DeviceLargePageSize;
first = false;
first_attr = cur_attr;
/* Advance the device address. */
device_address += DeviceLargePageSize;
}
}
return ResultSuccess();
/* The range is valid! */
return true;
}
bool KDevicePageTable::Compare(const KPageGroup &compare_pg, KDeviceVirtualAddress device_address) const {
/* Check whether the page group we expect for the virtual address matches the page group we're validating. */
KPageGroup calc_pg(std::addressof(Kernel::GetBlockInfoManager()));
return (R_SUCCEEDED(this->MakePageGroup(std::addressof(calc_pg), device_address, compare_pg.GetNumPages() * PageSize))) &&
calc_pg.IsEquivalentTo(compare_pg);
}
Result KDevicePageTable::Map(size_t *out_mapped_size, const KPageGroup &pg, KDeviceVirtualAddress device_address, ams::svc::MemoryPermission device_perm, bool refresh_mappings) {
Result KDevicePageTable::Map(size_t *out_mapped_size, KProcessPageTable *page_table, KProcessAddress process_address, size_t size, KDeviceVirtualAddress device_address, ams::svc::MemoryPermission device_perm, bool refresh_mappings) {
/* Clear the output size. */
*out_mapped_size = 0;
/* Map the pages. */
s32 num_pt = 0;
return this->MapImpl(out_mapped_size, num_pt, refresh_mappings ? 1 : std::numeric_limits<s32>::max(), pg, device_address, device_perm);
return this->MapImpl(out_mapped_size, num_pt, refresh_mappings ? 1 : std::numeric_limits<s32>::max(), page_table, process_address, size, device_address, device_perm, refresh_mappings);
}
Result KDevicePageTable::Unmap(const KPageGroup &pg, KDeviceVirtualAddress device_address) {
Result KDevicePageTable::Unmap(KProcessPageTable *page_table, KProcessAddress process_address, size_t size, KDeviceVirtualAddress device_address) {
/* Validate address/size. */
const size_t size = pg.GetNumPages() * PageSize;
MESOSPHERE_ASSERT((device_address & ~DeviceVirtualAddressMask) == 0);
MESOSPHERE_ASSERT(((device_address + size - 1) & ~DeviceVirtualAddressMask) == 0);
/* Ensure the page group is correct. */
R_UNLESS(this->Compare(pg, device_address), svc::ResultInvalidCurrentMemory());
R_UNLESS(this->Compare(page_table, process_address, size, device_address), svc::ResultInvalidCurrentMemory());
/* Unmap the pages. */
this->UnmapImpl(device_address, size, false);