kern: implement more of KMemoryManager through KPageBuffer slab init

libraries/libmesosphere/source/init/kern_init_slab_setup.cpp

@@ -135,6 +135,26 @@ namespace ams::kern::init {
         return size;
     }
 
+    void InitializeKPageBufferSlabHeap() {
+        const auto &counts = GetSlabResourceCounts();
+        const size_t num_pages = counts.num_KProcess + counts.num_KThread + (counts.num_KProcess + counts.num_KThread) / 8;
+        const size_t slab_size = num_pages * PageSize;
+
+        /* Reserve memory from the system resource limit. */
+        MESOSPHERE_ABORT_UNLESS(Kernel::GetSystemResourceLimit().Reserve(ams::svc::LimitableResource_PhysicalMemoryMax, slab_size));
+
+        /* Allocate memory for the slab. */
+        constexpr auto AllocateOption = KMemoryManager::EncodeOption(KMemoryManager::Pool_System, KMemoryManager::Direction_FromFront);
+        const KVirtualAddress slab_address = Kernel::GetMemoryManager().AllocateContinuous(num_pages, 1, AllocateOption);
+        MESOSPHERE_ABORT_UNLESS(slab_address != Null<KVirtualAddress>);
+
+        /* Open references to the slab. */
+        Kernel::GetMemoryManager().Open(slab_address, num_pages);
+
+        /* Initialize the slabheap. */
+        KPageBuffer::InitializeSlabHeap(GetVoidPointer(slab_address), slab_size);
+    }
+
     void InitializeSlabHeaps() {
         /* Get the start of the slab region, since that's where we'll be working. */
         KVirtualAddress address = KMemoryLayout::GetSlabRegionAddress();

libraries/libmesosphere/source/kern_k_memory_manager.cpp

@@ -87,6 +87,57 @@ namespace ams::kern {
         }
     }
 
+    KVirtualAddress KMemoryManager::AllocateContinuous(size_t num_pages, size_t align_pages, u32 option) {
+        /* Early return if we're allocating no pages. */
+        if (num_pages == 0) {
+            return Null<KVirtualAddress>;
+        }
+
+        /* Lock the pool that we're allocating from. */
+        const auto [pool, dir] = DecodeOption(option);
+        KScopedLightLock lk(this->pool_locks[pool]);
+
+        /* Choose a heap based on our page size request. */
+        const s32 heap_index = KPageHeap::GetAlignedBlockIndex(num_pages, align_pages);
+
+        /* Loop, trying to iterate from each block. */
+        Impl *chosen_manager = nullptr;
+        KVirtualAddress allocated_block = Null<KVirtualAddress>;
+        if (dir == Direction_FromBack) {
+            for (chosen_manager = this->pool_managers_tail[pool]; chosen_manager != nullptr; chosen_manager = chosen_manager->GetPrev()) {
+                allocated_block = chosen_manager->AllocateBlock(heap_index);
+                if (allocated_block != Null<KVirtualAddress>) {
+                    break;
+                }
+            }
+        } else {
+            for (chosen_manager = this->pool_managers_head[pool]; chosen_manager != nullptr; chosen_manager = chosen_manager->GetNext()) {
+                allocated_block = chosen_manager->AllocateBlock(heap_index);
+                if (allocated_block != Null<KVirtualAddress>) {
+                    break;
+                }
+            }
+        }
+
+        /* If we failed to allocate, quit now. */
+        if (allocated_block == Null<KVirtualAddress>) {
+            return Null<KVirtualAddress>;
+        }
+
+        /* If we allocated more than we need, free some. */
+        const size_t allocated_pages = KPageHeap::GetBlockNumPages(heap_index);
+        if (allocated_pages > num_pages) {
+            chosen_manager->Free(allocated_block + num_pages * PageSize, allocated_pages - num_pages);
+        }
+
+        /* Maintain the optimized memory bitmap, if we should. */
+        if (this->has_optimized_process[pool]) {
+            chosen_manager->TrackAllocationForOptimizedProcess(allocated_block, num_pages);
+        }
+
+        return allocated_block;
+    }
+
     size_t KMemoryManager::Impl::Initialize(const KMemoryRegion *region, Pool p, KVirtualAddress metadata, KVirtualAddress metadata_end) {
         /* Calculate metadata sizes. */
         const size_t ref_count_size      = (region->GetSize() / PageSize) * sizeof(u16);
@@ -107,9 +158,25 @@ namespace ams::kern {
         /* Initialize the manager's KPageHeap. */
         this->heap.Initialize(region->GetAddress(), region->GetSize(), metadata + manager_size, page_heap_size);
 
+        /* Free the memory to the heap. */
+        this->heap.Free(region->GetAddress(), region->GetSize() / PageSize);
+
+        /* Update the heap's used size. */
+        this->heap.UpdateUsedSize();
+
         return total_metadata_size;
     }
 
+    void KMemoryManager::Impl::TrackAllocationForOptimizedProcess(KVirtualAddress block, size_t num_pages) {
+        size_t offset = this->heap.GetPageOffset(block);
+        const size_t last = offset + num_pages - 1;
+        u64 *optimize_map = GetPointer<u64>(this->metadata_region);
+        while (offset <= last) {
+            optimize_map[offset / BITSIZEOF(u64)] &= ~(u64(1) << (offset % BITSIZEOF(u64)));
+            offset++;
+        }
+    }
+
     size_t KMemoryManager::Impl::CalculateMetadataOverheadSize(size_t region_size) {
         const size_t ref_count_size     = (region_size / PageSize) * sizeof(u16);
         const size_t optimize_map_size  = (util::AlignUp((region_size / PageSize), BITSIZEOF(u64)) / BITSIZEOF(u64)) * sizeof(u64);

libraries/libmesosphere/source/kern_k_page_heap.cpp

@@ -41,6 +41,87 @@ namespace ams::kern {
         MESOSPHERE_ABORT_UNLESS(KVirtualAddress(cur_bitmap_storage) <= metadata_end);
     }
 
+    size_t KPageHeap::GetNumFreePages() const {
+        size_t num_free = 0;
+
+        for (size_t i = 0; i < this->num_blocks; i++) {
+            num_free += this->blocks[i].GetNumFreePages();
+        }
+
+        return num_free;
+    }
+
+    KVirtualAddress KPageHeap::AllocateBlock(s32 index) {
+        const size_t needed_size = this->blocks[index].GetSize();
+
+        for (s32 i = index; i < static_cast<s32>(this->num_blocks); i++) {
+            if (const KVirtualAddress addr = this->blocks[index].PopBlock(); addr != Null<KVirtualAddress>) {
+                if (const size_t allocated_size = this->blocks[index].GetSize(); allocated_size > needed_size) {
+                    this->Free(addr + needed_size, (allocated_size - needed_size) / PageSize);
+                }
+                return addr;
+            }
+        }
+
+        return Null<KVirtualAddress>;
+    }
+
+    void KPageHeap::FreeBlock(KVirtualAddress block, s32 index) {
+        do {
+            block = this->blocks[index++].PushBlock(block);
+        } while (block != Null<KVirtualAddress>);
+    }
+
+    void KPageHeap::Free(KVirtualAddress addr, size_t num_pages) {
+        /* Freeing no pages is a no-op. */
+        if (num_pages == 0) {
+            return;
+        }
+
+        /* Find the largest block size that we can free, and free as many as possible. */
+        s32 big_index = static_cast<s32>(this->num_blocks) - 1;
+        const KVirtualAddress start  = addr;
+        const KVirtualAddress end    = addr + num_pages * PageSize;
+        KVirtualAddress before_start = start;
+        KVirtualAddress before_end   = start;
+        KVirtualAddress after_start  = end;
+        KVirtualAddress after_end    = end;
+        while (big_index >= 0) {
+            const size_t block_size = this->blocks[big_index].GetSize();
+            const KVirtualAddress big_start = util::AlignUp(GetInteger(start), block_size);
+            const KVirtualAddress big_end   = util::AlignDown(GetInteger(end), block_size);
+            if (big_start < big_end) {
+                /* Free as many big blocks as we can. */
+                for (auto block = big_start; block < big_end; block += block_size) {
+                    this->FreeBlock(block, big_index);
+                }
+                before_end  = big_start;
+                after_start = big_end;
+                break;
+            }
+            big_index--;
+        }
+        MESOSPHERE_ASSERT(big_index >= 0);
+
+        /* Free space before the big blocks. */
+        for (s32 i = big_index; i >= 0; i--) {
+            const size_t block_size = this->blocks[i].GetSize();
+            while (before_start + block_size <= before_end) {
+                before_end -= block_size;
+                this->FreeBlock(before_end, i);
+            }
+        }
+
+        /* Free space after the big blocks. */
+        for (s32 i = big_index; i >= 0; i--) {
+            const size_t block_size = this->blocks[i].GetSize();
+            while (after_start + block_size <= after_end) {
+                after_start += block_size;
+                this->FreeBlock(after_start, i);
+            }
+        }
+    }
+
     size_t KPageHeap::CalculateMetadataOverheadSize(size_t region_size, const size_t *block_shifts, size_t num_block_shifts) {
         size_t overhead_size = 0;
         for (size_t i = 0; i < num_block_shifts; i++) {

libraries/libmesosphere/source/kern_main.cpp

@@ -38,10 +38,11 @@ namespace ams::kern {
             /* Initialize KSystemControl. */
             KSystemControl::Initialize();
 
-            /* Initialize the memory manager. */
+            /* Initialize the memory manager and the KPageBuffer slabheap. */
             {
                 const auto &metadata_region = KMemoryLayout::GetMetadataPoolRegion();
                 Kernel::GetMemoryManager().Initialize(metadata_region.GetAddress(), metadata_region.GetSize());
+                init::InitializeKPageBufferSlabHeap();
             }
 
             /* Note: this is not actually done here, it's done later in main after more stuff is setup. */