Cool_Tools/Atmosphere
1
0
Fork 0
mirror of https://github.com/Atmosphere-NX/Atmosphere.git synced 2025-01-31 04:03:44 +01:00
Code Issues Packages Projects Releases Wiki Activity

kern: implement more of KMemoryManager through KPageBuffer slab init

Browse Source
This commit is contained in:
Michael Scire 2020-02-06 20:36:26 -08:00
parent f7d3d50f33
commit d9e6771e63
8 changed files with 418 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
libraries/libmesosphere/include/mesosphere/init/kern_init_slab_setup.hpp
View File

@ -40,6 +40,7 @@ namespace ams::kern::init {
const KSlabResourceCounts &GetSlabResourceCounts();
size_t CalculateTotalSlabHeapSize();
NOINLINE void InitializeKPageBufferSlabHeap();
NOINLINE void InitializeSlabHeaps();
}

4
libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp
View File

@ -467,6 +467,10 @@ namespace ams::kern {
return *GetVirtualMemoryRegionTree().FindFirstRegionByType(KMemoryRegionType_VirtualDramMetadataPool);
}
static NOINLINE KMemoryRegion &GetVirtualLinearRegion(KVirtualAddress address) {
return *GetVirtualLinearMemoryRegionTree().FindContainingRegion(GetInteger(address));
}
static NOINLINE auto GetCarveoutRegionExtents() {
return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionAttr_CarveoutProtected);
}

41
libraries/libmesosphere/include/mesosphere/kern_k_memory_manager.hpp
View File

@ -59,8 +59,28 @@ namespace ams::kern {
size_t Initialize(const KMemoryRegion *region, Pool pool, KVirtualAddress metadata_region, KVirtualAddress metadata_region_end);
constexpr ALWAYS_INLINE void SetNext(Impl *n) { this->next = n; }
constexpr ALWAYS_INLINE void SetPrev(Impl *n) { this->prev = n; }
KVirtualAddress AllocateBlock(s32 index) { return this->heap.AllocateBlock(index); }
void Free(KVirtualAddress addr, size_t num_pages) { this->heap.Free(addr, num_pages); }
void TrackAllocationForOptimizedProcess(KVirtualAddress block, size_t num_pages);
constexpr KVirtualAddress GetEndAddress() const { return this->heap.GetEndAddress(); }
constexpr void SetNext(Impl *n) { this->next = n; }
constexpr void SetPrev(Impl *n) { this->prev = n; }
constexpr Impl *GetNext() const { return this->next; }
constexpr Impl *GetPrev() const { return this->prev; }
void Open(KLightLock *pool_locks, KVirtualAddress address, size_t num_pages) {
KScopedLightLock lk(pool_locks[this->pool]);
size_t index = this->heap.GetPageOffset(address);
const size_t end = index + num_pages;
while (index < end) {
const RefCount ref_count = (++this->page_reference_counts[index++]);
MESOSPHERE_ABORT_UNLESS(ref_count > 0);
}
}
public:
static size_t CalculateMetadataOverheadSize(size_t region_size);
};
@ -72,6 +92,10 @@ namespace ams::kern {
size_t num_managers;
u64 optimized_process_ids[Pool_Count];
bool has_optimized_process[Pool_Count];
private:
Impl &GetManager(KVirtualAddress address) {
return this->managers[KMemoryLayout::GetVirtualLinearRegion(address).GetAttributes()];
}
public:
constexpr KMemoryManager()
: pool_locks(), pool_managers_head(), pool_managers_tail(), managers(), num_managers(), optimized_process_ids(), has_optimized_process()
@ -80,6 +104,19 @@ namespace ams::kern {
}
void Initialize(KVirtualAddress metadata_region, size_t metadata_region_size);
KVirtualAddress AllocateContinuous(size_t num_pages, size_t align_pages, u32 option);
void Open(KVirtualAddress address, size_t num_pages) {
/* Repeatedly open references until we've done so for all pages. */
while (num_pages) {
auto &manager = this->GetManager(address);
const size_t cur_pages = std::min(num_pages, (manager.GetEndAddress() - address) / PageSize);
manager.Open(this->pool_locks, address, cur_pages);
num_pages -= cur_pages;
address += cur_pages * PageSize;
}
}
public:
static size_t CalculateMetadataOverheadSize(size_t region_size) {
return Impl::CalculateMetadataOverheadSize(region_size);

214
libraries/libmesosphere/include/mesosphere/kern_k_page_heap.hpp
View File

@ -22,6 +22,33 @@ namespace ams::kern {
private:
static constexpr inline size_t MemoryBlockPageShifts[] = { 0xC, 0x10, 0x15, 0x16, 0x19, 0x1D, 0x1E };
static constexpr size_t NumMemoryBlockPageShifts = util::size(MemoryBlockPageShifts);
public:
static constexpr s32 GetAlignedBlockIndex(size_t num_pages, size_t align_pages) {
const size_t target_pages = std::max(num_pages, align_pages);
for (size_t i = 0; i < NumMemoryBlockPageShifts; i++) {
if (target_pages <= (size_t(1) << MemoryBlockPageShifts[i]) / PageSize) {
return static_cast<s32>(i);
}
}
return -1;
}
static constexpr s32 GetBlockIndex(size_t num_pages) {
for (size_t i = 0; i < NumMemoryBlockPageShifts; i++) {
if (num_pages >= (size_t(1) << MemoryBlockPageShifts[i]) / PageSize) {
return static_cast<s32>(i);
}
}
return -1;
}
static constexpr size_t GetBlockSize(size_t index) {
return size_t(1) << MemoryBlockPageShifts[index];
}
static constexpr size_t GetBlockNumPages(size_t index) {
return GetBlockSize(index) / PageSize;
}
private:
class Block {
private:
@ -31,27 +58,143 @@ namespace ams::kern {
private:
u64 *bit_storages[MaxDepth];
size_t num_bits;
size_t depth;
size_t used_depths;
public:
constexpr Bitmap() : bit_storages(), num_bits(), depth() { /* ... */ }
constexpr Bitmap() : bit_storages(), num_bits(), used_depths() { /* ... */ }
constexpr size_t GetNumBits() const { return this->num_bits; }
constexpr s32 GetHighestDepthIndex() const { return static_cast<s32>(this->used_depths) - 1; }
u64 *Initialize(u64 *storage, size_t size) {
/* Initially, everything is un-set. */
this->num_bits = 0;
/* Calculate the needed bitmap depth. */
this->depth = static_cast<size_t>(GetRequiredDepth(size));
MESOSPHERE_ASSERT(this->depth <= MaxDepth);
this->used_depths = static_cast<size_t>(GetRequiredDepth(size));
MESOSPHERE_ASSERT(this->used_depths <= MaxDepth);
/* Set the bitmap pointers. */
for (s32 d = static_cast<s32>(this->depth) - 1; d >= 0; d--) {
this->bit_storages[d] = storage;
for (s32 depth = this->GetHighestDepthIndex(); depth >= 0; depth--) {
this->bit_storages[depth] = storage;
size = util::AlignUp(size, BITSIZEOF(u64)) / BITSIZEOF(u64);
storage += size;
}
return storage;
}
ssize_t FindFreeBlock() const {
uintptr_t offset = 0;
s32 depth = 0;
do {
const u64 v = this->bit_storages[depth][offset];
if (v == 0) {
/* If depth is bigger than zero, then a previous level indicated a block was free. */
MESOSPHERE_ASSERT(depth == 0);
return -1;
}
offset = offset * BITSIZEOF(u64) + __builtin_ctzll(v);
++depth;
} while (depth < static_cast<s32>(this->used_depths));
return static_cast<ssize_t>(offset);
}
void SetBit(size_t offset) {
this->SetBit(this->GetHighestDepthIndex(), offset);
this->num_bits++;
}
void ClearBit(size_t offset) {
this->ClearBit(this->GetHighestDepthIndex(), offset);
this->num_bits--;
}
bool ClearRange(size_t offset, size_t count) {
s32 depth = this->GetHighestDepthIndex();
u64 *bits = this->bit_storages[depth];
size_t bit_ind = offset / BITSIZEOF(u64);
if (AMS_LIKELY(count < BITSIZEOF(u64))) {
const size_t shift = offset % BITSIZEOF(u64);
MESOSPHERE_ASSERT(shift + count <= BITSIZEOF(u64));
/* Check that all the bits are set. */
const u64 mask = ((u64(1) << count) - 1) << shift;
u64 v = bits[bit_ind];
if ((v & mask) != mask) {
return false;
}
/* Clear the bits. */
v &= ~mask;
bits[bit_ind] = v;
if (v == 0) {
this->ClearBit(depth - 1, bit_ind);
}
} else {
MESOSPHERE_ASSERT(offset % BITSIZEOF(u64) == 0);
MESOSPHERE_ASSERT(count % BITSIZEOF(u64) == 0);
/* Check that all the bits are set. */
size_t remaining = count;
size_t i = 0;
do {
if (bits[bit_ind + i++] != ~u64(0)) {
return false;
}
remaining -= BITSIZEOF(u64);
} while (remaining > 0);
/* Clear the bits. */
remaining = count;
i = 0;
do {
bits[bit_ind + i] = 0;
this->ClearBit(depth - 1, bit_ind + i);
i++;
remaining -= BITSIZEOF(u64);
} while (remaining > 0);
}
this->num_bits -= count;
return true;
}
private:
void SetBit(s32 depth, size_t offset) {
while (depth >= 0) {
size_t ind = offset / BITSIZEOF(u64);
size_t which = offset % BITSIZEOF(u64);
const u64 mask = u64(1) << which;
u64 *bit = std::addressof(this->bit_storages[depth][ind]);
u64 v = *bit;
MESOSPHERE_ASSERT((v & mask) == 0);
*bit = v | mask;
if (v) {
break;
}
offset = ind;
depth--;
}
}
void ClearBit(s32 depth, size_t offset) {
while (depth >= 0) {
size_t ind = offset / BITSIZEOF(u64);
size_t which = offset % BITSIZEOF(u64);
const u64 mask = u64(1) << which;
u64 *bit = std::addressof(this->bit_storages[depth][ind]);
u64 v = *bit;
MESOSPHERE_ASSERT((v & mask) != 0);
v &= ~mask;
*bit = v;
if (v) {
break;
}
offset = ind;
depth--;
}
}
private:
static constexpr s32 GetRequiredDepth(size_t region_size) {
s32 depth = 0;
@ -82,6 +225,13 @@ namespace ams::kern {
public:
constexpr Block() : bitmap(), heap_address(), end_offset(), block_shift(), next_block_shift() { /* ... */ }
constexpr size_t GetShift() const { return this->block_shift; }
constexpr size_t GetNextShift() const { return this->next_block_shift; }
constexpr size_t GetSize() const { return u64(1) << this->GetShift(); }
constexpr size_t GetNumPages() const { return this->GetSize() / PageSize; }
constexpr size_t GetNumFreeBlocks() const { return this->bitmap.GetNumBits(); }
constexpr size_t GetNumFreePages() const { return this->GetNumFreeBlocks() * this->GetNumPages(); }
u64 *Initialize(KVirtualAddress addr, size_t size, size_t bs, size_t nbs, u64 *bit_storage) {
/* Set shifts. */
this->block_shift = bs;
@ -89,18 +239,49 @@ namespace ams::kern {
/* Align up the address. */
KVirtualAddress end = addr + size;
const size_t align = (this->next_block_shift != 0) ? (1ul << this->next_block_shift) : (this->block_shift);
const size_t align = (this->next_block_shift != 0) ? (u64(1) << this->next_block_shift) : (this->block_shift);
addr = util::AlignDown(GetInteger(addr), align);
end = util::AlignUp(GetInteger(end), align);
this->heap_address = addr;
this->end_offset = (end - addr) / (1ul << this->block_shift);
this->end_offset = (end - addr) / (u64(1) << this->block_shift);
return this->bitmap.Initialize(bit_storage, this->end_offset);
}
KVirtualAddress PushBlock(KVirtualAddress address) {
/* Set the bit for the free block. */
size_t offset = (address - this->heap_address) >> this->GetShift();
this->bitmap.SetBit(offset);
/* If we have a next shift, try to clear the blocks below this one and return the new address. */
if (this->GetNextShift()) {
const size_t diff = u64(1) << (this->GetNextShift() - this->GetShift());
offset = util::AlignDown(offset, diff);
if (this->bitmap.ClearRange(offset, diff)) {
return this->heap_address + (offset << this->GetShift());
}
}
/* We couldn't coalesce, or we're already as big as possible. */
return Null<KVirtualAddress>;
}
KVirtualAddress PopBlock() {
/* Find a free block. */
ssize_t soffset = this->bitmap.FindFreeBlock();
if (soffset < 0) {
return Null<KVirtualAddress>;
}
const size_t offset = static_cast<size_t>(soffset);
/* Update our tracking and return it. */
this->bitmap.ClearBit(offset);
return this->heap_address + (offset << this->GetShift());
}
public:
static constexpr size_t CalculateMetadataOverheadSize(size_t region_size, size_t cur_block_shift, size_t next_block_shift) {
const size_t cur_block_size = (1ul << cur_block_shift);
const size_t next_block_size = (1ul << next_block_shift);
const size_t cur_block_size = (u64(1) << cur_block_shift);
const size_t next_block_size = (u64(1) << next_block_shift);
const size_t align = (next_block_shift != 0) ? next_block_size : cur_block_size;
return Bitmap::CalculateMetadataOverheadSize((align * 2 + util::AlignUp(region_size, align)) / cur_block_size);
}
@ -113,12 +294,25 @@ namespace ams::kern {
Block blocks[NumMemoryBlockPageShifts];
private:
void Initialize(KVirtualAddress heap_address, size_t heap_size, KVirtualAddress metadata_address, size_t metadata_size, const size_t *block_shifts, size_t num_block_shifts);
size_t GetNumFreePages() const;
void FreeBlock(KVirtualAddress block, s32 index);
public:
constexpr KPageHeap() : heap_address(), heap_size(), used_size(), num_blocks(), blocks() { /* ... */ }
constexpr KVirtualAddress GetEndAddress() const { return this->heap_address + this->heap_size; }
constexpr size_t GetPageOffset(KVirtualAddress block) const { return (block - this->heap_address) / PageSize; }
void Initialize(KVirtualAddress heap_address, size_t heap_size, KVirtualAddress metadata_address, size_t metadata_size) {
return Initialize(heap_address, heap_size, metadata_address, metadata_size, MemoryBlockPageShifts, NumMemoryBlockPageShifts);
}
void UpdateUsedSize() {
this->used_size = this->heap_size - (this->GetNumFreePages() * PageSize);
}
KVirtualAddress AllocateBlock(s32 index);
void Free(KVirtualAddress addr, size_t num_pages);
private:
static size_t CalculateMetadataOverheadSize(size_t region_size, const size_t *block_shifts, size_t num_block_shifts);
public:

20
libraries/libmesosphere/source/init/kern_init_slab_setup.cpp
View File

@ -135,6 +135,26 @@ namespace ams::kern::init {
return size;
}
void InitializeKPageBufferSlabHeap() {
const auto &counts = GetSlabResourceCounts();
const size_t num_pages = counts.num_KProcess + counts.num_KThread + (counts.num_KProcess + counts.num_KThread) / 8;
const size_t slab_size = num_pages * PageSize;
/* Reserve memory from the system resource limit. */
MESOSPHERE_ABORT_UNLESS(Kernel::GetSystemResourceLimit().Reserve(ams::svc::LimitableResource_PhysicalMemoryMax, slab_size));
/* Allocate memory for the slab. */
constexpr auto AllocateOption = KMemoryManager::EncodeOption(KMemoryManager::Pool_System, KMemoryManager::Direction_FromFront);
const KVirtualAddress slab_address = Kernel::GetMemoryManager().AllocateContinuous(num_pages, 1, AllocateOption);
MESOSPHERE_ABORT_UNLESS(slab_address != Null<KVirtualAddress>);
/* Open references to the slab. */
Kernel::GetMemoryManager().Open(slab_address, num_pages);
/* Initialize the slabheap. */
KPageBuffer::InitializeSlabHeap(GetVoidPointer(slab_address), slab_size);
}
void InitializeSlabHeaps() {
/* Get the start of the slab region, since that's where we'll be working. */
KVirtualAddress address = KMemoryLayout::GetSlabRegionAddress();

67
libraries/libmesosphere/source/kern_k_memory_manager.cpp
View File

@ -87,6 +87,57 @@ namespace ams::kern {
}
}
KVirtualAddress KMemoryManager::AllocateContinuous(size_t num_pages, size_t align_pages, u32 option) {
/* Early return if we're allocating no pages. */
if (num_pages == 0) {
return Null<KVirtualAddress>;
}
/* Lock the pool that we're allocating from. */
const auto [pool, dir] = DecodeOption(option);
KScopedLightLock lk(this->pool_locks[pool]);
/* Choose a heap based on our page size request. */
const s32 heap_index = KPageHeap::GetAlignedBlockIndex(num_pages, align_pages);
/* Loop, trying to iterate from each block. */
Impl *chosen_manager = nullptr;
KVirtualAddress allocated_block = Null<KVirtualAddress>;
if (dir == Direction_FromBack) {
for (chosen_manager = this->pool_managers_tail[pool]; chosen_manager != nullptr; chosen_manager = chosen_manager->GetPrev()) {
allocated_block = chosen_manager->AllocateBlock(heap_index);
if (allocated_block != Null<KVirtualAddress>) {
break;
}
}
} else {
for (chosen_manager = this->pool_managers_head[pool]; chosen_manager != nullptr; chosen_manager = chosen_manager->GetNext()) {
allocated_block = chosen_manager->AllocateBlock(heap_index);
if (allocated_block != Null<KVirtualAddress>) {
break;
}
}
}
/* If we failed to allocate, quit now. */
if (allocated_block == Null<KVirtualAddress>) {
return Null<KVirtualAddress>;
}
/* If we allocated more than we need, free some. */
const size_t allocated_pages = KPageHeap::GetBlockNumPages(heap_index);
if (allocated_pages > num_pages) {
chosen_manager->Free(allocated_block + num_pages * PageSize, allocated_pages - num_pages);
}
/* Maintain the optimized memory bitmap, if we should. */
if (this->has_optimized_process[pool]) {
chosen_manager->TrackAllocationForOptimizedProcess(allocated_block, num_pages);
}
return allocated_block;
}
size_t KMemoryManager::Impl::Initialize(const KMemoryRegion *region, Pool p, KVirtualAddress metadata, KVirtualAddress metadata_end) {
/* Calculate metadata sizes. */
const size_t ref_count_size = (region->GetSize() / PageSize) * sizeof(u16);
@ -107,9 +158,25 @@ namespace ams::kern {
/* Initialize the manager's KPageHeap. */
this->heap.Initialize(region->GetAddress(), region->GetSize(), metadata + manager_size, page_heap_size);
/* Free the memory to the heap. */
this->heap.Free(region->GetAddress(), region->GetSize() / PageSize);
/* Update the heap's used size. */
this->heap.UpdateUsedSize();
return total_metadata_size;
}
void KMemoryManager::Impl::TrackAllocationForOptimizedProcess(KVirtualAddress block, size_t num_pages) {
size_t offset = this->heap.GetPageOffset(block);
const size_t last = offset + num_pages - 1;
u64 *optimize_map = GetPointer<u64>(this->metadata_region);
while (offset <= last) {
optimize_map[offset / BITSIZEOF(u64)] &= ~(u64(1) << (offset % BITSIZEOF(u64)));
offset++;
}
}
size_t KMemoryManager::Impl::CalculateMetadataOverheadSize(size_t region_size) {
const size_t ref_count_size = (region_size / PageSize) * sizeof(u16);
const size_t optimize_map_size = (util::AlignUp((region_size / PageSize), BITSIZEOF(u64)) / BITSIZEOF(u64)) * sizeof(u64);

81
libraries/libmesosphere/source/kern_k_page_heap.cpp
View File

@ -41,6 +41,87 @@ namespace ams::kern {
MESOSPHERE_ABORT_UNLESS(KVirtualAddress(cur_bitmap_storage) <= metadata_end);
}
size_t KPageHeap::GetNumFreePages() const {
size_t num_free = 0;
for (size_t i = 0; i < this->num_blocks; i++) {
num_free += this->blocks[i].GetNumFreePages();
}
return num_free;
}
KVirtualAddress KPageHeap::AllocateBlock(s32 index) {
const size_t needed_size = this->blocks[index].GetSize();
for (s32 i = index; i < static_cast<s32>(this->num_blocks); i++) {
if (const KVirtualAddress addr = this->blocks[index].PopBlock(); addr != Null<KVirtualAddress>) {
if (const size_t allocated_size = this->blocks[index].GetSize(); allocated_size > needed_size) {
this->Free(addr + needed_size, (allocated_size - needed_size) / PageSize);
}
return addr;
}
}
return Null<KVirtualAddress>;
}
void KPageHeap::FreeBlock(KVirtualAddress block, s32 index) {
do {
block = this->blocks[index++].PushBlock(block);
} while (block != Null<KVirtualAddress>);
}
void KPageHeap::Free(KVirtualAddress addr, size_t num_pages) {
/* Freeing no pages is a no-op. */
if (num_pages == 0) {
return;
}
/* Find the largest block size that we can free, and free as many as possible. */
s32 big_index = static_cast<s32>(this->num_blocks) - 1;
const KVirtualAddress start = addr;
const KVirtualAddress end = addr + num_pages * PageSize;
KVirtualAddress before_start = start;
KVirtualAddress before_end = start;
KVirtualAddress after_start = end;
KVirtualAddress after_end = end;
while (big_index >= 0) {
const size_t block_size = this->blocks[big_index].GetSize();
const KVirtualAddress big_start = util::AlignUp(GetInteger(start), block_size);
const KVirtualAddress big_end = util::AlignDown(GetInteger(end), block_size);
if (big_start < big_end) {
/* Free as many big blocks as we can. */
for (auto block = big_start; block < big_end; block += block_size) {
this->FreeBlock(block, big_index);
}
before_end = big_start;
after_start = big_end;
break;
}
big_index--;
}
MESOSPHERE_ASSERT(big_index >= 0);
/* Free space before the big blocks. */
for (s32 i = big_index; i >= 0; i--) {
const size_t block_size = this->blocks[i].GetSize();
while (before_start + block_size <= before_end) {
before_end -= block_size;
this->FreeBlock(before_end, i);
}
}
/* Free space after the big blocks. */
for (s32 i = big_index; i >= 0; i--) {
const size_t block_size = this->blocks[i].GetSize();
while (after_start + block_size <= after_end) {
after_start += block_size;
this->FreeBlock(after_start, i);
}
}
}
size_t KPageHeap::CalculateMetadataOverheadSize(size_t region_size, const size_t *block_shifts, size_t num_block_shifts) {
size_t overhead_size = 0;
for (size_t i = 0; i < num_block_shifts; i++) {

3
libraries/libmesosphere/source/kern_main.cpp
View File

@ -38,10 +38,11 @@ namespace ams::kern {
/* Initialize KSystemControl. */
KSystemControl::Initialize();
/* Initialize the memory manager. */
/* Initialize the memory manager and the KPageBuffer slabheap. */
{
const auto &metadata_region = KMemoryLayout::GetMetadataPoolRegion();
Kernel::GetMemoryManager().Initialize(metadata_region.GetAddress(), metadata_region.GetSize());
init::InitializeKPageBufferSlabHeap();
}
/* Note: this is not actually done here, it's done later in main after more stuff is setup. */