diff --git a/libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp b/libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp index 194950f3e..330ce9c59 100644 --- a/libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp +++ b/libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp @@ -18,169 +18,9 @@ #include #include #include +#include -namespace ams::kern::init { - - constexpr size_t L1BlockSize = 1_GB; - constexpr size_t L2BlockSize = 2_MB; - constexpr size_t L2ContiguousBlockSize = 0x10 * L2BlockSize; - constexpr size_t L3BlockSize = PageSize; - constexpr size_t L3ContiguousBlockSize = 0x10 * L3BlockSize; - - class PageTableEntry { - public: - enum Permission : u64 { - Permission_KernelRWX = ((0ul << 53) | (1ul << 54) | (0ul << 6)), - Permission_KernelRX = ((0ul << 53) | (1ul << 54) | (2ul << 6)), - Permission_KernelR = ((1ul << 53) | (1ul << 54) | (2ul << 6)), - Permission_KernelRW = ((1ul << 53) | (1ul << 54) | (0ul << 6)), - - Permission_UserRX = ((1ul << 53) | (0ul << 54) | (3ul << 6)), - Permission_UserR = ((1ul << 53) | (1ul << 54) | (3ul << 6)), - Permission_UserRW = ((1ul << 53) | (1ul << 54) | (1ul << 6)), - }; - - enum Shareable : u64 { - Shareable_NonShareable = (0 << 8), - Shareable_OuterShareable = (2 << 8), - Shareable_InnerShareable = (3 << 8), - }; - - /* Official attributes are: */ - /* 0x00, 0x04, 0xFF, 0x44. 4-7 are unused. */ - enum PageAttribute : u64 { - PageAttribute_Device_nGnRnE = (0 << 2), - PageAttribute_Device_nGnRE = (1 << 2), - PageAttribute_NormalMemory = (2 << 2), - PageAttribute_NormalMemoryNotCacheable = (3 << 2), - }; - - enum AccessFlag : u64 { - AccessFlag_NotAccessed = (0 << 10), - AccessFlag_Accessed = (1 << 10), - }; - protected: - u64 attributes; - public: - /* Take in a raw attribute. */ - constexpr ALWAYS_INLINE PageTableEntry(u64 attr) : attributes(attr) { /* ... */ } - - /* Extend a previous attribute. */ - constexpr ALWAYS_INLINE PageTableEntry(const PageTableEntry &rhs, u64 new_attr) : attributes(rhs.attributes | new_attr) { /* ... */ } - - /* Construct a new attribute. */ - constexpr ALWAYS_INLINE PageTableEntry(Permission perm, PageAttribute p_a, Shareable share) - : attributes(static_cast(perm) | static_cast(AccessFlag_Accessed) | static_cast(p_a) | static_cast(share)) - { - /* ... */ - } - protected: - constexpr ALWAYS_INLINE u64 GetBits(size_t offset, size_t count) const { - return (this->attributes >> offset) & ((1ul << count) - 1); - } - - constexpr ALWAYS_INLINE u64 SelectBits(size_t offset, size_t count) const { - return this->attributes & (((1ul << count) - 1) << offset); - } - public: - constexpr ALWAYS_INLINE bool IsUserExecuteNever() const { return this->GetBits(54, 1) != 0; } - constexpr ALWAYS_INLINE bool IsPrivilegedExecuteNever() const { return this->GetBits(53, 1) != 0; } - constexpr ALWAYS_INLINE bool IsContiguous() const { return this->GetBits(52, 1) != 0; } - constexpr ALWAYS_INLINE AccessFlag GetAccessFlag() const { return static_cast(this->GetBits(10, 1)); } - constexpr ALWAYS_INLINE Shareable GetShareable() const { return static_cast(this->GetBits(8, 2)); } - constexpr ALWAYS_INLINE PageAttribute GetPageAttribute() const { return static_cast(this->GetBits(2, 3)); } - constexpr ALWAYS_INLINE bool IsNonSecure() const { return this->GetBits(5, 1) != 0; } - constexpr ALWAYS_INLINE bool IsBlock() const { return this->GetBits(0, 2) == 0x1; } - constexpr ALWAYS_INLINE bool IsTable() const { return this->GetBits(0, 2) == 0x3; } - - /* Should not be called except by derived classes. */ - constexpr ALWAYS_INLINE u64 GetRawAttributes() const { - return this->attributes; - } - }; - - static_assert(sizeof(PageTableEntry) == sizeof(u64)); - - constexpr PageTableEntry InvalidPageTableEntry = PageTableEntry(0); - - constexpr size_t MaxPageTableEntries = PageSize / sizeof(PageTableEntry); - - class L1PageTableEntry : public PageTableEntry { - public: - constexpr ALWAYS_INLINE L1PageTableEntry(KPhysicalAddress phys_addr, bool pxn) - : PageTableEntry((0x3ul << 60) | (static_cast(pxn) << 59) | GetInteger(phys_addr) | 0x3) - { - /* ... */ - } - - constexpr ALWAYS_INLINE L1PageTableEntry(KPhysicalAddress phys_addr, const PageTableEntry &attr, bool contig) - : PageTableEntry(attr, (static_cast(contig) << 52) | GetInteger(phys_addr) | 0x1) - { - /* ... */ - } - - constexpr ALWAYS_INLINE KPhysicalAddress GetBlock() const { - return this->SelectBits(30, 18); - } - - constexpr ALWAYS_INLINE KPhysicalAddress GetTable() const { - return this->SelectBits(12, 36); - } - - constexpr ALWAYS_INLINE bool IsCompatibleWithAttribute(const PageTableEntry &rhs, bool contig) const { - /* Check whether this has the same permission/etc as the desired attributes. */ - return L1PageTableEntry(this->GetBlock(), rhs, contig).GetRawAttributes() == this->GetRawAttributes(); - } - }; - - class L2PageTableEntry : public PageTableEntry { - public: - constexpr ALWAYS_INLINE L2PageTableEntry(KPhysicalAddress phys_addr, bool pxn) - : PageTableEntry((0x3ul << 60) | (static_cast(pxn) << 59) | GetInteger(phys_addr) | 0x3) - { - /* ... */ - } - - constexpr ALWAYS_INLINE L2PageTableEntry(KPhysicalAddress phys_addr, const PageTableEntry &attr, bool contig) - : PageTableEntry(attr, (static_cast(contig) << 52) | GetInteger(phys_addr) | 0x1) - { - /* ... */ - } - - constexpr ALWAYS_INLINE KPhysicalAddress GetBlock() const { - return this->SelectBits(21, 27); - } - - constexpr ALWAYS_INLINE KPhysicalAddress GetTable() const { - return this->SelectBits(12, 36); - } - - constexpr ALWAYS_INLINE bool IsCompatibleWithAttribute(const PageTableEntry &rhs, bool contig) const { - /* Check whether this has the same permission/etc as the desired attributes. */ - return L2PageTableEntry(this->GetBlock(), rhs, contig).GetRawAttributes() == this->GetRawAttributes(); - } - }; - - class L3PageTableEntry : public PageTableEntry { - public: - constexpr ALWAYS_INLINE L3PageTableEntry(KPhysicalAddress phys_addr, const PageTableEntry &attr, bool contig) - : PageTableEntry(attr, (static_cast(contig) << 52) | GetInteger(phys_addr) | 0x3) - { - /* ... */ - } - - constexpr ALWAYS_INLINE bool IsBlock() const { return this->GetBits(0, 2) == 0x3; } - - constexpr ALWAYS_INLINE KPhysicalAddress GetBlock() const { - return this->SelectBits(12, 36); - } - - constexpr ALWAYS_INLINE bool IsCompatibleWithAttribute(const PageTableEntry &rhs, bool contig) const { - /* Check whether this has the same permission/etc as the desired attributes. */ - return L3PageTableEntry(this->GetBlock(), rhs, contig).GetRawAttributes() == this->GetRawAttributes(); - } - }; - +namespace ams::kern::arm64::init { class KInitialPageTable { public: diff --git a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu.hpp b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu.hpp index 81ba01474..5a4b27651 100644 --- a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu.hpp +++ b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu.hpp @@ -124,11 +124,28 @@ namespace ams::kern::arm64::cpu { ClearPageToZeroImpl(page); } + ALWAYS_INLINE void InvalidateTlbByAsid(u32 asid) { + const u64 value = (static_cast(asid) << 48); + __asm__ __volatile__("tlbi aside1is, %[value]" :: [value]"r"(static_cast(value) << 48) : "memory"); + EnsureInstructionConsistency(); + } + + ALWAYS_INLINE void InvalidateTlbByAsidAndVa(u32 asid, KProcessAddress virt_addr) { + const u64 value = (static_cast(asid) << 48) | ((GetInteger(virt_addr) >> 12) & 0xFFFFFFFFFFFul); + __asm__ __volatile__("tlbi aside1is, %[value]" :: [value]"r"(value) : "memory"); + EnsureInstructionConsistency(); + } + ALWAYS_INLINE void InvalidateEntireTlb() { __asm__ __volatile__("tlbi vmalle1is" ::: "memory"); EnsureInstructionConsistency(); } + ALWAYS_INLINE void InvalidateEntireTlbDataOnly() { + __asm__ __volatile__("tlbi vmalle1is" ::: "memory"); + DataSynchronizationBarrier(); + } + ALWAYS_INLINE uintptr_t GetCoreLocalRegionAddress() { register uintptr_t x18 asm("x18"); __asm__ __volatile__("" : [x18]"=r"(x18)); diff --git a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu_system_registers.hpp b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu_system_registers.hpp index d43eaeb15..1fe333783 100644 --- a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu_system_registers.hpp +++ b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu_system_registers.hpp @@ -97,7 +97,13 @@ namespace ams::kern::arm64::cpu { constexpr ALWAYS_INLINE void SetBits(size_t offset, size_t count, u64 value) { const u64 mask = ((1ul << count) - 1) << offset; this->value &= ~mask; - this->value |= (value & mask) << offset; + this->value |= (value & (mask >> offset)) << offset; + } + + constexpr ALWAYS_INLINE void SetBitsDirect(size_t offset, size_t count, u64 value) { + const u64 mask = ((1ul << count) - 1) << offset; + this->value &= ~mask; + this->value |= (value & mask); } constexpr ALWAYS_INLINE void SetBit(size_t offset, bool enabled) { diff --git a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table.hpp b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table.hpp index 53c9acc4f..c0911554f 100644 --- a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table.hpp +++ b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table.hpp @@ -29,6 +29,127 @@ namespace ams::kern::arm64 { KPageTableManager *manager; u64 ttbr; u8 asid; + private: + enum BlockType { + BlockType_L3Block, + BlockType_L3ContiguousBlock, + BlockType_L2Block, + +#ifdef ATMOSPHERE_BOARD_NINTENDO_SWITCH + BlockType_L2TegraSmmuBlock, +#endif + + BlockType_L2ContiguousBlock, + BlockType_L1Block, + + BlockType_Count, + }; + + static_assert(L3BlockSize == PageSize); + static constexpr size_t ContiguousPageSize = L3ContiguousBlockSize; + +#ifdef ATMOSPHERE_BOARD_NINTENDO_SWITCH + static constexpr size_t L2TegraSmmuBlockSize = 2 * L2BlockSize; +#endif + static constexpr size_t BlockSizes[BlockType_Count] = { + [BlockType_L3Block] = L3BlockSize, + [BlockType_L3ContiguousBlock] = L3ContiguousBlockSize, + [BlockType_L2Block] = L2BlockSize, +#ifdef ATMOSPHERE_BOARD_NINTENDO_SWITCH + [BlockType_L2TegraSmmuBlock] = L2TegraSmmuBlockSize, +#endif + [BlockType_L2ContiguousBlock] = L2ContiguousBlockSize, + [BlockType_L1Block] = L1BlockSize, + }; + + static constexpr size_t GetBlockSize(BlockType type) { + return BlockSizes[type]; + } + + static constexpr BlockType GetBlockType(size_t size) { + switch (size) { + case L3BlockSize: return BlockType_L3Block; + case L3ContiguousBlockSize: return BlockType_L3ContiguousBlock; + case L2BlockSize: return BlockType_L2Block; +#ifdef ATMOSPHERE_BOARD_NINTENDO_SWITCH + case L2TegraSmmuBlockSize: return BlockType_L2TegraSmmuBlock; +#endif + case L2ContiguousBlockSize: return BlockType_L2ContiguousBlock; + case L1BlockSize: return BlockType_L1Block; + MESOSPHERE_UNREACHABLE_DEFAULT_CASE(); + } + } + protected: + virtual Result Operate(PageLinkedList *page_list, KProcessAddress virt_addr, size_t num_pages, KPhysicalAddress phys_addr, bool is_pa_valid, const KPageProperties properties, OperationType operation, bool reuse_ll) override; + virtual Result Operate(PageLinkedList *page_list, KProcessAddress virt_addr, size_t num_pages, const KPageGroup *page_group, const KPageProperties properties, OperationType operation, bool reuse_ll) override; + virtual void FinalizeUpdate(PageLinkedList *page_list) override; + + KPageTableManager &GetPageTableManager() { return *this->manager; } + const KPageTableManager &GetPageTableManager() const { return *this->manager; } + private: + constexpr PageTableEntry GetEntryTemplate(const KPageProperties properties) const { + /* Set basic attributes. */ + PageTableEntry entry; + entry.SetPrivilegedExecuteNever(true); + entry.SetAccessFlag(PageTableEntry::AccessFlag_Accessed); + entry.SetShareable(PageTableEntry::Shareable_InnerShareable); + + if (!this->IsKernel()) { + entry.SetGlobal(false); + } + + /* Set page attribute. */ + if (properties.io) { + MESOSPHERE_ABORT_UNLESS(!properties.io); + MESOSPHERE_ABORT_UNLESS((properties.perm & (KMemoryPermission_KernelExecute | KMemoryPermission_UserExecute)) == 0); + + entry.SetPageAttribute(PageTableEntry::PageAttribute_Device_nGnRnE) + .SetUserExecuteNever(true); + } else if (properties.uncached) { + MESOSPHERE_ABORT_UNLESS((properties.perm & (KMemoryPermission_KernelExecute | KMemoryPermission_UserExecute)) == 0); + + entry.SetPageAttribute(PageTableEntry::PageAttribute_NormalMemoryNotCacheable); + } else { + entry.SetPageAttribute(PageTableEntry::PageAttribute_NormalMemory); + } + + /* Set user execute never bit. */ + if (properties.perm != KMemoryPermission_UserReadExecute) { + MESOSPHERE_ABORT_UNLESS((properties.perm & (KMemoryPermission_KernelExecute | KMemoryPermission_UserExecute)) == 0); + entry.SetUserExecuteNever(true); + } + + /* Set can be contiguous. */ + entry.SetContiguousAllowed(!properties.non_contiguous); + + /* Set AP[1] based on perm. */ + switch (properties.perm & KMemoryPermission_UserReadWrite) { + case KMemoryPermission_UserReadWrite: + case KMemoryPermission_UserRead: + entry.SetUserAccessible(true); + break; + case KMemoryPermission_KernelReadWrite: + case KMemoryPermission_KernelRead: + entry.SetUserAccessible(false); + break; + MESOSPHERE_UNREACHABLE_DEFAULT_CASE(); + } + + /* Set AP[2] based on perm. */ + switch (properties.perm & KMemoryPermission_UserReadWrite) { + case KMemoryPermission_UserReadWrite: + case KMemoryPermission_KernelReadWrite: + entry.SetReadOnly(false); + break; + case KMemoryPermission_KernelRead: + case KMemoryPermission_UserRead: + entry.SetReadOnly(true); + break; + MESOSPHERE_UNREACHABLE_DEFAULT_CASE(); + } + + return entry; + } public: constexpr KPageTable() : KPageTableBase(), manager(), ttbr(), asid() { /* ... */ } @@ -41,6 +162,63 @@ namespace ams::kern::arm64 { NOINLINE Result InitializeForKernel(void *table, KVirtualAddress start, KVirtualAddress end); Result Finalize(); + private: + Result Map(KProcessAddress virt_addr, KPhysicalAddress phys_addr, size_t num_pages, PageTableEntry entry_template, PageLinkedList *page_list, bool reuse_ll); + Result Unmap(KProcessAddress virt_addr, size_t num_pages, KPageGroup *pg, PageLinkedList *page_list, bool force, bool reuse_ll); + + Result MapContiguous(KProcessAddress virt_addr, KPhysicalAddress phys_addr, size_t num_pages, PageTableEntry entry_template, PageLinkedList *page_list, bool reuse_ll); + + bool MergePages(KProcessAddress virt_addr, PageLinkedList *page_list); + + static void PteDataSynchronizationBarrier() { + cpu::DataSynchronizationBarrierInnerShareable(); + } + + static void ClearPageTable(KVirtualAddress table) { + cpu::ClearPageToZero(GetVoidPointer(table)); + } + + void OnTableUpdated() const { + cpu::InvalidateTlbByAsid(this->asid); + } + + void OnKernelTableUpdated() const { + cpu::InvalidateEntireTlbDataOnly(); + } + + void NoteUpdated() const { + cpu::DataSynchronizationBarrier(); + + if (this->IsKernel()) { + this->OnKernelTableUpdated(); + } else { + this->OnTableUpdated(); + } + } + + KVirtualAddress AllocatePageTable(PageLinkedList *page_list, bool reuse_ll) { + KVirtualAddress table = this->GetPageTableManager().Allocate(); + + if (table == Null) { + if (reuse_ll && page_list->Peek()) { + table = KVirtualAddress(reinterpret_cast(page_list->Pop())); + } else { + return Null; + } + } + + ClearPageTable(table); + + MESOSPHERE_ASSERT(this->GetPageTableManager().GetRefCount(table) == 0); + + return table; + } + + void FreePageTable(PageLinkedList *page_list, KVirtualAddress table) const { + MESOSPHERE_ASSERT(this->GetPageTableManager().IsInPageTableHeap(table)); + MESOSPHERE_ASSERT(this->GetPageTableManager().GetRefCount(table) == 0); + page_list->Push(table); + } }; } diff --git a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table_entry.hpp b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table_entry.hpp new file mode 100644 index 000000000..74e7c7e18 --- /dev/null +++ b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table_entry.hpp @@ -0,0 +1,287 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include +#include +#include + +namespace ams::kern::arm64 { + + constexpr size_t L1BlockSize = 1_GB; + constexpr size_t L2BlockSize = 2_MB; + constexpr size_t L2ContiguousBlockSize = 0x10 * L2BlockSize; + constexpr size_t L3BlockSize = PageSize; + constexpr size_t L3ContiguousBlockSize = 0x10 * L3BlockSize; + + class PageTableEntry { + public: + struct InvalidTag{}; + + enum Permission : u64 { + Permission_KernelRWX = ((0ul << 53) | (1ul << 54) | (0ul << 6)), + Permission_KernelRX = ((0ul << 53) | (1ul << 54) | (2ul << 6)), + Permission_KernelR = ((1ul << 53) | (1ul << 54) | (2ul << 6)), + Permission_KernelRW = ((1ul << 53) | (1ul << 54) | (0ul << 6)), + + Permission_UserRX = ((1ul << 53) | (0ul << 54) | (3ul << 6)), + Permission_UserR = ((1ul << 53) | (1ul << 54) | (3ul << 6)), + Permission_UserRW = ((1ul << 53) | (1ul << 54) | (1ul << 6)), + }; + + enum Shareable : u64 { + Shareable_NonShareable = (0 << 8), + Shareable_OuterShareable = (2 << 8), + Shareable_InnerShareable = (3 << 8), + }; + + /* Official attributes are: */ + /* 0x00, 0x04, 0xFF, 0x44. 4-7 are unused. */ + enum PageAttribute : u64 { + PageAttribute_Device_nGnRnE = (0 << 2), + PageAttribute_Device_nGnRE = (1 << 2), + PageAttribute_NormalMemory = (2 << 2), + PageAttribute_NormalMemoryNotCacheable = (3 << 2), + }; + + enum AccessFlag : u64 { + AccessFlag_NotAccessed = (0 << 10), + AccessFlag_Accessed = (1 << 10), + }; + + enum Type : u64 { + Type_None = 0x0, + Type_L1Block = 0x1, + Type_L1Table = 0x3, + Type_L2Block = 0x1, + Type_L2Table = 0x3, + Type_L3Block = 0x3, + }; + + enum ContigType : u64 { + ContigType_NotContiguous = (0x0ul << 52), + ContigType_Contiguous = (0x1ul << 52), + }; + protected: + u64 attributes; + public: + /* Take in a raw attribute. */ + constexpr ALWAYS_INLINE PageTableEntry() : attributes() { /* ... */ } + constexpr ALWAYS_INLINE PageTableEntry(u64 attr) : attributes(attr) { /* ... */ } + + constexpr ALWAYS_INLINE PageTableEntry(InvalidTag) : attributes(0) { /* ... */ } + + /* Extend a previous attribute. */ + constexpr ALWAYS_INLINE PageTableEntry(const PageTableEntry &rhs, u64 new_attr) : attributes(rhs.attributes | new_attr) { /* ... */ } + + /* Construct a new attribute. */ + constexpr ALWAYS_INLINE PageTableEntry(Permission perm, PageAttribute p_a, Shareable share) + : attributes(static_cast(perm) | static_cast(AccessFlag_Accessed) | static_cast(p_a) | static_cast(share)) + { + /* ... */ + } + protected: + constexpr ALWAYS_INLINE u64 GetBits(size_t offset, size_t count) const { + return (this->attributes >> offset) & ((1ul << count) - 1); + } + + constexpr ALWAYS_INLINE u64 SelectBits(size_t offset, size_t count) const { + return this->attributes & (((1ul << count) - 1) << offset); + } + + constexpr ALWAYS_INLINE void SetBits(size_t offset, size_t count, u64 value) { + const u64 mask = ((1ul << count) - 1) << offset; + this->attributes &= ~mask; + this->attributes |= (value & (mask >> offset)) << offset; + } + + constexpr ALWAYS_INLINE void SetBitsDirect(size_t offset, size_t count, u64 value) { + const u64 mask = ((1ul << count) - 1) << offset; + this->attributes &= ~mask; + this->attributes |= (value & mask); + } + + constexpr ALWAYS_INLINE void SetBit(size_t offset, bool enabled) { + const u64 mask = 1ul << offset; + if (enabled) { + this->attributes |= mask; + } else { + this->attributes &= ~mask; + } + } + public: + constexpr ALWAYS_INLINE bool IsContiguousAllowed() const { return this->GetBits(55, 1) != 0; } + constexpr ALWAYS_INLINE bool IsUserExecuteNever() const { return this->GetBits(54, 1) != 0; } + constexpr ALWAYS_INLINE bool IsPrivilegedExecuteNever() const { return this->GetBits(53, 1) != 0; } + constexpr ALWAYS_INLINE bool IsContiguous() const { return this->GetBits(52, 1) != 0; } + constexpr ALWAYS_INLINE bool IsGlobal() const { return this->GetBits(11, 1) == 0; } + constexpr ALWAYS_INLINE AccessFlag GetAccessFlag() const { return static_cast(this->GetBits(10, 1)); } + constexpr ALWAYS_INLINE Shareable GetShareable() const { return static_cast(this->GetBits(8, 2)); } + constexpr ALWAYS_INLINE PageAttribute GetPageAttribute() const { return static_cast(this->GetBits(2, 3)); } + constexpr ALWAYS_INLINE bool IsReadOnly() const { return this->GetBits(7, 1) != 0; } + constexpr ALWAYS_INLINE bool IsUserAccessible() const { return this->GetBits(6, 1) != 0; } + constexpr ALWAYS_INLINE bool IsNonSecure() const { return this->GetBits(5, 1) != 0; } + constexpr ALWAYS_INLINE bool IsBlock() const { return this->GetBits(0, 2) == 0x1; } + constexpr ALWAYS_INLINE bool IsTable() const { return this->GetBits(0, 2) == 0x3; } + + constexpr ALWAYS_INLINE decltype(auto) SetContiguousAllowed(bool en) { this->SetBit(55, !en); return *this; } + constexpr ALWAYS_INLINE decltype(auto) SetUserExecuteNever(bool en) { this->SetBit(54, en); return *this; } + constexpr ALWAYS_INLINE decltype(auto) SetPrivilegedExecuteNever(bool en) { this->SetBit(53, en); return *this; } + constexpr ALWAYS_INLINE decltype(auto) SetContiguous(bool en) { this->SetBit(52, en); return *this; } + constexpr ALWAYS_INLINE decltype(auto) SetGlobal(bool en) { this->SetBit(11, !en); return *this; } + constexpr ALWAYS_INLINE decltype(auto) SetAccessFlag(AccessFlag f) { this->SetBitsDirect(10, 1, f); return *this; } + constexpr ALWAYS_INLINE decltype(auto) SetShareable(Shareable s) { this->SetBitsDirect(8, 2, s); return *this; } + constexpr ALWAYS_INLINE decltype(auto) SetReadOnly(bool en) { this->SetBit(7, en); return *this; } + constexpr ALWAYS_INLINE decltype(auto) SetUserAccessible(bool en) { this->SetBit(7, en); return *this; } + constexpr ALWAYS_INLINE decltype(auto) SetPageAttribute(PageAttribute a) { this->SetBitsDirect(2, 3, a); return *this; } + + constexpr ALWAYS_INLINE u64 GetEntryTemplate() const { + constexpr u64 Mask = (0xFFF0000000000FFFul & ~u64(0x3ul | (0x1ul << 52))); + return this->attributes & Mask; + } + + constexpr ALWAYS_INLINE bool Is(u64 attr) const { + return this->attributes == attr; + } + + protected: + constexpr ALWAYS_INLINE u64 GetRawAttributes() const { + return this->attributes; + } + }; + + static_assert(sizeof(PageTableEntry) == sizeof(u64)); + + constexpr inline PageTableEntry InvalidPageTableEntry = PageTableEntry(PageTableEntry::InvalidTag{}); + + constexpr inline size_t MaxPageTableEntries = PageSize / sizeof(PageTableEntry); + + class L1PageTableEntry : public PageTableEntry { + public: + constexpr ALWAYS_INLINE L1PageTableEntry(InvalidTag) : PageTableEntry(InvalidTag{}) { /* ... */ } + + constexpr ALWAYS_INLINE L1PageTableEntry(KPhysicalAddress phys_addr, bool pxn) + : PageTableEntry((0x3ul << 60) | (static_cast(pxn) << 59) | GetInteger(phys_addr) | 0x3) + { + /* ... */ + } + + constexpr ALWAYS_INLINE L1PageTableEntry(KPhysicalAddress phys_addr, bool is_kernel, bool pxn) + : PageTableEntry(((is_kernel ? 0x3ul : 0) << 60) | (static_cast(pxn) << 59) | GetInteger(phys_addr) | 0x3) + { + /* ... */ + } + + constexpr ALWAYS_INLINE L1PageTableEntry(KPhysicalAddress phys_addr, const PageTableEntry &attr, bool contig) + : PageTableEntry(attr, (static_cast(contig) << 52) | GetInteger(phys_addr) | 0x1) + { + /* ... */ + } + + constexpr ALWAYS_INLINE KPhysicalAddress GetBlock() const { + return this->SelectBits(30, 18); + } + + constexpr ALWAYS_INLINE KPhysicalAddress GetTable() const { + return this->SelectBits(12, 36); + } + + constexpr ALWAYS_INLINE bool GetTable(KPhysicalAddress &out) const { + if (this->IsTable()) { + out = this->GetTable(); + return true; + } else { + return false; + } + } + + constexpr ALWAYS_INLINE bool IsCompatibleWithAttribute(const PageTableEntry &rhs, bool contig) const { + /* Check whether this has the same permission/etc as the desired attributes. */ + return L1PageTableEntry(this->GetBlock(), rhs, contig).GetRawAttributes() == this->GetRawAttributes(); + } + }; + + class L2PageTableEntry : public PageTableEntry { + public: + constexpr ALWAYS_INLINE L2PageTableEntry(InvalidTag) : PageTableEntry(InvalidTag{}) { /* ... */ } + + constexpr ALWAYS_INLINE L2PageTableEntry(KPhysicalAddress phys_addr, bool pxn) + : PageTableEntry((0x3ul << 60) | (static_cast(pxn) << 59) | GetInteger(phys_addr) | 0x3) + { + /* ... */ + } + + constexpr ALWAYS_INLINE L2PageTableEntry(KPhysicalAddress phys_addr, bool is_kernel, bool pxn) + : PageTableEntry(((is_kernel ? 0x3ul : 0) << 60) | (static_cast(pxn) << 59) | GetInteger(phys_addr) | 0x3) + { + /* ... */ + } + + constexpr ALWAYS_INLINE L2PageTableEntry(KPhysicalAddress phys_addr, const PageTableEntry &attr, bool contig) + : PageTableEntry(attr, (static_cast(contig) << 52) | GetInteger(phys_addr) | 0x1) + { + /* ... */ + } + + constexpr ALWAYS_INLINE KPhysicalAddress GetBlock() const { + return this->SelectBits(21, 27); + } + + constexpr ALWAYS_INLINE KPhysicalAddress GetTable() const { + return this->SelectBits(12, 36); + } + + constexpr ALWAYS_INLINE bool GetTable(KPhysicalAddress &out) const { + if (this->IsTable()) { + out = this->GetTable(); + return true; + } else { + return false; + } + } + + constexpr ALWAYS_INLINE bool IsCompatibleWithAttribute(const PageTableEntry &rhs, bool contig) const { + /* Check whether this has the same permission/etc as the desired attributes. */ + return L2PageTableEntry(this->GetBlock(), rhs, contig).GetRawAttributes() == this->GetRawAttributes(); + } + }; + + class L3PageTableEntry : public PageTableEntry { + public: + constexpr ALWAYS_INLINE L3PageTableEntry(InvalidTag) : PageTableEntry(InvalidTag{}) { /* ... */ } + + constexpr ALWAYS_INLINE L3PageTableEntry(KPhysicalAddress phys_addr, const PageTableEntry &attr, bool contig) + : PageTableEntry(attr, (static_cast(contig) << 52) | GetInteger(phys_addr) | 0x3) + { + /* ... */ + } + + constexpr ALWAYS_INLINE bool IsBlock() const { return this->GetBits(0, 2) == 0x3; } + + constexpr ALWAYS_INLINE KPhysicalAddress GetBlock() const { + return this->SelectBits(12, 36); + } + + constexpr ALWAYS_INLINE bool IsCompatibleWithAttribute(const PageTableEntry &rhs, bool contig) const { + /* Check whether this has the same permission/etc as the desired attributes. */ + return L3PageTableEntry(this->GetBlock(), rhs, contig).GetRawAttributes() == this->GetRawAttributes(); + } + }; + + constexpr inline L1PageTableEntry InvalidL1PageTableEntry = L1PageTableEntry(PageTableEntry::InvalidTag{}); + constexpr inline L2PageTableEntry InvalidL2PageTableEntry = L2PageTableEntry(PageTableEntry::InvalidTag{}); + constexpr inline L3PageTableEntry InvalidL3PageTableEntry = L3PageTableEntry(PageTableEntry::InvalidTag{}); + +} diff --git a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table_impl.hpp b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table_impl.hpp index d92315ab3..6ed2e2113 100644 --- a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table_impl.hpp +++ b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table_impl.hpp @@ -18,6 +18,7 @@ #include #include #include +#include namespace ams::kern::arm64 { @@ -28,15 +29,48 @@ namespace ams::kern::arm64 { NON_COPYABLE(KPageTableImpl); NON_MOVEABLE(KPageTableImpl); private: - u64 *table; + static constexpr size_t PageBits = __builtin_ctzll(PageSize); + static constexpr size_t NumLevels = 3; + static constexpr size_t LevelBits = 9; + static_assert(NumLevels > 0); + + static constexpr size_t AddressBits = (NumLevels - 1) * LevelBits + PageBits; + static_assert(AddressBits <= BITSIZEOF(u64)); + static constexpr size_t AddressSpaceSize = (1ull << AddressBits); + private: + L1PageTableEntry *table; bool is_kernel; u32 num_entries; + public: + ALWAYS_INLINE KVirtualAddress GetTableEntry(KVirtualAddress table, size_t index) { + return table + index * sizeof(PageTableEntry); + } + + ALWAYS_INLINE L1PageTableEntry *GetL1Entry(KProcessAddress address) { + return GetPointer(GetTableEntry(KVirtualAddress(this->table), (GetInteger(address) >> (PageBits + LevelBits * 2)) & (this->num_entries - 1))); + } + + ALWAYS_INLINE L2PageTableEntry *GetL2EntryFromTable(KVirtualAddress table, KProcessAddress address) { + return GetPointer(GetTableEntry(table, (GetInteger(address) >> (PageBits + LevelBits * 1)) & ((1ul << LevelBits) - 1))); + } + + ALWAYS_INLINE L2PageTableEntry *GetL2Entry(const L1PageTableEntry *entry, KProcessAddress address) { + return GetL2EntryFromTable(KMemoryLayout::GetLinearVirtualAddress(entry->GetTable()), address); + } + + ALWAYS_INLINE L3PageTableEntry *GetL3EntryFromTable(KVirtualAddress table, KProcessAddress address) { + return GetPointer(GetTableEntry(table, (GetInteger(address) >> (PageBits + LevelBits * 0)) & ((1ul << LevelBits) - 1))); + } + + ALWAYS_INLINE L3PageTableEntry *GetL3Entry(const L2PageTableEntry *entry, KProcessAddress address) { + return GetL3EntryFromTable(KMemoryLayout::GetLinearVirtualAddress(entry->GetTable()), address); + } public: constexpr KPageTableImpl() : table(), is_kernel(), num_entries() { /* ... */ } NOINLINE void InitializeForKernel(void *tb, KVirtualAddress start, KVirtualAddress end); - u64 *Finalize(); + L1PageTableEntry *Finalize(); }; } diff --git a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_supervisor_page_table.hpp b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_supervisor_page_table.hpp index 9254b34b4..e79faf6ca 100644 --- a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_supervisor_page_table.hpp +++ b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_supervisor_page_table.hpp @@ -30,6 +30,10 @@ namespace ams::kern::arm64 { NOINLINE void Initialize(s32 core_id); NOINLINE void Activate(); void Finalize(s32 core_id); + + Result MapPages(KProcessAddress *out_addr, size_t num_pages, size_t alignment, KPhysicalAddress phys_addr, KProcessAddress region_start, size_t region_num_pages, KMemoryState state, KMemoryPermission perm) { + return this->page_table.MapPages(out_addr, num_pages, alignment, phys_addr, region_start, region_num_pages, state, perm); + } }; } diff --git a/libraries/libmesosphere/include/mesosphere/init/kern_init_page_table_select.hpp b/libraries/libmesosphere/include/mesosphere/init/kern_init_page_table_select.hpp index c68b00d56..7a22b33cc 100644 --- a/libraries/libmesosphere/include/mesosphere/init/kern_init_page_table_select.hpp +++ b/libraries/libmesosphere/include/mesosphere/init/kern_init_page_table_select.hpp @@ -14,9 +14,16 @@ * along with this program. If not, see . */ #pragma once +#include #ifdef ATMOSPHERE_ARCH_ARM64 #include + + namespace ams::kern::init { + using ams::kern::arm64::PageTableEntry; + using ams::kern::arm64::init::KInitialPageTable; + using ams::kern::arm64::init::KInitialPageAllocator; + } #else #error "Unknown architecture for KInitialPageTable" #endif diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_dynamic_slab_heap.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_dynamic_slab_heap.hpp index 6fb9c3eca..2b8e8fbd2 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_dynamic_slab_heap.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_dynamic_slab_heap.hpp @@ -103,8 +103,11 @@ namespace ams::kern { } } - /* Update our tracking. */ if (AMS_LIKELY(allocated != nullptr)) { + /* Construct the object. */ + new (allocated) T(); + + /* Update our tracking. */ size_t used = ++this->used; size_t peak = this->peak; while (peak < used) { diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_light_lock.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_light_lock.hpp index 5de6009d8..0c0f601a7 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_light_lock.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_light_lock.hpp @@ -59,6 +59,9 @@ namespace ams::kern { void LockSlowPath(uintptr_t owner, uintptr_t cur_thread); void UnlockSlowPath(uintptr_t cur_thread); + + bool IsLocked() const { return this->tag != 0; } + bool IsLockedByCurrentThread() const { return (this->tag | 0x1ul) == (reinterpret_cast(GetCurrentThreadPointer()) | 0x1ul); } }; using KScopedLightLock = KScopedLock; diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_memory_block.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_memory_block.hpp index f7df252e2..7162cdb81 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_memory_block.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_memory_block.hpp @@ -23,6 +23,7 @@ namespace ams::kern { enum KMemoryState : u32 { KMemoryState_None = 0, KMemoryState_Mask = 0xFF, + KMemoryState_All = ~KMemoryState_None, KMemoryState_FlagCanReprotect = (1 << 8), KMemoryState_FlagCanDebug = (1 << 9), @@ -133,24 +134,25 @@ namespace ams::kern { enum KMemoryPermission : u8 { KMemoryPermission_None = 0, - - KMemoryPermission_UserRead = ams::svc::MemoryPermission_Read, - KMemoryPermission_UserWrite = ams::svc::MemoryPermission_Write, - KMemoryPermission_UserExecute = ams::svc::MemoryPermission_Execute, - - KMemoryPermission_UserReadWrite = ams::svc::MemoryPermission_ReadWrite, - KMemoryPermission_UserReadExecute = ams::svc::MemoryPermission_ReadExecute, - - KMemoryPermission_UserMask = KMemoryPermission_UserRead | KMemoryPermission_UserWrite | KMemoryPermission_UserExecute, + KMemoryPermission_All = static_cast(~KMemoryPermission_None), KMemoryPermission_KernelShift = 3, - KMemoryPermission_KernelRead = KMemoryPermission_UserRead << KMemoryPermission_KernelShift, - KMemoryPermission_KernelWrite = KMemoryPermission_UserWrite << KMemoryPermission_KernelShift, - KMemoryPermission_KernelExecute = KMemoryPermission_UserExecute << KMemoryPermission_KernelShift, + KMemoryPermission_KernelRead = ams::svc::MemoryPermission_Read << KMemoryPermission_KernelShift, + KMemoryPermission_KernelWrite = ams::svc::MemoryPermission_Write << KMemoryPermission_KernelShift, + KMemoryPermission_KernelExecute = ams::svc::MemoryPermission_Execute << KMemoryPermission_KernelShift, KMemoryPermission_KernelReadWrite = KMemoryPermission_KernelRead | KMemoryPermission_KernelWrite, KMemoryPermission_KernelReadExecute = KMemoryPermission_KernelRead | KMemoryPermission_KernelExecute, + + KMemoryPermission_UserRead = ams::svc::MemoryPermission_Read | KMemoryPermission_KernelRead, + KMemoryPermission_UserWrite = ams::svc::MemoryPermission_Write | KMemoryPermission_KernelWrite, + KMemoryPermission_UserExecute = ams::svc::MemoryPermission_Execute, + + KMemoryPermission_UserReadWrite = KMemoryPermission_UserRead | KMemoryPermission_UserWrite, + KMemoryPermission_UserReadExecute = KMemoryPermission_UserRead | KMemoryPermission_UserExecute, + + KMemoryPermission_UserMask = ams::svc::MemoryPermission_Read | ams::svc::MemoryPermission_Write | ams::svc::MemoryPermission_Execute, }; constexpr KMemoryPermission ConvertToKMemoryPermission(ams::svc::MemoryPermission perm) { @@ -160,6 +162,7 @@ namespace ams::kern { enum KMemoryAttribute : u8 { KMemoryAttribute_None = 0x00, KMemoryAttribute_Mask = 0x7F, + KMemoryAttribute_All = KMemoryAttribute_Mask, KMemoryAttribute_DontCareMask = 0x80, KMemoryAttribute_Locked = ams::svc::MemoryAttribute_Locked, @@ -182,17 +185,15 @@ namespace ams::kern { u16 device_use_count; constexpr ams::svc::MemoryInfo GetSvcMemoryInfo() const { - ams::svc::MemoryInfo svc_info = {}; - - svc_info.addr = this->address; - svc_info.size = this->size; - svc_info.state = static_cast(this->state & KMemoryState_Mask); - svc_info.attr = static_cast(this->attribute & KMemoryAttribute_Mask); - svc_info.perm = static_cast(this->perm & KMemoryPermission_UserMask); - svc_info.ipc_refcount = this->ipc_lock_count; - svc_info.device_refcount = this->device_use_count; - - return svc_info; + return { + .addr = this->address, + .size = this->size, + .state = static_cast(this->state & KMemoryState_Mask), + .attr = static_cast(this->attribute & KMemoryAttribute_Mask), + .perm = static_cast(this->perm & KMemoryPermission_UserMask), + .ipc_refcount = this->ipc_lock_count, + .device_refcount = this->device_use_count, + }; } constexpr uintptr_t GetAddress() const { @@ -258,18 +259,16 @@ namespace ams::kern { } constexpr KMemoryInfo GetMemoryInfo() const { - KMemoryInfo info = {}; - - info.address = GetInteger(this->GetAddress()); - info.size = this->GetSize(); - info.state = this->memory_state; - info.perm = this->perm; - info.attribute = this->attribute; - info.original_perm = this->original_perm; - info.ipc_lock_count = this->ipc_lock_count; - info.device_use_count = this->device_use_count; - - return info; + return { + .address = GetInteger(this->GetAddress()), + .size = this->GetSize(), + .state = this->memory_state, + .perm = this->perm, + .attribute = this->attribute, + .original_perm = this->original_perm, + .ipc_lock_count = this->ipc_lock_count, + .device_use_count = this->device_use_count, + }; } public: constexpr KMemoryBlock() diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_memory_block_manager.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_memory_block_manager.hpp index 1185653d7..ca24b2b99 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_memory_block_manager.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_memory_block_manager.hpp @@ -79,16 +79,21 @@ namespace ams::kern { using const_iterator = MemoryBlockTree::const_iterator; private: MemoryBlockTree memory_block_tree; - KProcessAddress start; - KProcessAddress end; + KProcessAddress start_address; + KProcessAddress end_address; public: - constexpr KMemoryBlockManager() : memory_block_tree(), start(), end() { /* ... */ } + constexpr KMemoryBlockManager() : memory_block_tree(), start_address(), end_address() { /* ... */ } + + iterator end() { return this->memory_block_tree.end(); } + const_iterator end() const { return this->memory_block_tree.end(); } + const_iterator cend() const { return this->memory_block_tree.cend(); } Result Initialize(KProcessAddress st, KProcessAddress nd, KMemoryBlockSlabManager *slab_manager); void Finalize(KMemoryBlockSlabManager *slab_manager); - void Update(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages, KMemoryState state, KMemoryPermission perm, KMemoryAttribute attr); + KProcessAddress FindFreeArea(KProcessAddress region_start, size_t region_num_pages, size_t num_pages, size_t alignment, size_t offset, size_t guard_pages) const; + void Update(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages, KMemoryState state, KMemoryPermission perm, KMemoryAttribute attr); iterator FindIterator(KProcessAddress address) const { return this->memory_block_tree.find(KMemoryBlock(address, 1, KMemoryState_Free, KMemoryPermission_None, KMemoryAttribute_None)); diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp index 8fb96c6a3..a5216097c 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp @@ -131,10 +131,10 @@ namespace ams::kern { static constexpr ALWAYS_INLINE int Compare(const KMemoryRegion &lhs, const KMemoryRegion &rhs) { if (lhs.GetAddress() < rhs.GetAddress()) { return -1; - } else if (lhs.GetLastAddress() > rhs.GetLastAddress()) { - return 1; - } else { + } else if (lhs.GetAddress() <= rhs.GetLastAddress()) { return 0; + } else { + return 1; } } public: @@ -246,11 +246,7 @@ namespace ams::kern { constexpr ALWAYS_INLINE KMemoryRegionTree() : tree() { /* ... */ } public: iterator FindContainingRegion(uintptr_t address) { - auto it = this->find(KMemoryRegion(address, 1, 0, 0)); - MESOSPHERE_INIT_ABORT_UNLESS(it != this->end()); - MESOSPHERE_INIT_ABORT_UNLESS(it->Contains(address)); - - return it; + return this->find(KMemoryRegion(address, 1, 0, 0)); } iterator FindFirstRegionByTypeAttr(u32 type_id, u32 attr = 0) { @@ -483,6 +479,16 @@ namespace ams::kern { return *GetVirtualLinearMemoryRegionTree().FindContainingRegion(GetInteger(address)); } + static NOINLINE bool IsHeapPhysicalAddress(KMemoryRegion **out, KPhysicalAddress address) { + if (auto it = GetPhysicalLinearMemoryRegionTree().FindContainingRegion(GetInteger(address)); it != GetPhysicalLinearMemoryRegionTree().end() && it->IsDerivedFrom(KMemoryRegionType_DramNonKernel)) { + if (out) { + *out = std::addressof(*it); + } + return true; + } + return false; + } + static NOINLINE std::tuple GetTotalAndKernelMemorySizes() { size_t total_size = 0, kernel_size = 0; for (auto it = GetPhysicalMemoryRegionTree().cbegin(); it != GetPhysicalMemoryRegionTree().cend(); it++) { diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_memory_manager.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_memory_manager.hpp index 206d9b1d3..7213dc7e7 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_memory_manager.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_memory_manager.hpp @@ -21,6 +21,8 @@ namespace ams::kern { + class KPageGroup; + class KMemoryManager { public: enum Pool { @@ -135,6 +137,18 @@ namespace ams::kern { Impl &GetManager(KVirtualAddress address) { return this->managers[KMemoryLayout::GetVirtualLinearRegion(address).GetAttributes()]; } + + constexpr Impl *GetFirstManager(Pool pool, Direction dir) { + return dir == Direction_FromBack ? this->pool_managers_tail[pool] : this->pool_managers_head[pool]; + } + + constexpr Impl *GetNextManager(Impl *cur, Direction dir) { + if (dir == Direction_FromBack) { + return cur->GetPrev(); + } else { + return cur->GetNext(); + } + } public: constexpr KMemoryManager() : pool_locks(), pool_managers_head(), pool_managers_tail(), managers(), num_managers(), optimized_process_ids(), has_optimized_process() @@ -144,7 +158,8 @@ namespace ams::kern { NOINLINE void Initialize(KVirtualAddress metadata_region, size_t metadata_region_size); - KVirtualAddress AllocateContinuous(size_t num_pages, size_t align_pages, u32 option); + NOINLINE KVirtualAddress AllocateContinuous(size_t num_pages, size_t align_pages, u32 option); + NOINLINE Result Allocate(KPageGroup *out, size_t num_pages, u32 option); void Open(KVirtualAddress address, size_t num_pages) { /* Repeatedly open references until we've done so for all pages. */ diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_page_buffer.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_page_buffer.hpp index b9a99c2c2..980fc6c36 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_page_buffer.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_page_buffer.hpp @@ -27,6 +27,10 @@ namespace ams::kern { std::memset(buffer, 0, sizeof(buffer)); } + ALWAYS_INLINE KPhysicalAddress GetPhysicalAddress() const { + return KMemoryLayout::GetLinearPhysicalAddress(KVirtualAddress(this)); + } + static ALWAYS_INLINE KPageBuffer *FromPhysicalAddress(KPhysicalAddress phys_addr) { const KVirtualAddress virt_addr = KMemoryLayout::GetLinearVirtualAddress(phys_addr); diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_page_group.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_page_group.hpp index c3de0a955..e70d6cc5f 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_page_group.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_page_group.hpp @@ -27,7 +27,7 @@ namespace ams::kern { KVirtualAddress address; size_t num_pages; public: - constexpr KBlockInfo() : address(), num_pages() { /* ... */ } + constexpr KBlockInfo() : util::IntrusiveListBaseNode(), address(), num_pages() { /* ... */ } constexpr void Initialize(KVirtualAddress addr, size_t np) { this->address = addr; @@ -82,9 +82,9 @@ namespace ams::kern { BlockInfoList block_list; KBlockInfoManager *manager; public: - KPageGroup() : block_list(), manager() { /* ... */ } + explicit KPageGroup(KBlockInfoManager *m) : block_list(), manager(m) { /* ... */ } + ~KPageGroup() { this->Finalize(); } - void Initialize(KBlockInfoManager *m); void Finalize(); iterator begin() const { return this->block_list.begin(); } @@ -107,4 +107,14 @@ namespace ams::kern { } }; + class KScopedPageGroup { + private: + KPageGroup *group; + public: + explicit ALWAYS_INLINE KScopedPageGroup(KPageGroup *gp) : group(gp) { group->Open(); } + explicit ALWAYS_INLINE KScopedPageGroup(KPageGroup &gp) : KScopedPageGroup(std::addressof(gp)) { /* ... */ } + ALWAYS_INLINE ~KScopedPageGroup() { group->Close(); } + + }; + } \ No newline at end of file diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_page_heap.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_page_heap.hpp index 913a95630..54feb2231 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_page_heap.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_page_heap.hpp @@ -34,9 +34,9 @@ namespace ams::kern { } static constexpr s32 GetBlockIndex(size_t num_pages) { - for (size_t i = 0; i < NumMemoryBlockPageShifts; i++) { + for (s32 i = static_cast(NumMemoryBlockPageShifts) - 1; i >= 0; i--) { if (num_pages >= (size_t(1) << MemoryBlockPageShifts[i]) / PageSize) { - return static_cast(i); + return i; } } return -1; diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_page_table_base.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_page_table_base.hpp index c5937a388..f2a3c3053 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_page_table_base.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_page_table_base.hpp @@ -24,6 +24,14 @@ namespace ams::kern { + struct KPageProperties { + KMemoryPermission perm; + bool io; + bool uncached; + bool non_contiguous; + }; + static_assert(std::is_trivial::value); + class KPageTableBase { NON_COPYABLE(KPageTableBase); NON_MOVEABLE(KPageTableBase); @@ -34,6 +42,60 @@ namespace ams::kern { MemoryFillValue_Ipc = 'Y', MemoryFillValue_Heap = 'Z', }; + + enum OperationType { + OperationType_Map = 0, + OperationType_MapGroup = 1, + OperationType_Unmap = 2, + /* TODO: perm/attr operations */ + }; + + + struct PageLinkedList { + private: + struct Node { + Node *next; + u8 buffer[PageSize - sizeof(Node *)]; + }; + static_assert(std::is_pod::value); + private: + Node *root; + public: + constexpr PageLinkedList() : root(nullptr) { /* ... */ } + + void Push(Node *n) { + MESOSPHERE_ASSERT(util::IsAligned(reinterpret_cast(n), PageSize)); + n->next = this->root; + this->root = n; + } + + void Push(KVirtualAddress addr) { + this->Push(GetPointer(addr)); + } + + Node *Peek() const { return this->root; } + + Node *Pop() { + Node *r = this->root; + this->root = this->root->next; + return r; + } + }; + static_assert(std::is_trivially_destructible::value); + + static constexpr u32 DefaultMemoryIgnoreAttr = KMemoryAttribute_DontCareMask | KMemoryAttribute_IpcLocked | KMemoryAttribute_DeviceShared; + private: + class KScopedPageTableUpdater { + private: + KPageTableBase *page_table; + PageLinkedList ll; + public: + ALWAYS_INLINE explicit KScopedPageTableUpdater(KPageTableBase *pt) : page_table(pt), ll() { /* ... */ } + ALWAYS_INLINE explicit KScopedPageTableUpdater(KPageTableBase &pt) : KScopedPageTableUpdater(std::addressof(pt)) { /* ... */ } + ALWAYS_INLINE ~KScopedPageTableUpdater() { this->page_table->FinalizeUpdate(this->GetPageList()); } + + PageLinkedList *GetPageList() { return std::addressof(this->ll); } + }; private: KProcessAddress address_space_start; KProcessAddress address_space_end; @@ -63,7 +125,7 @@ namespace ams::kern { KMemoryBlockSlabManager *memory_block_slab_manager; KBlockInfoManager *block_info_manager; KMemoryRegion *cached_physical_linear_region; - KMemoryRegion *cached_physical_non_kernel_dram_region; + KMemoryRegion *cached_physical_heap_region; KMemoryRegion *cached_virtual_managed_pool_dram_region; MemoryFillValue heap_fill_value; MemoryFillValue ipc_fill_value; @@ -75,7 +137,7 @@ namespace ams::kern { kernel_map_region_end(), alias_code_region_start(), alias_code_region_end(), code_region_start(), code_region_end(), max_heap_size(), max_physical_memory_size(), general_lock(), map_physical_memory_lock(), impl(), memory_block_manager(), allocate_option(), address_space_size(), is_kernel(), enable_aslr(), memory_block_slab_manager(), block_info_manager(), - cached_physical_linear_region(), cached_physical_non_kernel_dram_region(), cached_virtual_managed_pool_dram_region(), + cached_physical_linear_region(), cached_physical_heap_region(), cached_virtual_managed_pool_dram_region(), heap_fill_value(), ipc_fill_value(), stack_fill_value() { /* ... */ @@ -84,6 +146,59 @@ namespace ams::kern { NOINLINE Result InitializeForKernel(bool is_64_bit, void *table, KVirtualAddress start, KVirtualAddress end); void Finalize(); + + constexpr bool IsKernel() const { return this->is_kernel; } + constexpr bool IsAslrEnabled() const { return this->enable_aslr; } + + constexpr bool Contains(KProcessAddress addr) const { + return this->address_space_start <= addr && addr <= this->address_space_end - 1; + } + + constexpr bool Contains(KProcessAddress addr, size_t size) const { + return this->address_space_start <= addr && addr < addr + size && addr + size - 1 <= this->address_space_end - 1; + } + + KProcessAddress GetRegionAddress(KMemoryState state) const; + size_t GetRegionSize(KMemoryState state) const; + bool Contains(KProcessAddress addr, size_t size, KMemoryState state) const; + protected: + virtual Result Operate(PageLinkedList *page_list, KProcessAddress virt_addr, size_t num_pages, KPhysicalAddress phys_addr, bool is_pa_valid, const KPageProperties properties, OperationType operation, bool reuse_ll) = 0; + virtual Result Operate(PageLinkedList *page_list, KProcessAddress virt_addr, size_t num_pages, const KPageGroup *page_group, const KPageProperties properties, OperationType operation, bool reuse_ll) = 0; + virtual void FinalizeUpdate(PageLinkedList *page_list) = 0; + + KPageTableImpl &GetImpl() { return this->impl; } + const KPageTableImpl &GetImpl() const { return this->impl; } + + bool IsLockedByCurrentThread() const { return this->general_lock.IsLockedByCurrentThread(); } + + bool IsHeapPhysicalAddress(KPhysicalAddress phys_addr) { + if (this->cached_physical_heap_region && this->cached_physical_heap_region->Contains(GetInteger(phys_addr))) { + return true; + } + return KMemoryLayout::IsHeapPhysicalAddress(&this->cached_physical_heap_region, phys_addr); + } + + bool ContainsPages(KProcessAddress addr, size_t num_pages) const { + return (this->address_space_start <= addr) && (num_pages <= (this->address_space_end - this->address_space_start) / PageSize) && (addr + num_pages * PageSize - 1 <= this->address_space_end - 1); + } + private: + constexpr size_t GetNumGuardPages() const { return this->IsKernel() ? 1 : 4; } + ALWAYS_INLINE KProcessAddress FindFreeArea(KProcessAddress region_start, size_t region_num_pages, size_t num_pages, size_t alignment, size_t offset, size_t guard_pages) const; + + Result CheckMemoryState(const KMemoryInfo &info, u32 state_mask, u32 state, u32 perm_mask, u32 perm, u32 attr_mask, u32 attr) const; + Result CheckMemoryState(KMemoryState *out_state, KMemoryPermission *out_perm, KMemoryAttribute *out_attr, KProcessAddress addr, size_t size, u32 state_mask, u32 state, u32 perm_mask, u32 perm, u32 attr_mask, u32 attr, u32 ignore_attr = DefaultMemoryIgnoreAttr) const; + Result CheckMemoryState(KProcessAddress addr, size_t size, u32 state_mask, u32 state, u32 perm_mask, u32 perm, u32 attr_mask, u32 attr, u32 ignore_attr = DefaultMemoryIgnoreAttr) const { + return this->CheckMemoryState(nullptr, nullptr, nullptr, addr, size, state_mask, state, perm_mask, perm, attr_mask, attr, ignore_attr); + } + + Result QueryInfoImpl(KMemoryInfo *out_info, ams::svc::PageInfo *out_page, KProcessAddress address) const; + Result AllocateAndMapPagesImpl(PageLinkedList *page_list, KProcessAddress address, size_t num_pages, const KPageProperties properties); + + NOINLINE Result MapPages(KProcessAddress *out_addr, size_t num_pages, size_t alignment, KPhysicalAddress phys_addr, bool is_pa_valid, KProcessAddress region_start, size_t region_num_pages, KMemoryState state, KMemoryPermission perm); + public: + Result MapPages(KProcessAddress *out_addr, size_t num_pages, size_t alignment, KPhysicalAddress phys_addr, KProcessAddress region_start, size_t region_num_pages, KMemoryState state, KMemoryPermission perm) { + return this->MapPages(out_addr, num_pages, alignment, phys_addr, true, region_start, region_num_pages, state, perm); + } public: static ALWAYS_INLINE KVirtualAddress GetLinearVirtualAddress(KPhysicalAddress addr) { return KMemoryLayout::GetLinearVirtualAddress(addr); @@ -92,6 +207,18 @@ namespace ams::kern { static ALWAYS_INLINE KPhysicalAddress GetLinearPhysicalAddress(KVirtualAddress addr) { return KMemoryLayout::GetLinearPhysicalAddress(addr); } + + static ALWAYS_INLINE KVirtualAddress GetHeapVirtualAddress(KPhysicalAddress addr) { + return GetLinearVirtualAddress(addr); + } + + static ALWAYS_INLINE KVirtualAddress GetPageTableVirtualAddress(KPhysicalAddress addr) { + return GetLinearVirtualAddress(addr); + } + + static ALWAYS_INLINE KPhysicalAddress GetPageTablePhysicalAddress(KVirtualAddress addr) { + return GetLinearPhysicalAddress(addr); + } }; } diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_page_table_manager.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_page_table_manager.hpp index 6695adc57..1b12e82d1 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_page_table_manager.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_page_table_manager.hpp @@ -93,6 +93,10 @@ namespace ams::kern { *this->GetRefCountPointer(addr) -= count; return this->GetRefCount(addr) == 0; } + + constexpr bool IsInPageTableHeap(KVirtualAddress addr) const { + return this->IsInRange(addr); + } }; } diff --git a/libraries/libmesosphere/include/mesosphere/svc/kern_svc_results.hpp b/libraries/libmesosphere/include/mesosphere/svc/kern_svc_results.hpp index 3dff785ea..78d3599c1 100644 --- a/libraries/libmesosphere/include/mesosphere/svc/kern_svc_results.hpp +++ b/libraries/libmesosphere/include/mesosphere/svc/kern_svc_results.hpp @@ -35,7 +35,7 @@ namespace ams::kern::svc { /* 103 */ using ::ams::svc::ResultOutOfResource; /* 104 */ using ::ams::svc::ResultOutOfMemory; /* 105 */ using ::ams::svc::ResultOutOfHandles; - /* 106 */ using ::ams::svc::ResultInvalidCurrentMemoryState; + /* 106 */ using ::ams::svc::ResultInvalidCurrentMemory; /* 108 */ using ::ams::svc::ResultInvalidNewMemoryPermissions; diff --git a/libraries/libmesosphere/source/arch/arm64/kern_k_page_table.cpp b/libraries/libmesosphere/source/arch/arm64/kern_k_page_table.cpp index 07663da03..d2bd5e1f0 100644 --- a/libraries/libmesosphere/source/arch/arm64/kern_k_page_table.cpp +++ b/libraries/libmesosphere/source/arch/arm64/kern_k_page_table.cpp @@ -42,4 +42,309 @@ namespace ams::kern::arm64 { Result KPageTable::Finalize() { MESOSPHERE_TODO_IMPLEMENT(); } + + Result KPageTable::Operate(PageLinkedList *page_list, KProcessAddress virt_addr, size_t num_pages, KPhysicalAddress phys_addr, bool is_pa_valid, const KPageProperties properties, OperationType operation, bool reuse_ll) { + /* Check validity of parameters. */ + MESOSPHERE_ASSERT(this->IsLockedByCurrentThread()); + MESOSPHERE_ASSERT(num_pages > 0); + MESOSPHERE_ASSERT(util::IsAligned(GetInteger(virt_addr), PageSize)); + MESOSPHERE_ASSERT(this->ContainsPages(virt_addr, num_pages)); + + if (operation == OperationType_Map) { + MESOSPHERE_ABORT_UNLESS(is_pa_valid); + MESOSPHERE_ASSERT(util::IsAligned(GetInteger(phys_addr), PageSize)); + } else { + MESOSPHERE_ABORT_UNLESS(!is_pa_valid); + } + + if (operation == OperationType_Unmap) { + MESOSPHERE_TODO("operation == OperationType_Unmap"); + } else { + auto entry_template = this->GetEntryTemplate(properties); + + switch (operation) { + case OperationType_Map: + return this->MapContiguous(virt_addr, phys_addr, num_pages, entry_template, page_list, reuse_ll); + MESOSPHERE_UNREACHABLE_DEFAULT_CASE(); + } + } + } + + Result KPageTable::Operate(PageLinkedList *page_list, KProcessAddress virt_addr, size_t num_pages, const KPageGroup *page_group, const KPageProperties properties, OperationType operation, bool reuse_ll) { + MESOSPHERE_TODO_IMPLEMENT(); + } + + Result KPageTable::Map(KProcessAddress virt_addr, KPhysicalAddress phys_addr, size_t num_pages, PageTableEntry entry_template, PageLinkedList *page_list, bool reuse_ll) { + MESOSPHERE_ASSERT(this->IsLockedByCurrentThread()); + MESOSPHERE_ASSERT(util::IsAligned(GetInteger(virt_addr), PageSize)); + MESOSPHERE_ASSERT(util::IsAligned(GetInteger(phys_addr), PageSize)); + + auto &impl = this->GetImpl(); + KVirtualAddress l2_virt = Null; + KVirtualAddress l3_virt = Null; + int l2_open_count = 0; + int l3_open_count = 0; + + /* Iterate, mapping each page. */ + for (size_t i = 0; i < num_pages; i++) { + KPhysicalAddress l3_phys = Null; + bool l2_allocated = false; + + /* If we have no L3 table, we should get or allocate one. */ + if (l3_virt == Null) { + KPhysicalAddress l2_phys = Null; + + /* If we have no L2 table, we should get or allocate one. */ + if (l2_virt == Null) { + if (L1PageTableEntry *l1_entry = impl.GetL1Entry(virt_addr); !l1_entry->GetTable(l2_phys)) { + /* Allocate table. */ + l2_virt = AllocatePageTable(page_list, reuse_ll); + R_UNLESS(l2_virt != Null, svc::ResultOutOfResource()); + + /* Set the entry. */ + l2_phys = GetPageTablePhysicalAddress(l2_virt); + PteDataSynchronizationBarrier(); + *l1_entry = L1PageTableEntry(l2_phys, this->IsKernel(), true); + PteDataSynchronizationBarrier(); + l2_allocated = true; + } else { + l2_virt = GetPageTableVirtualAddress(l2_phys); + } + } + MESOSPHERE_ASSERT(l2_virt != Null); + + if (L2PageTableEntry *l2_entry = impl.GetL2EntryFromTable(l2_virt, virt_addr); !l2_entry->GetTable(l3_phys)) { + /* Allocate table. */ + l3_virt = AllocatePageTable(page_list, reuse_ll); + if (l3_virt == Null) { + /* Cleanup the L2 entry. */ + if (l2_allocated) { + *impl.GetL1Entry(virt_addr) = InvalidL1PageTableEntry; + this->NoteUpdated(); + FreePageTable(page_list, l2_virt); + } else if (this->GetPageTableManager().IsInPageTableHeap(l2_virt) && l2_open_count > 0) { + this->GetPageTableManager().Open(l2_virt, l2_open_count); + } + return svc::ResultOutOfResource(); + } + + /* Set the entry. */ + l3_phys = GetPageTablePhysicalAddress(l3_virt); + PteDataSynchronizationBarrier(); + *l2_entry = L2PageTableEntry(l3_phys, this->IsKernel(), true); + PteDataSynchronizationBarrier(); + l2_open_count++; + } else { + l3_virt = GetPageTableVirtualAddress(l3_phys); + } + } + MESOSPHERE_ASSERT(l3_virt != Null); + + /* Map the page. */ + *impl.GetL3EntryFromTable(l3_virt, virt_addr) = L3PageTableEntry(phys_addr, entry_template, false); + l3_open_count++; + virt_addr += PageSize; + phys_addr += PageSize; + + /* Account for hitting end of table. */ + if (util::IsAligned(GetInteger(virt_addr), L2BlockSize)) { + if (this->GetPageTableManager().IsInPageTableHeap(l3_virt)) { + this->GetPageTableManager().Open(l3_virt, l3_open_count); + } + l3_virt = Null; + l3_open_count = 0; + + if (util::IsAligned(GetInteger(virt_addr), L1BlockSize)) { + if (this->GetPageTableManager().IsInPageTableHeap(l2_virt) && l2_open_count > 0) { + this->GetPageTableManager().Open(l2_virt, l2_open_count); + } + l2_virt = Null; + l2_open_count = 0; + } + } + } + + /* Perform any remaining opens. */ + if (l2_open_count > 0 && this->GetPageTableManager().IsInPageTableHeap(l2_virt)) { + this->GetPageTableManager().Open(l2_virt, l2_open_count); + } + if (l3_open_count > 0 && this->GetPageTableManager().IsInPageTableHeap(l3_virt)) { + this->GetPageTableManager().Open(l3_virt, l3_open_count); + } + + return ResultSuccess(); + } + + Result KPageTable::Unmap(KProcessAddress virt_addr, size_t num_pages, KPageGroup *pg, PageLinkedList *page_list, bool force, bool reuse_ll) { + MESOSPHERE_TODO_IMPLEMENT(); + } + + Result KPageTable::MapContiguous(KProcessAddress virt_addr, KPhysicalAddress phys_addr, size_t num_pages, PageTableEntry entry_template, PageLinkedList *page_list, bool reuse_ll) { + MESOSPHERE_ASSERT(this->IsLockedByCurrentThread()); + MESOSPHERE_LOG("KPageTable::MapContiguous(%016lx, %016lx, %zu)\n", GetInteger(virt_addr), GetInteger(phys_addr), num_pages); + + /* Cache initial addresses for use on cleanup. */ + const KProcessAddress orig_virt_addr = virt_addr; + const KPhysicalAddress orig_phys_addr = phys_addr; + + size_t remaining_pages = num_pages; + + if (num_pages < ContiguousPageSize / PageSize) { + auto guard = SCOPE_GUARD { MESOSPHERE_R_ABORT_UNLESS(this->Unmap(orig_virt_addr, num_pages, nullptr, page_list, true, true)); }; + R_TRY(this->Map(virt_addr, phys_addr, num_pages, entry_template, page_list, reuse_ll)); + guard.Cancel(); + } else { + MESOSPHERE_TODO("Contiguous mapping"); + (void)remaining_pages; + } + + /* Perform what coalescing we can. */ + this->MergePages(orig_virt_addr, page_list); + if (num_pages > 1) { + this->MergePages(orig_virt_addr + (num_pages - 1) * PageSize, page_list); + } + + /* Open references to the pages, if we should. */ + if (IsHeapPhysicalAddress(orig_phys_addr)) { + Kernel::GetMemoryManager().Open(GetHeapVirtualAddress(orig_phys_addr), num_pages); + } + + return ResultSuccess(); + } + + bool KPageTable::MergePages(KProcessAddress virt_addr, PageLinkedList *page_list) { + MESOSPHERE_ASSERT(this->IsLockedByCurrentThread()); + + auto &impl = this->GetImpl(); + bool merged = false; + + /* If there's no L1 table, don't bother. */ + L1PageTableEntry *l1_entry = impl.GetL1Entry(virt_addr); + if (!l1_entry->IsTable()) { + return merged; + } + + /* Examine and try to merge the L2 table. */ + L2PageTableEntry *l2_entry = impl.GetL2Entry(l1_entry, virt_addr); + if (l2_entry->IsTable()) { + /* We have an L3 entry. */ + L3PageTableEntry *l3_entry = impl.GetL3Entry(l2_entry, virt_addr); + if (!l3_entry->IsBlock() || !l3_entry->IsContiguousAllowed()) { + return merged; + } + + /* If it's not contiguous, try to make it so. */ + if (!l3_entry->IsContiguous()) { + virt_addr = util::AlignDown(GetInteger(virt_addr), L3ContiguousBlockSize); + KPhysicalAddress phys_addr = util::AlignDown(GetInteger(l3_entry->GetBlock()), L3ContiguousBlockSize); + const u64 entry_template = l3_entry->GetEntryTemplate(); + + /* Validate that we can merge. */ + for (size_t i = 0; i < L3ContiguousBlockSize / L3BlockSize; i++) { + if (!impl.GetL3Entry(l2_entry, virt_addr + L3BlockSize * i)->Is(entry_template | GetInteger(phys_addr + PageSize * i) | PageTableEntry::Type_L3Block)) { + return merged; + } + } + + /* Merge! */ + for (size_t i = 0; i < L3ContiguousBlockSize / L3BlockSize; i++) { + impl.GetL3Entry(l2_entry, virt_addr + L3BlockSize * i)->SetContiguous(true); + } + + /* Note that we updated. */ + this->NoteUpdated(); + merged = true; + } + + /* We might be able to upgrade a contiguous set of L3 entries into an L2 block. */ + virt_addr = util::AlignDown(GetInteger(virt_addr), L2BlockSize); + KPhysicalAddress phys_addr = util::AlignDown(GetInteger(l3_entry->GetBlock()), L2BlockSize); + const u64 entry_template = l3_entry->GetEntryTemplate(); + + /* Validate that we can merge. */ + for (size_t i = 0; i < L2BlockSize / L3ContiguousBlockSize; i++) { + if (!impl.GetL3Entry(l2_entry, virt_addr + L3BlockSize * i)->Is(entry_template | GetInteger(phys_addr + L3ContiguousBlockSize * i) | PageTableEntry::ContigType_Contiguous)) { + return merged; + } + } + + /* Merge! */ + PteDataSynchronizationBarrier(); + *l2_entry = L2PageTableEntry(phys_addr, entry_template, false); + + /* Note that we updated. */ + this->NoteUpdated(); + merged = true; + + /* Free the L3 table. */ + KVirtualAddress l3_table = util::AlignDown(reinterpret_cast(l3_entry), PageSize); + if (this->GetPageTableManager().IsInPageTableHeap(l3_table)) { + this->GetPageTableManager().Close(l3_table, L2BlockSize / L3BlockSize); + this->FreePageTable(page_list, l3_table); + } + } + if (l2_entry->IsBlock()) { + /* If it's not contiguous, try to make it so. */ + if (!l2_entry->IsContiguous()) { + virt_addr = util::AlignDown(GetInteger(virt_addr), L2ContiguousBlockSize); + KPhysicalAddress phys_addr = util::AlignDown(GetInteger(l2_entry->GetBlock()), L2ContiguousBlockSize); + const u64 entry_template = l2_entry->GetEntryTemplate(); + + /* Validate that we can merge. */ + for (size_t i = 0; i < L2ContiguousBlockSize / L2BlockSize; i++) { + if (!impl.GetL2Entry(l1_entry, virt_addr + L2BlockSize * i)->Is(entry_template | GetInteger(phys_addr + PageSize * i) | PageTableEntry::Type_L2Block)) { + return merged; + } + } + + /* Merge! */ + for (size_t i = 0; i < L2ContiguousBlockSize / L2BlockSize; i++) { + impl.GetL2Entry(l1_entry, virt_addr + L2BlockSize * i)->SetContiguous(true); + } + + /* Note that we updated. */ + this->NoteUpdated(); + merged = true; + } + + /* We might be able to upgrade a contiguous set of L2 entries into an L1 block. */ + virt_addr = util::AlignDown(GetInteger(virt_addr), L1BlockSize); + KPhysicalAddress phys_addr = util::AlignDown(GetInteger(l2_entry->GetBlock()), L1BlockSize); + const u64 entry_template = l2_entry->GetEntryTemplate(); + + /* Validate that we can merge. */ + for (size_t i = 0; i < L1BlockSize / L2ContiguousBlockSize; i++) { + if (!impl.GetL2Entry(l1_entry, virt_addr + L3BlockSize * i)->Is(entry_template | GetInteger(phys_addr + L2ContiguousBlockSize * i) | PageTableEntry::ContigType_Contiguous)) { + return merged; + } + } + + /* Merge! */ + PteDataSynchronizationBarrier(); + *l1_entry = L1PageTableEntry(phys_addr, entry_template, false); + + /* Note that we updated. */ + this->NoteUpdated(); + merged = true; + + /* Free the L2 table. */ + KVirtualAddress l2_table = util::AlignDown(reinterpret_cast(l2_entry), PageSize); + if (this->GetPageTableManager().IsInPageTableHeap(l2_table)) { + this->GetPageTableManager().Close(l2_table, L1BlockSize / L2BlockSize); + this->FreePageTable(page_list, l2_table); + } + } + + return merged; + } + + void KPageTable::FinalizeUpdate(PageLinkedList *page_list) { + while (page_list->Peek()) { + KVirtualAddress page = KVirtualAddress(page_list->Pop()); + MESOSPHERE_ASSERT(this->GetPageTableManager().IsInPageTableHeap(page)); + MESOSPHERE_ASSERT(this->GetPageTableManager().GetRefCount(page) == 0); + this->GetPageTableManager().Free(page); + } + } + } diff --git a/libraries/libmesosphere/source/arch/arm64/kern_k_page_table_impl.cpp b/libraries/libmesosphere/source/arch/arm64/kern_k_page_table_impl.cpp index 06bee77cb..f27562bc0 100644 --- a/libraries/libmesosphere/source/arch/arm64/kern_k_page_table_impl.cpp +++ b/libraries/libmesosphere/source/arch/arm64/kern_k_page_table_impl.cpp @@ -17,26 +17,13 @@ namespace ams::kern::arm64 { - namespace { - - constexpr size_t PageBits = __builtin_ctzll(PageSize); - constexpr size_t NumLevels = 3; - constexpr size_t LevelBits = 9; - static_assert(NumLevels > 0); - - constexpr size_t AddressBits = (NumLevels - 1) * LevelBits + PageBits; - static_assert(AddressBits <= BITSIZEOF(u64)); - constexpr size_t AddressSpaceSize = (1ull << AddressBits); - - } - void KPageTableImpl::InitializeForKernel(void *tb, KVirtualAddress start, KVirtualAddress end) { - this->table = static_cast(tb); + this->table = static_cast(tb); this->is_kernel = true; this->num_entries = util::AlignUp(end - start, AddressSpaceSize) / AddressSpaceSize; } - u64 *KPageTableImpl::Finalize() { + L1PageTableEntry *KPageTableImpl::Finalize() { return this->table; } diff --git a/libraries/libmesosphere/source/kern_k_memory_block_manager.cpp b/libraries/libmesosphere/source/kern_k_memory_block_manager.cpp index c3095e40c..fb9014b78 100644 --- a/libraries/libmesosphere/source/kern_k_memory_block_manager.cpp +++ b/libraries/libmesosphere/source/kern_k_memory_block_manager.cpp @@ -23,13 +23,13 @@ namespace ams::kern { R_UNLESS(start_block != nullptr, svc::ResultOutOfResource()); /* Set our start and end. */ - this->start = st; - this->end = nd; - MESOSPHERE_ASSERT(util::IsAligned(GetInteger(this->start), PageSize)); - MESOSPHERE_ASSERT(util::IsAligned(GetInteger(this->end), PageSize)); + this->start_address = st; + this->end_address = nd; + MESOSPHERE_ASSERT(util::IsAligned(GetInteger(this->start_address), PageSize)); + MESOSPHERE_ASSERT(util::IsAligned(GetInteger(this->end_address), PageSize)); /* Initialize and insert the block. */ - start_block->Initialize(this->start, (this->end - this->start) / PageSize, KMemoryState_Free, KMemoryPermission_None, KMemoryAttribute_None); + start_block->Initialize(this->start_address, (this->end_address - this->start_address) / PageSize, KMemoryState_Free, KMemoryPermission_None, KMemoryAttribute_None); this->memory_block_tree.insert(*start_block); return ResultSuccess(); @@ -47,6 +47,37 @@ namespace ams::kern { MESOSPHERE_ASSERT(this->memory_block_tree.empty()); } + KProcessAddress KMemoryBlockManager::FindFreeArea(KProcessAddress region_start, size_t region_num_pages, size_t num_pages, size_t alignment, size_t offset, size_t guard_pages) const { + if (num_pages > 0) { + const KProcessAddress region_end = region_start + region_num_pages * PageSize; + const KProcessAddress region_last = region_end - 1; + for (const_iterator it = this->FindIterator(region_start); it != this->memory_block_tree.cend(); it++) { + const KMemoryInfo info = it->GetMemoryInfo(); + if (region_last < info.GetAddress()) { + break; + } + if (info.state != KMemoryState_Free) { + continue; + } + + KProcessAddress area = (info.GetAddress() <= GetInteger(region_start)) ? region_start : info.GetAddress(); + area += guard_pages * PageSize; + + const KProcessAddress offset_area = util::AlignDown(GetInteger(area), alignment) + offset; + area = (area <= offset_area) ? offset_area : offset_area + alignment; + + const KProcessAddress area_end = area + num_pages * PageSize + guard_pages * PageSize; + const KProcessAddress area_last = area_end - 1; + + if (info.GetAddress() <= GetInteger(area) && area < area_last && area_last <= region_last && GetInteger(area_last) <= info.GetLastAddress()) { + return area; + } + } + } + + return Null; + } + void KMemoryBlockManager::Update(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages, KMemoryState state, KMemoryPermission perm, KMemoryAttribute attr) { /* Ensure for auditing that we never end up with an invalid tree. */ KScopedMemoryBlockManagerAuditor auditor(this); @@ -101,7 +132,7 @@ namespace ams::kern { /* Find the iterator now that we've updated. */ it = this->FindIterator(address); - if (address != this->start) { + if (address != this->start_address) { it--; } diff --git a/libraries/libmesosphere/source/kern_k_memory_layout.cpp b/libraries/libmesosphere/source/kern_k_memory_layout.cpp index ef6ae623b..240ec475d 100644 --- a/libraries/libmesosphere/source/kern_k_memory_layout.cpp +++ b/libraries/libmesosphere/source/kern_k_memory_layout.cpp @@ -135,7 +135,6 @@ namespace ams::kern { namespace { - constexpr PageTableEntry KernelRwDataAttribute(PageTableEntry::Permission_KernelRW, PageTableEntry::PageAttribute_NormalMemory, PageTableEntry::Shareable_InnerShareable); constexpr size_t CarveoutAlignment = 0x20000; diff --git a/libraries/libmesosphere/source/kern_k_memory_manager.cpp b/libraries/libmesosphere/source/kern_k_memory_manager.cpp index ac51f0570..b633a6eb7 100644 --- a/libraries/libmesosphere/source/kern_k_memory_manager.cpp +++ b/libraries/libmesosphere/source/kern_k_memory_manager.cpp @@ -103,19 +103,10 @@ namespace ams::kern { /* Loop, trying to iterate from each block. */ Impl *chosen_manager = nullptr; KVirtualAddress allocated_block = Null; - if (dir == Direction_FromBack) { - for (chosen_manager = this->pool_managers_tail[pool]; chosen_manager != nullptr; chosen_manager = chosen_manager->GetPrev()) { - allocated_block = chosen_manager->AllocateBlock(heap_index); - if (allocated_block != Null) { - break; - } - } - } else { - for (chosen_manager = this->pool_managers_head[pool]; chosen_manager != nullptr; chosen_manager = chosen_manager->GetNext()) { - allocated_block = chosen_manager->AllocateBlock(heap_index); - if (allocated_block != Null) { - break; - } + for (chosen_manager = this->GetFirstManager(pool, dir); chosen_manager != nullptr; chosen_manager = this->GetNextManager(chosen_manager, dir)) { + allocated_block = chosen_manager->AllocateBlock(heap_index); + if (allocated_block != Null) { + break; } } @@ -138,6 +129,70 @@ namespace ams::kern { return allocated_block; } + Result KMemoryManager::Allocate(KPageGroup *out, size_t num_pages, u32 option) { + MESOSPHERE_ASSERT(out != nullptr); + MESOSPHERE_ASSERT(out->GetNumPages() == 0); + + /* Early return if we're allocating no pages. */ + if (num_pages == 0) { + return ResultSuccess(); + } + + /* Lock the pool that we're allocating from. */ + const auto [pool, dir] = DecodeOption(option); + KScopedLightLock lk(this->pool_locks[pool]); + + /* Choose a heap based on our page size request. */ + const s32 heap_index = KPageHeap::GetBlockIndex(num_pages); + R_UNLESS(0 <= heap_index, svc::ResultOutOfMemory()); + + /* Ensure that we don't leave anything un-freed. */ + auto group_guard = SCOPE_GUARD { + for (const auto &it : *out) { + auto &manager = this->GetManager(it.GetAddress()); + const size_t num_pages = std::min(it.GetNumPages(), (manager.GetEndAddress() - it.GetAddress()) / PageSize); + manager.Free(it.GetAddress(), num_pages); + } + out->Finalize(); + }; + + /* Keep allocating until we've allocated all our pages. */ + for (s32 index = heap_index; index >= 0 && num_pages > 0; index--) { + const size_t pages_per_alloc = KPageHeap::GetBlockNumPages(index); + for (Impl *cur_manager = this->GetFirstManager(pool, dir); cur_manager != nullptr; cur_manager = this->GetNextManager(cur_manager, dir)) { + while (num_pages >= pages_per_alloc) { + /* Allocate a block. */ + KVirtualAddress allocated_block = cur_manager->AllocateBlock(index); + if (allocated_block == Null) { + break; + } + + /* Safely add it to our group. */ + { + auto block_guard = SCOPE_GUARD { cur_manager->Free(allocated_block, pages_per_alloc); }; + R_TRY(out->AddBlock(allocated_block, pages_per_alloc)); + block_guard.Cancel(); + } + + /* Maintain the optimized memory bitmap, if we should. */ + if (this->has_optimized_process[pool]) { + cur_manager->TrackAllocationForOptimizedProcess(allocated_block, pages_per_alloc); + } + + num_pages -= pages_per_alloc; + } + } + } + + /* Only succeed if we allocated as many pages as we wanted. */ + MESOSPHERE_ASSERT(num_pages >= 0); + R_UNLESS(num_pages == 0, svc::ResultOutOfMemory()); + + /* We succeeded! */ + group_guard.Cancel(); + return ResultSuccess(); + } + size_t KMemoryManager::Impl::Initialize(const KMemoryRegion *region, Pool p, KVirtualAddress metadata, KVirtualAddress metadata_end) { /* Calculate metadata sizes. */ const size_t ref_count_size = (region->GetSize() / PageSize) * sizeof(u16); diff --git a/libraries/libmesosphere/source/kern_k_page_group.cpp b/libraries/libmesosphere/source/kern_k_page_group.cpp index 80da0388e..3e81eb593 100644 --- a/libraries/libmesosphere/source/kern_k_page_group.cpp +++ b/libraries/libmesosphere/source/kern_k_page_group.cpp @@ -17,10 +17,6 @@ namespace ams::kern { - void KPageGroup::Initialize(KBlockInfoManager *m) { - this->manager = m; - } - void KPageGroup::Finalize() { auto it = this->block_list.begin(); while (it != this->block_list.end()) { diff --git a/libraries/libmesosphere/source/kern_k_page_heap.cpp b/libraries/libmesosphere/source/kern_k_page_heap.cpp index fd8d25f44..3db3c605b 100644 --- a/libraries/libmesosphere/source/kern_k_page_heap.cpp +++ b/libraries/libmesosphere/source/kern_k_page_heap.cpp @@ -104,7 +104,7 @@ namespace ams::kern { MESOSPHERE_ASSERT(big_index >= 0); /* Free space before the big blocks. */ - for (s32 i = big_index; i >= 0; i--) { + for (s32 i = big_index - 1; i >= 0; i--) { const size_t block_size = this->blocks[i].GetSize(); while (before_start + block_size <= before_end) { before_end -= block_size; @@ -113,11 +113,11 @@ namespace ams::kern { } /* Free space after the big blocks. */ - for (s32 i = big_index; i >= 0; i--) { + for (s32 i = big_index - 1; i >= 0; i--) { const size_t block_size = this->blocks[i].GetSize(); while (after_start + block_size <= after_end) { - after_start += block_size; this->FreeBlock(after_start, i); + after_start += block_size; } } } diff --git a/libraries/libmesosphere/source/kern_k_page_table_base.cpp b/libraries/libmesosphere/source/kern_k_page_table_base.cpp index b2fc76ecb..5aa48b2fd 100644 --- a/libraries/libmesosphere/source/kern_k_page_table_base.cpp +++ b/libraries/libmesosphere/source/kern_k_page_table_base.cpp @@ -51,7 +51,7 @@ namespace ams::kern { this->stack_fill_value = MemoryFillValue_Zero; this->cached_physical_linear_region = nullptr; - this->cached_physical_non_kernel_dram_region = nullptr; + this->cached_physical_heap_region = nullptr; this->cached_virtual_managed_pool_dram_region = nullptr; /* Initialize our implementation. */ @@ -67,4 +67,279 @@ namespace ams::kern { this->memory_block_manager.Finalize(this->memory_block_slab_manager); MESOSPHERE_TODO("cpu::InvalidateEntireInstructionCache();"); } + + KProcessAddress KPageTableBase::GetRegionAddress(KMemoryState state) const { + switch (state) { + case KMemoryState_Free: + case KMemoryState_Kernel: + return this->address_space_start; + case KMemoryState_Normal: + return this->heap_region_start; + case KMemoryState_Ipc: + case KMemoryState_NonSecureIpc: + case KMemoryState_NonDeviceIpc: + return this->alias_region_start; + case KMemoryState_Stack: + return this->stack_region_start; + case KMemoryState_Io: + case KMemoryState_Static: + case KMemoryState_ThreadLocal: + return this->kernel_map_region_start; + case KMemoryState_Shared: + case KMemoryState_AliasCode: + case KMemoryState_AliasCodeData: + case KMemoryState_Transfered: + case KMemoryState_SharedTransfered: + case KMemoryState_SharedCode: + case KMemoryState_GeneratedCode: + case KMemoryState_CodeOut: + return this->alias_code_region_start; + case KMemoryState_Code: + case KMemoryState_CodeData: + return this->code_region_start; + MESOSPHERE_UNREACHABLE_DEFAULT_CASE(); + } + } + + size_t KPageTableBase::GetRegionSize(KMemoryState state) const { + switch (state) { + case KMemoryState_Free: + case KMemoryState_Kernel: + return this->address_space_end - this->address_space_start; + case KMemoryState_Normal: + return this->heap_region_end - this->heap_region_start; + case KMemoryState_Ipc: + case KMemoryState_NonSecureIpc: + case KMemoryState_NonDeviceIpc: + return this->alias_region_end - this->alias_region_start; + case KMemoryState_Stack: + return this->stack_region_end - this->stack_region_start; + case KMemoryState_Io: + case KMemoryState_Static: + case KMemoryState_ThreadLocal: + return this->kernel_map_region_end - this->kernel_map_region_start; + case KMemoryState_Shared: + case KMemoryState_AliasCode: + case KMemoryState_AliasCodeData: + case KMemoryState_Transfered: + case KMemoryState_SharedTransfered: + case KMemoryState_SharedCode: + case KMemoryState_GeneratedCode: + case KMemoryState_CodeOut: + return this->alias_code_region_end - this->alias_code_region_start; + case KMemoryState_Code: + case KMemoryState_CodeData: + return this->code_region_end - this->code_region_start; + MESOSPHERE_UNREACHABLE_DEFAULT_CASE(); + } + } + + bool KPageTableBase::Contains(KProcessAddress addr, size_t size, KMemoryState state) const { + const KProcessAddress end = addr + size; + const KProcessAddress last = end - 1; + + const KProcessAddress region_start = this->GetRegionAddress(state); + const size_t region_size = this->GetRegionSize(state); + + const bool is_in_region = region_start <= addr && addr < end && last <= region_start + region_size - 1; + const bool is_in_heap = !(end <= this->heap_region_start || this->heap_region_end <= addr); + const bool is_in_alias = !(end <= this->alias_region_start || this->alias_region_end <= addr); + switch (state) { + case KMemoryState_Free: + case KMemoryState_Kernel: + return is_in_region; + case KMemoryState_Io: + case KMemoryState_Static: + case KMemoryState_Code: + case KMemoryState_CodeData: + case KMemoryState_Shared: + case KMemoryState_AliasCode: + case KMemoryState_AliasCodeData: + case KMemoryState_Stack: + case KMemoryState_ThreadLocal: + case KMemoryState_Transfered: + case KMemoryState_SharedTransfered: + case KMemoryState_SharedCode: + case KMemoryState_GeneratedCode: + case KMemoryState_CodeOut: + return is_in_region && !is_in_heap && !is_in_alias; + case KMemoryState_Normal: + MESOSPHERE_ASSERT(is_in_heap); + return is_in_region && !is_in_alias; + case KMemoryState_Ipc: + case KMemoryState_NonSecureIpc: + case KMemoryState_NonDeviceIpc: + MESOSPHERE_ASSERT(is_in_alias); + return is_in_region && !is_in_heap; + default: + return false; + } + } + + Result KPageTableBase::CheckMemoryState(const KMemoryInfo &info, u32 state_mask, u32 state, u32 perm_mask, u32 perm, u32 attr_mask, u32 attr) const { + /* Validate the states match expectation. */ + R_UNLESS((info.state & state_mask) == state, svc::ResultInvalidCurrentMemory()); + R_UNLESS((info.perm & perm_mask) == perm, svc::ResultInvalidCurrentMemory()); + R_UNLESS((info.attribute & attr_mask) == attr, svc::ResultInvalidCurrentMemory()); + + return ResultSuccess(); + } + + Result KPageTableBase::CheckMemoryState(KMemoryState *out_state, KMemoryPermission *out_perm, KMemoryAttribute *out_attr, KProcessAddress addr, size_t size, u32 state_mask, u32 state, u32 perm_mask, u32 perm, u32 attr_mask, u32 attr, u32 ignore_attr) const { + MESOSPHERE_ASSERT(this->IsLockedByCurrentThread()); + + /* Get information about the first block. */ + const KProcessAddress last_addr = addr + size - 1; + KMemoryBlockManager::const_iterator it = this->memory_block_manager.FindIterator(addr); + KMemoryInfo info = it->GetMemoryInfo(); + + /* Validate all blocks in the range have correct state. */ + const KMemoryState first_state = info.state; + const KMemoryPermission first_perm = info.perm; + const KMemoryAttribute first_attr = info.attribute; + while (true) { + /* Validate the current block. */ + R_UNLESS(info.state == first_state, svc::ResultInvalidCurrentMemory()); + R_UNLESS(info.perm == first_perm, svc::ResultInvalidCurrentMemory()); + R_UNLESS((info.attribute | ignore_attr) == (first_attr | ignore_attr), svc::ResultInvalidCurrentMemory()); + + /* Validate against the provided masks. */ + R_TRY(this->CheckMemoryState(info, state_mask, state, perm_mask, perm, attr_mask, attr)); + + /* Break once we're done. */ + if (last_addr <= info.GetLastAddress()) { + break; + } + + /* Advance our iterator. */ + it++; + MESOSPHERE_ASSERT(it != this->memory_block_manager.cend()); + info = it->GetMemoryInfo(); + } + + /* Write output state. */ + if (out_state) { + *out_state = first_state; + } + if (out_perm) { + *out_perm = first_perm; + } + if (out_attr) { + *out_attr = static_cast(first_attr & ~ignore_attr); + } + return ResultSuccess(); + } + + Result KPageTableBase::QueryInfoImpl(KMemoryInfo *out_info, ams::svc::PageInfo *out_page, KProcessAddress address) const { + MESOSPHERE_ASSERT(this->IsLockedByCurrentThread()); + MESOSPHERE_ASSERT(out_info != nullptr); + MESOSPHERE_ASSERT(out_page != nullptr); + + const KMemoryBlock *block = this->memory_block_manager.FindBlock(address); + R_UNLESS(block != nullptr, svc::ResultInvalidCurrentMemory()); + + *out_info = block->GetMemoryInfo(); + out_page->flags = 0; + return ResultSuccess(); + } + + KProcessAddress KPageTableBase::FindFreeArea(KProcessAddress region_start, size_t region_num_pages, size_t num_pages, size_t alignment, size_t offset, size_t guard_pages) const { + KProcessAddress address = Null; + + if (num_pages <= region_num_pages) { + if (this->IsAslrEnabled()) { + /* Try to directly find a free area up to 8 times. */ + for (size_t i = 0; i < 8; i++) { + const size_t random_offset = KSystemControl::GenerateRandomRange(0, (region_num_pages - num_pages - guard_pages) * PageSize / alignment) * alignment; + const KProcessAddress candidate = util::AlignDown(GetInteger(region_start + random_offset), alignment) + offset; + + KMemoryInfo info; + ams::svc::PageInfo page_info; + MESOSPHERE_R_ABORT_UNLESS(this->QueryInfoImpl(&info, &page_info, candidate)); + + if (info.state != KMemoryState_Free) { continue; } + if (!(region_start <= candidate)) { continue; } + if (!(info.GetAddress() + guard_pages * PageSize <= GetInteger(candidate))) { continue; } + if (!(candidate + (num_pages + guard_pages) * PageSize - 1 <= info.GetLastAddress())) { continue; } + if (!(candidate + (num_pages + guard_pages) * PageSize - 1 <= region_start + region_num_pages * PageSize - 1)) { continue; } + + address = candidate; + break; + } + /* Fall back to finding the first free area with a random offset. */ + if (address == Null) { + /* NOTE: Nintendo does not account for guard pages here. */ + /* This may theoretically cause an offset to be chosen that cannot be mapped. */ + /* TODO: Should we account for guard pages? */ + const size_t offset_pages = KSystemControl::GenerateRandomRange(0, region_num_pages - num_pages); + address = this->memory_block_manager.FindFreeArea(region_start + offset_pages * PageSize, region_num_pages - offset_pages, num_pages, alignment, offset, guard_pages); + } + } + /* Find the first free area. */ + if (address == Null) { + address = this->memory_block_manager.FindFreeArea(region_start, region_num_pages, num_pages, alignment, offset, guard_pages); + } + } + + return address; + } + + Result KPageTableBase::AllocateAndMapPagesImpl(PageLinkedList *page_list, KProcessAddress address, size_t num_pages, const KPageProperties properties) { + /* Create a page group to hold the pages we allocate. */ + KPageGroup pg(this->block_info_manager); + + /* Allocate the pages. */ + R_TRY(Kernel::GetMemoryManager().Allocate(std::addressof(pg), num_pages, this->allocate_option)); + + /* Ensure that the page group is open while we work with it. */ + KScopedPageGroup spg(pg); + + /* Clear all pages. */ + for (const auto &it : pg) { + std::memset(GetVoidPointer(it.GetAddress()), this->heap_fill_value, it.GetSize()); + } + + /* Map the pages. */ + return this->Operate(page_list, address, num_pages, std::addressof(pg), properties, OperationType_MapGroup, false); + } + + Result KPageTableBase::MapPages(KProcessAddress *out_addr, size_t num_pages, size_t alignment, KPhysicalAddress phys_addr, bool is_pa_valid, KProcessAddress region_start, size_t region_num_pages, KMemoryState state, KMemoryPermission perm) { + MESOSPHERE_ASSERT(util::IsAligned(alignment, PageSize) && alignment >= PageSize); + + /* Ensure this is a valid map request. */ + R_UNLESS(this->Contains(region_start, region_num_pages * PageSize, state), svc::ResultInvalidCurrentMemory()); + R_UNLESS(num_pages < region_num_pages, svc::ResultOutOfMemory()); + + /* Lock the table. */ + KScopedLightLock lk(this->general_lock); + + /* Find a random address to map at. */ + KProcessAddress addr = this->FindFreeArea(region_start, region_num_pages, num_pages, alignment, 0, this->GetNumGuardPages()); + R_UNLESS(addr != Null, svc::ResultOutOfMemory()); + MESOSPHERE_ASSERT(util::IsAligned(GetInteger(addr), alignment)); + MESOSPHERE_ASSERT(this->Contains(addr, num_pages * PageSize, state)); + MESOSPHERE_R_ASSERT(this->CheckMemoryState(addr, num_pages * PageSize, KMemoryState_All, KMemoryState_Free, KMemoryPermission_All, KMemoryPermission_None, KMemoryAttribute_All, KMemoryAttribute_None)); + + /* Create an update allocator. */ + KMemoryBlockManagerUpdateAllocator allocator(this->memory_block_slab_manager); + R_TRY(allocator.GetResult()); + + /* We're going to perform an update, so create a helper. */ + KScopedPageTableUpdater updater(this); + + /* Perform mapping operation. */ + const KPageProperties properties = { perm, false, false, false }; + if (is_pa_valid) { + R_TRY(this->Operate(updater.GetPageList(), addr, num_pages, phys_addr, true, properties, OperationType_Map, false)); + } else { + R_TRY(this->AllocateAndMapPagesImpl(updater.GetPageList(), addr, num_pages, properties)); + } + + /* Update the blocks. */ + this->memory_block_manager.Update(&allocator, addr, num_pages, state, perm, KMemoryAttribute_None); + + /* We successfully mapped the pages. */ + *out_addr = addr; + return ResultSuccess(); + } } diff --git a/libraries/libmesosphere/source/kern_k_thread.cpp b/libraries/libmesosphere/source/kern_k_thread.cpp index fc2ec73ac..edc8e34b3 100644 --- a/libraries/libmesosphere/source/kern_k_thread.cpp +++ b/libraries/libmesosphere/source/kern_k_thread.cpp @@ -200,14 +200,18 @@ namespace ams::kern { KPageBuffer *page = KPageBuffer::Allocate(); R_UNLESS(page != nullptr, svc::ResultOutOfResource()); - /* Map the stack page. */ KProcessAddress stack_top = Null; { + KProcessAddress stack_bottom = Null; auto page_guard = SCOPE_GUARD { KPageBuffer::Free(page); }; - MESOSPHERE_TODO("R_TRY(Kernel::GetSupervisorPageTable().Map); ..."); - (void)(stack_region); + R_TRY(Kernel::GetKernelPageTable().MapPages(std::addressof(stack_bottom), 1, PageSize, page->GetPhysicalAddress(), stack_region.GetAddress(), + stack_region.GetSize() / PageSize, KMemoryState_Kernel, KMemoryPermission_KernelReadWrite)); page_guard.Cancel(); + + + /* Calculate top of the stack. */ + stack_top = stack_bottom + PageSize; } /* Initialize the thread. */ diff --git a/libraries/libmesosphere/source/libc/kern_new.cpp b/libraries/libmesosphere/source/libc/kern_env.cpp similarity index 93% rename from libraries/libmesosphere/source/libc/kern_new.cpp rename to libraries/libmesosphere/source/libc/kern_env.cpp index 8ce1e726f..c17ea9bdc 100644 --- a/libraries/libmesosphere/source/libc/kern_new.cpp +++ b/libraries/libmesosphere/source/libc/kern_env.cpp @@ -21,4 +21,8 @@ void operator delete (void *deleted) throw() { void operator delete (void *deleted, size_t size) throw() { MESOSPHERE_PANIC("operator delete(void *, size_t) was called: %p %zu", deleted, size); -} \ No newline at end of file +} + +void abort() { + MESOSPHERE_PANIC("abort() was called"); +} diff --git a/mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp b/mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp index 7ce13538c..722649bb2 100644 --- a/mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp +++ b/mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp @@ -90,7 +90,7 @@ namespace ams::kern::init { g_initial_page_allocator.Initialize(initial_page_allocator_state); /* Ensure that the T1SZ is correct (and what we expect). */ - MESOSPHERE_INIT_ABORT_UNLESS((cpu::TranslationControlRegisterAccessor().GetT1Size() / L1BlockSize) == MaxPageTableEntries); + MESOSPHERE_INIT_ABORT_UNLESS((cpu::TranslationControlRegisterAccessor().GetT1Size() / arm64::L1BlockSize) == arm64::MaxPageTableEntries); /* Create page table object for use during initialization. */ KInitialPageTable ttbr1_table(util::AlignDown(cpu::GetTtbr1El1(), PageSize), KInitialPageTable::NoClear{});