mirror of
https://github.com/Atmosphere-NX/Atmosphere.git
synced 2024-11-24 07:40:26 +01:00
kern: devirtualize most things that are free to devirtualize (see #1672)
This commit is contained in:
parent
aaa3770806
commit
d0cd511c0e
@ -34,19 +34,22 @@ namespace ams::kern::arch::arm64::init {
|
||||
}
|
||||
}
|
||||
|
||||
class KInitialPageTable {
|
||||
/* NOTE: Nintendo uses virtual functions, rather than a concept + template. */
|
||||
template<typename T>
|
||||
concept IsInitialPageAllocator = requires (T &t, KPhysicalAddress phys_addr, size_t size) {
|
||||
{ t.Allocate(size) } -> std::same_as<KPhysicalAddress>;
|
||||
{ t.Free(phys_addr, size) } -> std::same_as<void>;
|
||||
};
|
||||
|
||||
template<IsInitialPageAllocator _PageAllocator>
|
||||
class KInitialPageTableTemplate {
|
||||
public:
|
||||
class IPageAllocator {
|
||||
public:
|
||||
virtual KPhysicalAddress Allocate(size_t size) = 0;
|
||||
virtual void Free(KPhysicalAddress phys_addr, size_t size) = 0;
|
||||
};
|
||||
using PageAllocator = _PageAllocator;
|
||||
private:
|
||||
KPhysicalAddress m_l1_tables[2];
|
||||
u32 m_num_entries[2];
|
||||
|
||||
public:
|
||||
KInitialPageTable(KVirtualAddress start_address, KVirtualAddress end_address, IPageAllocator &allocator) {
|
||||
KInitialPageTableTemplate(KVirtualAddress start_address, KVirtualAddress end_address, PageAllocator &allocator) {
|
||||
/* Set tables. */
|
||||
m_l1_tables[0] = AllocateNewPageTable(allocator);
|
||||
m_l1_tables[1] = AllocateNewPageTable(allocator);
|
||||
@ -56,7 +59,7 @@ namespace ams::kern::arch::arm64::init {
|
||||
m_num_entries[1] = ((end_address / L1BlockSize) & (MaxPageTableEntries - 1)) - ((start_address / L1BlockSize) & (MaxPageTableEntries - 1)) + 1;
|
||||
}
|
||||
|
||||
KInitialPageTable() {
|
||||
KInitialPageTableTemplate() {
|
||||
/* Set tables. */
|
||||
m_l1_tables[0] = util::AlignDown(cpu::GetTtbr0El1(), PageSize);
|
||||
m_l1_tables[1] = util::AlignDown(cpu::GetTtbr1El1(), PageSize);
|
||||
@ -95,7 +98,7 @@ namespace ams::kern::arch::arm64::init {
|
||||
return l3_table + ((GetInteger(address) / L3BlockSize) & (MaxPageTableEntries - 1));
|
||||
}
|
||||
|
||||
static ALWAYS_INLINE KPhysicalAddress AllocateNewPageTable(IPageAllocator &allocator) {
|
||||
static ALWAYS_INLINE KPhysicalAddress AllocateNewPageTable(PageAllocator &allocator) {
|
||||
auto address = allocator.Allocate(PageSize);
|
||||
ClearNewPageTable(address);
|
||||
return address;
|
||||
@ -323,7 +326,7 @@ namespace ams::kern::arch::arm64::init {
|
||||
}
|
||||
}
|
||||
public:
|
||||
void NOINLINE Map(KVirtualAddress virt_addr, size_t size, KPhysicalAddress phys_addr, const PageTableEntry &attr, IPageAllocator &allocator) {
|
||||
void NOINLINE Map(KVirtualAddress virt_addr, size_t size, KPhysicalAddress phys_addr, const PageTableEntry &attr, PageAllocator &allocator) {
|
||||
/* Ensure that addresses and sizes are page aligned. */
|
||||
MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), PageSize));
|
||||
MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(phys_addr), PageSize));
|
||||
@ -700,10 +703,9 @@ namespace ams::kern::arch::arm64::init {
|
||||
this->PhysicallyRandomize(virt_addr, size, L3BlockSize, do_copy);
|
||||
cpu::StoreEntireCacheForInit();
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
class KInitialPageAllocator final : public KInitialPageTable::IPageAllocator {
|
||||
class KInitialPageAllocator final {
|
||||
private:
|
||||
static constexpr inline size_t FreeUnitSize = BITSIZEOF(u64) * PageSize;
|
||||
struct FreeListEntry {
|
||||
@ -807,11 +809,11 @@ namespace ams::kern::arch::arm64::init {
|
||||
}
|
||||
}
|
||||
|
||||
virtual KPhysicalAddress Allocate(size_t size) override {
|
||||
KPhysicalAddress Allocate(size_t size) {
|
||||
return this->Allocate(size, size);
|
||||
}
|
||||
|
||||
virtual void Free(KPhysicalAddress phys_addr, size_t size) override {
|
||||
void Free(KPhysicalAddress phys_addr, size_t size) {
|
||||
auto **prev_next = std::addressof(m_state.free_head);
|
||||
auto *new_chunk = reinterpret_cast<FreeListEntry *>(GetInteger(phys_addr));
|
||||
if (auto *cur = m_state.free_head; cur != nullptr) {
|
||||
@ -863,5 +865,8 @@ namespace ams::kern::arch::arm64::init {
|
||||
*prev_next = new_chunk;
|
||||
}
|
||||
};
|
||||
static_assert(IsInitialPageAllocator<KInitialPageAllocator>);
|
||||
|
||||
using KInitialPageTable = KInitialPageTableTemplate<KInitialPageAllocator>;
|
||||
|
||||
}
|
||||
|
@ -34,8 +34,9 @@ namespace ams::kern::arch::arm64 {
|
||||
|
||||
static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
|
||||
public:
|
||||
virtual Result GetThreadContextImpl(ams::svc::ThreadContext *out, KThread *thread, u32 context_flags) override;
|
||||
virtual Result SetThreadContextImpl(const ams::svc::ThreadContext &ctx, KThread *thread, u32 context_flags) override;
|
||||
/* NOTE: These are virtual in Nintendo's kernel. */
|
||||
Result GetThreadContextImpl(ams::svc::ThreadContext *out, KThread *thread, u32 context_flags);
|
||||
Result SetThreadContextImpl(const ams::svc::ThreadContext &ctx, KThread *thread, u32 context_flags);
|
||||
private:
|
||||
Result GetFpuContext(ams::svc::ThreadContext *out, KThread *thread, u32 context_flags);
|
||||
Result SetFpuContext(const ams::svc::ThreadContext &ctx, KThread *thread, u32 context_flags);
|
||||
|
@ -38,11 +38,11 @@ namespace ams::kern {
|
||||
static constexpr inline ClassTokenType ClassToken() { return ::ams::kern::ClassToken<CLASS>; } \
|
||||
public: \
|
||||
using BaseClass = BASE_CLASS; \
|
||||
static constexpr ALWAYS_INLINE TypeObj GetStaticTypeObj() { \
|
||||
static consteval ALWAYS_INLINE TypeObj GetStaticTypeObj() { \
|
||||
constexpr ClassTokenType Token = ClassToken(); \
|
||||
return TypeObj(TypeName, Token); \
|
||||
} \
|
||||
static constexpr ALWAYS_INLINE const char *GetStaticTypeName() { return TypeName; } \
|
||||
static consteval ALWAYS_INLINE const char *GetStaticTypeName() { return TypeName; } \
|
||||
virtual TypeObj GetTypeObj() const { return GetStaticTypeObj(); } \
|
||||
virtual const char *GetTypeName() { return GetStaticTypeName(); } \
|
||||
private:
|
||||
@ -143,7 +143,8 @@ namespace ams::kern {
|
||||
/* is already using CRTP for slab heap, we have devirtualized it for performance gain. */
|
||||
/* virtual void Finalize() { MESOSPHERE_ASSERT_THIS(); } */
|
||||
|
||||
virtual KProcess *GetOwner() const { return nullptr; }
|
||||
/* NOTE: This is a virtual function which is unused-except-for-debug in Nintendo's kernel. */
|
||||
/* virtual KProcess *GetOwner() const { return nullptr; } */
|
||||
|
||||
u32 GetReferenceCount() const {
|
||||
return m_ref_count.GetValue();
|
||||
|
@ -60,9 +60,38 @@ namespace ams::kern {
|
||||
void Initialize() { MESOSPHERE_ASSERT_THIS(); }
|
||||
void Finalize() { MESOSPHERE_ASSERT_THIS(); }
|
||||
|
||||
void Register(KAutoObjectWithList *obj);
|
||||
void Unregister(KAutoObjectWithList *obj);
|
||||
size_t GetOwnedCount(KProcess *owner);
|
||||
void Register(KAutoObjectWithList *obj) {
|
||||
MESOSPHERE_ASSERT_THIS();
|
||||
|
||||
KScopedLightLock lk(m_lock);
|
||||
|
||||
m_object_list.insert(*obj);
|
||||
}
|
||||
|
||||
void Unregister(KAutoObjectWithList *obj) {
|
||||
MESOSPHERE_ASSERT_THIS();
|
||||
|
||||
KScopedLightLock lk(m_lock);
|
||||
|
||||
m_object_list.erase(m_object_list.iterator_to(*obj));
|
||||
}
|
||||
|
||||
template<typename T> requires (std::derived_from<T, KAutoObjectWithList> && requires (const T &t) { { t.GetOwner() } -> std::convertible_to<const KProcess *>; })
|
||||
size_t GetOwnedCount(const KProcess *owner) {
|
||||
MESOSPHERE_ASSERT_THIS();
|
||||
|
||||
KScopedLightLock lk(m_lock);
|
||||
|
||||
size_t count = 0;
|
||||
|
||||
for (const auto &obj : m_object_list) {
|
||||
if (const T * const derived = obj.DynamicCast<T *>(); derived != nullptr && derived->GetOwner() == owner) {
|
||||
++count;
|
||||
}
|
||||
}
|
||||
|
||||
return count;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
|
@ -38,6 +38,7 @@ namespace ams::kern {
|
||||
bool Is64Bit() const;
|
||||
public:
|
||||
void Initialize();
|
||||
void Finalize();
|
||||
|
||||
Result Attach(KProcess *process);
|
||||
Result BreakProcess();
|
||||
@ -52,9 +53,6 @@ namespace ams::kern {
|
||||
Result GetThreadContext(ams::svc::ThreadContext *out, u64 thread_id, u32 context_flags);
|
||||
Result SetThreadContext(const ams::svc::ThreadContext &ctx, u64 thread_id, u32 context_flags);
|
||||
|
||||
virtual Result GetThreadContextImpl(ams::svc::ThreadContext *out, KThread *thread, u32 context_flags) = 0;
|
||||
virtual Result SetThreadContextImpl(const ams::svc::ThreadContext &ctx, KThread *thread, u32 context_flags) = 0;
|
||||
|
||||
Result GetRunningThreadInfo(ams::svc::LastThreadContext *out_context, u64 *out_thread_id);
|
||||
|
||||
Result GetDebugEventInfo(ams::svc::lp64::DebugEventInfo *out);
|
||||
@ -82,8 +80,10 @@ namespace ams::kern {
|
||||
template<typename T> requires (std::same_as<T, ams::svc::lp64::DebugEventInfo> || std::same_as<T, ams::svc::ilp32::DebugEventInfo>)
|
||||
Result GetDebugEventInfoImpl(T *out);
|
||||
public:
|
||||
virtual void OnFinalizeSynchronizationObject() override;
|
||||
virtual bool IsSignaled() const override;
|
||||
private:
|
||||
/* NOTE: This is public/virtual override in Nintendo's kernel. */
|
||||
void OnFinalizeSynchronizationObject();
|
||||
private:
|
||||
static Result ProcessDebugEvent(ams::svc::DebugEvent event, uintptr_t param0, uintptr_t param1, uintptr_t param2, uintptr_t param3, uintptr_t param4);
|
||||
public:
|
||||
|
@ -46,7 +46,7 @@ namespace ams::kern {
|
||||
|
||||
static void PostDestroy(uintptr_t arg);
|
||||
|
||||
virtual KProcess *GetOwner() const override { return m_owner; }
|
||||
KProcess *GetOwner() const { return m_owner; }
|
||||
|
||||
KReadableEvent &GetReadableEvent() { return m_readable_event; }
|
||||
|
||||
|
@ -35,7 +35,8 @@ namespace ams::kern {
|
||||
constexpr ALWAYS_INLINE explicit KSynchronizationObject(util::ConstantInitializeTag) : KAutoObjectWithList(util::ConstantInitialize), m_thread_list_head(), m_thread_list_tail() { MESOSPHERE_ASSERT_THIS(); }
|
||||
ALWAYS_INLINE explicit KSynchronizationObject() : m_thread_list_head(), m_thread_list_tail() { MESOSPHERE_ASSERT_THIS(); }
|
||||
|
||||
virtual void OnFinalizeSynchronizationObject() { MESOSPHERE_ASSERT_THIS(); }
|
||||
/* NOTE: This is a virtual function which is overridden only by KDebugBase in Nintendo's kernel. */
|
||||
/* virtual void OnFinalizeSynchronizationObject() { MESOSPHERE_ASSERT_THIS(); } */
|
||||
|
||||
void NotifyAvailable(Result result);
|
||||
ALWAYS_INLINE void NotifyAvailable() {
|
||||
|
@ -620,7 +620,7 @@ namespace ams::kern {
|
||||
void Finalize();
|
||||
|
||||
virtual bool IsSignaled() const override;
|
||||
virtual void OnTimer() override;
|
||||
void OnTimer();
|
||||
virtual void DoWorkerTask() override;
|
||||
public:
|
||||
static constexpr bool IsConditionVariableThreadTreeValid() {
|
||||
@ -674,4 +674,8 @@ namespace ams::kern {
|
||||
GetCurrentThread().SetClosedObject(this);
|
||||
}
|
||||
|
||||
ALWAYS_INLINE void KTimerTask::OnTimer() {
|
||||
static_cast<KThread *>(this)->OnTimer();
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -41,8 +41,9 @@ namespace ams::kern {
|
||||
return m_time;
|
||||
}
|
||||
|
||||
virtual void OnTimer() = 0;
|
||||
|
||||
/* NOTE: This is virtual in Nintendo's kernel. Prior to 13.0.0, KWaitObject was also a TimerTask; this is no longer the case. */
|
||||
/* Since this is now KThread exclusive, we have devirtualized (see inline declaration for this inside kern_kthread.hpp). */
|
||||
void OnTimer();
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -1,53 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) Atmosphère-NX
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify it
|
||||
* under the terms and conditions of the GNU General Public License,
|
||||
* version 2, as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
#include <mesosphere.hpp>
|
||||
|
||||
namespace ams::kern {
|
||||
|
||||
|
||||
void KAutoObjectWithListContainer::Register(KAutoObjectWithList *obj) {
|
||||
MESOSPHERE_ASSERT_THIS();
|
||||
|
||||
KScopedLightLock lk(m_lock);
|
||||
|
||||
m_object_list.insert(*obj);
|
||||
}
|
||||
|
||||
void KAutoObjectWithListContainer::Unregister(KAutoObjectWithList *obj) {
|
||||
MESOSPHERE_ASSERT_THIS();
|
||||
|
||||
KScopedLightLock lk(m_lock);
|
||||
|
||||
m_object_list.erase(m_object_list.iterator_to(*obj));
|
||||
}
|
||||
|
||||
size_t KAutoObjectWithListContainer::GetOwnedCount(KProcess *owner) {
|
||||
MESOSPHERE_ASSERT_THIS();
|
||||
|
||||
KScopedLightLock lk(m_lock);
|
||||
|
||||
size_t count = 0;
|
||||
|
||||
for (auto &obj : m_object_list) {
|
||||
if (obj.GetOwner() == owner) {
|
||||
count++;
|
||||
}
|
||||
}
|
||||
|
||||
return count;
|
||||
}
|
||||
|
||||
}
|
@ -528,7 +528,8 @@ namespace ams::kern {
|
||||
}
|
||||
|
||||
/* Get the thread context. */
|
||||
return this->GetThreadContextImpl(out, thread, context_flags);
|
||||
static_assert(std::derived_from<KDebug, KDebugBase>);
|
||||
return static_cast<KDebug *>(this)->GetThreadContextImpl(out, thread, context_flags);
|
||||
}
|
||||
}
|
||||
|
||||
@ -619,7 +620,8 @@ namespace ams::kern {
|
||||
}
|
||||
|
||||
/* Set the thread context. */
|
||||
return this->SetThreadContextImpl(ctx, thread, context_flags);
|
||||
static_assert(std::derived_from<KDebug, KDebugBase>);
|
||||
return static_cast<KDebug *>(this)->SetThreadContextImpl(ctx, thread, context_flags);
|
||||
}
|
||||
}
|
||||
|
||||
@ -984,6 +986,14 @@ namespace ams::kern {
|
||||
return this->GetDebugEventInfoImpl(out);
|
||||
}
|
||||
|
||||
void KDebugBase::Finalize() {
|
||||
/* Perform base finalization. */
|
||||
KSynchronizationObject::Finalize();
|
||||
|
||||
/* Perform post-synchronization finalization. */
|
||||
this->OnFinalizeSynchronizationObject();
|
||||
}
|
||||
|
||||
void KDebugBase::OnFinalizeSynchronizationObject() {
|
||||
/* Detach from our process, if we have one. */
|
||||
if (this->IsAttached() && this->OpenProcess()) {
|
||||
|
@ -83,7 +83,8 @@ namespace ams::kern {
|
||||
}
|
||||
#endif
|
||||
|
||||
this->OnFinalizeSynchronizationObject();
|
||||
/* NOTE: In Nintendo's kernel, the following is virtual and called here. */
|
||||
/* this->OnFinalizeSynchronizationObject(); */
|
||||
}
|
||||
|
||||
Result KSynchronizationObject::Wait(s32 *out_index, KSynchronizationObject **objects, const s32 num_objects, s64 timeout) {
|
||||
|
@ -62,7 +62,7 @@ namespace ams::kern::init::loader {
|
||||
}
|
||||
}
|
||||
|
||||
void SetupInitialIdentityMapping(KInitialPageTable &init_pt, uintptr_t base_address, uintptr_t kernel_size, uintptr_t page_table_region, size_t page_table_region_size, KInitialPageTable::IPageAllocator &allocator) {
|
||||
void SetupInitialIdentityMapping(KInitialPageTable &init_pt, uintptr_t base_address, uintptr_t kernel_size, uintptr_t page_table_region, size_t page_table_region_size, KInitialPageTable::PageAllocator &allocator) {
|
||||
/* Map in an RWX identity mapping for the kernel. */
|
||||
constexpr PageTableEntry KernelRWXIdentityAttribute(PageTableEntry::Permission_KernelRWX, PageTableEntry::PageAttribute_NormalMemory, PageTableEntry::Shareable_InnerShareable, PageTableEntry::MappingFlag_Mapped);
|
||||
init_pt.Map(base_address, kernel_size, base_address, KernelRWXIdentityAttribute, allocator);
|
||||
|
Loading…
Reference in New Issue
Block a user