core/cpu_core_manager: Create threads separately from initialization.

Our initialization process is a little wonky than one would expect when
it comes to code flow. We initialize the CPU last, as opposed to
hardware, where the CPU obviously needs to be first, otherwise nothing
else would work, and we have code that adds checks to get around this.

For example, in the page table setting code, we check to see if the
system is turned on before we even notify the CPU instances of a page
table switch. This results in dead code (at the moment), because the
only time a page table switch will occur is when the system is *not*
running, preventing the emulated CPU instances from being notified of a
page table switch in a convenient manner (technically the code path
could be taken, but we don't emulate the process creation svc handlers
yet).

This moves the threads creation into its own member function of the core
manager and restores a little order (and predictability) to our
initialization process.

Previously, in the multi-threaded cases, we'd kick off several threads
before even the main kernel process was created and ready to execute (gross!).
Now the initialization process is like so:

Initialization:
  1. Timers

  2. CPU

  3. Kernel

  4. Filesystem stuff (kind of gross, but can be amended trivially)

  5. Applet stuff (ditto in terms of being kind of gross)

  6. Main process (will be moved into the loading step in a following
                   change)

  7. Telemetry (this should be initialized last in the future).

  8. Services (4 and 5 should ideally be alongside this).

  9. GDB (gross. Uses namespace scope state. Needs to be refactored into a
          class or booted altogether).

  10. Renderer

  11. GPU (will also have its threads created in a separate step in a
           following change).

Which... isn't *ideal* per-se, however getting rid of the wonky
intertwining of CPU state initialization out of this mix gets rid of
most of the footguns when it comes to our initialization process.

src/core/arm/arm_interface.h

@@ -7,6 +7,10 @@
 #include <array>
 #include "common/common_types.h"
 
+namespace Common {
+struct PageTable;
+}
+
 namespace Kernel {
 enum class VMAPermission : u8;
 }
@@ -49,8 +53,14 @@ public:
     /// Clear all instruction cache
     virtual void ClearInstructionCache() = 0;
 
-    /// Notify CPU emulation that page tables have changed
+    /// Notifies CPU emulation that the current page table has changed.
-    virtual void PageTableChanged() = 0;
+    ///
+    /// @param new_page_table                 The new page table.
+    /// @param new_address_space_size_in_bits The new usable size of the address space in bits.
+    ///                                       This can be either 32, 36, or 39 on official software.
+    ///
+    virtual void PageTableChanged(Common::PageTable& new_page_table,
+                                  std::size_t new_address_space_size_in_bits) = 0;
 
     /**
      * Set the Program Counter to an address

src/core/arm/dynarmic/arm_dynarmic.cpp

@@ -14,7 +14,6 @@
 #include "core/core_timing.h"
 #include "core/core_timing_util.h"
 #include "core/gdbstub/gdbstub.h"
-#include "core/hle/kernel/kernel.h"
 #include "core/hle/kernel/process.h"
 #include "core/hle/kernel/svc.h"
 #include "core/hle/kernel/vm_manager.h"
@@ -129,18 +128,16 @@ public:
     u64 tpidr_el0 = 0;
 };
 
-std::unique_ptr<Dynarmic::A64::Jit> ARM_Dynarmic::MakeJit() const {
+std::unique_ptr<Dynarmic::A64::Jit> ARM_Dynarmic::MakeJit(Common::PageTable& page_table,
-    auto* current_process = system.Kernel().CurrentProcess();
+                                                          std::size_t address_space_bits) const {
-    auto** const page_table = current_process->VMManager().page_table.pointers.data();
-
     Dynarmic::A64::UserConfig config;
 
     // Callbacks
     config.callbacks = cb.get();
 
     // Memory
-    config.page_table = reinterpret_cast<void**>(page_table);
+    config.page_table = reinterpret_cast<void**>(page_table.pointers.data());
-    config.page_table_address_space_bits = current_process->VMManager().GetAddressSpaceWidth();
+    config.page_table_address_space_bits = address_space_bits;
     config.silently_mirror_page_table = false;
 
     // Multi-process state
@@ -176,12 +173,7 @@ ARM_Dynarmic::ARM_Dynarmic(System& system, ExclusiveMonitor& exclusive_monitor,
                            std::size_t core_index)
     : cb(std::make_unique<ARM_Dynarmic_Callbacks>(*this)), inner_unicorn{system},
       core_index{core_index}, system{system},
-      exclusive_monitor{dynamic_cast<DynarmicExclusiveMonitor&>(exclusive_monitor)} {
+      exclusive_monitor{dynamic_cast<DynarmicExclusiveMonitor&>(exclusive_monitor)} {}
-    ThreadContext ctx{};
-    inner_unicorn.SaveContext(ctx);
-    PageTableChanged();
-    LoadContext(ctx);
-}
 
 ARM_Dynarmic::~ARM_Dynarmic() = default;
 
@@ -276,8 +268,9 @@ void ARM_Dynarmic::ClearExclusiveState() {
     jit->ClearExclusiveState();
 }
 
-void ARM_Dynarmic::PageTableChanged() {
+void ARM_Dynarmic::PageTableChanged(Common::PageTable& page_table,
-    jit = MakeJit();
+                                    std::size_t new_address_space_size_in_bits) {
+    jit = MakeJit(page_table, new_address_space_size_in_bits);
 }
 
 DynarmicExclusiveMonitor::DynarmicExclusiveMonitor(std::size_t core_count) : monitor(core_count) {}

src/core/arm/dynarmic/arm_dynarmic.h

@@ -48,10 +48,12 @@ public:
     void ClearExclusiveState() override;
 
     void ClearInstructionCache() override;
-    void PageTableChanged() override;
+    void PageTableChanged(Common::PageTable& new_page_table,
+                          std::size_t new_address_space_size_in_bits) override;
 
 private:
-    std::unique_ptr<Dynarmic::A64::Jit> MakeJit() const;
+    std::unique_ptr<Dynarmic::A64::Jit> MakeJit(Common::PageTable& page_table,
+                                                std::size_t address_space_bits) const;
 
     friend class ARM_Dynarmic_Callbacks;
     std::unique_ptr<ARM_Dynarmic_Callbacks> cb;

src/core/arm/unicorn/arm_unicorn.h

@@ -41,7 +41,7 @@ public:
     void Run() override;
     void Step() override;
     void ClearInstructionCache() override;
-    void PageTableChanged() override{};
+    void PageTableChanged(Common::PageTable&, std::size_t) override {}
     void RecordBreak(GDBStub::BreakpointAddress bkpt);
 
 private:

src/core/core.cpp

@@ -81,7 +81,7 @@ FileSys::VirtualFile GetGameFileFromPath(const FileSys::VirtualFilesystem& vfs,
     return vfs->OpenFile(path, FileSys::Mode::Read);
 }
 struct System::Impl {
-    explicit Impl(System& system) : kernel{system} {}
+    explicit Impl(System& system) : kernel{system}, cpu_core_manager{system} {}
 
     Cpu& CurrentCpuCore() {
         return cpu_core_manager.GetCurrentCore();
@@ -99,6 +99,7 @@ struct System::Impl {
         LOG_DEBUG(HW_Memory, "initialized OK");
 
         core_timing.Initialize();
+        cpu_core_manager.Initialize();
         kernel.Initialize();
 
         const auto current_time = std::chrono::duration_cast<std::chrono::seconds>(
@@ -142,8 +143,6 @@ struct System::Impl {
             gpu_core = std::make_unique<VideoCommon::GPUSynch>(system, *renderer);
         }
 
-        cpu_core_manager.Initialize(system);
-
         LOG_DEBUG(Core, "Initialized OK");
 
         // Reset counters and set time origin to current frame
@@ -188,6 +187,10 @@ struct System::Impl {
                                              static_cast<u32>(load_result));
         }
 
+        // Main process has been loaded and been made current.
+        // Begin CPU execution.
+        cpu_core_manager.StartThreads();
+
         status = ResultStatus::Success;
         return status;
     }

src/core/cpu_core_manager.cpp

@@ -19,17 +19,19 @@ void RunCpuCore(const System& system, Cpu& cpu_state) {
 }
 } // Anonymous namespace
 
-CpuCoreManager::CpuCoreManager() = default;
+CpuCoreManager::CpuCoreManager(System& system) : system{system} {}
 CpuCoreManager::~CpuCoreManager() = default;
 
-void CpuCoreManager::Initialize(System& system) {
+void CpuCoreManager::Initialize() {
     barrier = std::make_unique<CpuBarrier>();
     exclusive_monitor = Cpu::MakeExclusiveMonitor(cores.size());
 
     for (std::size_t index = 0; index < cores.size(); ++index) {
         cores[index] = std::make_unique<Cpu>(system, *exclusive_monitor, *barrier, index);
     }
+}
 
+void CpuCoreManager::StartThreads() {
     // Create threads for CPU cores 1-3, and build thread_to_cpu map
     // CPU core 0 is run on the main thread
     thread_to_cpu[std::this_thread::get_id()] = cores[0].get();

src/core/cpu_core_manager.h

@@ -18,7 +18,7 @@ class System;
 
 class CpuCoreManager {
 public:
-    CpuCoreManager();
+    explicit CpuCoreManager(System& system);
     CpuCoreManager(const CpuCoreManager&) = delete;
     CpuCoreManager(CpuCoreManager&&) = delete;
 
@@ -27,7 +27,8 @@ public:
     CpuCoreManager& operator=(const CpuCoreManager&) = delete;
     CpuCoreManager& operator=(CpuCoreManager&&) = delete;
 
-    void Initialize(System& system);
+    void Initialize();
+    void StartThreads();
     void Shutdown();
 
     Cpu& GetCore(std::size_t index);
@@ -54,6 +55,8 @@ private:
 
     /// Map of guest threads to CPU cores
     std::map<std::thread::id, Cpu*> thread_to_cpu;
+
+    System& system;
 };
 
 } // namespace Core

src/core/hle/kernel/kernel.cpp

@@ -182,7 +182,12 @@ void KernelCore::AppendNewProcess(SharedPtr<Process> process) {
 
 void KernelCore::MakeCurrentProcess(Process* process) {
     impl->current_process = process;
-    Memory::SetCurrentPageTable(&process->VMManager().page_table);
+
+    if (process == nullptr) {
+        return;
+    }
+
+    Memory::SetCurrentPageTable(*process);
 }
 
 Process* KernelCore::CurrentProcess() {

src/core/hle/kernel/process.cpp

@@ -107,7 +107,7 @@ ResultCode Process::LoadFromMetadata(const FileSys::ProgramMetadata& metadata) {
 
     vm_manager.Reset(metadata.GetAddressSpaceType());
     // Ensure that the potentially resized page table is seen by CPU backends.
-    Memory::SetCurrentPageTable(&vm_manager.page_table);
+    Memory::SetCurrentPageTable(*this);
 
     const auto& caps = metadata.GetKernelCapabilities();
     const auto capability_init_result =

src/core/memory.cpp

@@ -26,16 +26,16 @@ namespace Memory {
 
 static Common::PageTable* current_page_table = nullptr;
 
-void SetCurrentPageTable(Common::PageTable* page_table) {
+void SetCurrentPageTable(Kernel::Process& process) {
-    current_page_table = page_table;
+    current_page_table = &process.VMManager().page_table;
+
+    const std::size_t address_space_width = process.VMManager().GetAddressSpaceWidth();
 
     auto& system = Core::System::GetInstance();
-    if (system.IsPoweredOn()) {
+    system.ArmInterface(0).PageTableChanged(*current_page_table, address_space_width);
-        system.ArmInterface(0).PageTableChanged();
+    system.ArmInterface(1).PageTableChanged(*current_page_table, address_space_width);
-        system.ArmInterface(1).PageTableChanged();
+    system.ArmInterface(2).PageTableChanged(*current_page_table, address_space_width);
-        system.ArmInterface(2).PageTableChanged();
+    system.ArmInterface(3).PageTableChanged(*current_page_table, address_space_width);
-        system.ArmInterface(3).PageTableChanged();
-    }
 }
 
 static void MapPages(Common::PageTable& page_table, VAddr base, u64 size, u8* memory,

src/core/memory.h

@@ -40,8 +40,9 @@ enum : VAddr {
     KERNEL_REGION_END = KERNEL_REGION_VADDR + KERNEL_REGION_SIZE,
 };
 
-/// Changes the currently active page table.
+/// Changes the currently active page table to that of
-void SetCurrentPageTable(Common::PageTable* page_table);
+/// the given process instance.
+void SetCurrentPageTable(Kernel::Process& process);
 
 /// Determines if the given VAddr is valid for the specified process.
 bool IsValidVirtualAddress(const Kernel::Process& process, VAddr vaddr);