kern: update initial cache management to match latest kernel

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu.hpp

@@ -172,10 +172,8 @@ namespace ams::kern::arch::arm64::cpu {
 
     /* Cache management helpers. */
     void ClearPageToZeroImpl(void *);
-    void FlushEntireDataCacheSharedForInit();
-    void FlushEntireDataCacheLocalForInit();
-    void InvalidateEntireInstructionCacheForInit();
     void StoreEntireCacheForInit();
+    void FlushEntireCacheForInit();
 
     void FlushEntireDataCache();
 

libraries/libmesosphere/source/arch/arm64/kern_cpu.cpp

@@ -262,27 +262,6 @@ namespace ams::kern::arch::arm64::cpu {
             __asm__ __volatile__("dc csw, %[v]" :: [v]"r"(sw_value) : "memory");
         }
 
-        template<bool Init, typename F>
-        ALWAYS_INLINE void PerformCacheOperationBySetWayShared(F f) {
-            CacheLineIdRegisterAccessor clidr_el1;
-            const int levels_of_coherency   = clidr_el1.GetLevelsOfCoherency();
-            const int levels_of_unification = clidr_el1.GetLevelsOfUnification();
-
-            for (int level = levels_of_coherency; level >= levels_of_unification; level--) {
-                PerformCacheOperationBySetWayImpl<Init>(level, f);
-            }
-        }
-
-        template<bool Init, typename F>
-        ALWAYS_INLINE void PerformCacheOperationBySetWayLocal(F f) {
-            CacheLineIdRegisterAccessor clidr_el1;
-            const int levels_of_unification = clidr_el1.GetLevelsOfUnification();
-
-            for (int level = levels_of_unification - 1; level >= 0; level--) {
-                PerformCacheOperationBySetWayImpl<Init>(level, f);
-            }
-        }
-
         void StoreDataCacheBySetWay(int level) {
             PerformCacheOperationBySetWayImpl<false>(level, StoreDataCacheLineBySetWayImpl);
             cpu::DataSynchronizationBarrier();
@@ -361,24 +340,63 @@ namespace ams::kern::arch::arm64::cpu {
 
     }
 
-    void FlushEntireDataCacheSharedForInit() {
+    void StoreEntireCacheForInit() {
-        return PerformCacheOperationBySetWayShared<true>(FlushDataCacheLineBySetWayImpl);
+        /* Store local. */
+        {
+            CacheLineIdRegisterAccessor clidr_el1;
+            const int levels_of_unification = clidr_el1.GetLevelsOfUnification();
+
+            for (int level = 0; level != levels_of_unification; ++level) {
+                PerformCacheOperationBySetWayImpl<true>(level, StoreDataCacheLineBySetWayImpl);
+            }
+        }
+
+        /* Store shared. */
+        {
+            CacheLineIdRegisterAccessor clidr_el1;
+            const int levels_of_coherency   = clidr_el1.GetLevelsOfCoherency();
+            const int levels_of_unification = clidr_el1.GetLevelsOfUnification();
+
+            for (int level = levels_of_unification; level <= levels_of_coherency; ++level) {
+                PerformCacheOperationBySetWayImpl<true>(level, StoreDataCacheLineBySetWayImpl);
+            }
+        }
+
+        /* Data synchronization barrier. */
+        DataSynchronizationBarrierInnerShareable();
+
+        /* Invalidate instruction cache. */
+        InvalidateEntireInstructionCacheLocalImpl();
+
+        /* Ensure local instruction consistency. */
+        DataSynchronizationBarrierInnerShareable();
+        InstructionMemoryBarrier();
     }
 
-    void FlushEntireDataCacheLocalForInit() {
+    void FlushEntireCacheForInit() {
-        return PerformCacheOperationBySetWayLocal<true>(FlushDataCacheLineBySetWayImpl);
+        /* Flush data cache. */
-    }
+        {
+            /* Get levels of coherence/unificaiton. */
+            CacheLineIdRegisterAccessor clidr_el1;
+            const int levels_of_coherency   = clidr_el1.GetLevelsOfCoherency();
 
-    void InvalidateEntireInstructionCacheForInit() {
+            /* Store cache from L1 up to (level of coherence - 1). */
+            for (int level = 0; level < levels_of_coherency - 1; ++level) {
+                PerformCacheOperationBySetWayImpl<true>(level, StoreDataCacheLineBySetWayImpl);
+            }
+
+            /* Flush cache from (level of coherence - 1) down to L0. */
+            for (int level = levels_of_coherency; level > 0; --level) {
+                PerformCacheOperationBySetWayImpl<true>(level - 1, FlushDataCacheLineBySetWayImpl);
+            }
+        }
+
+        /* Invalidate instruction cache. */
         InvalidateEntireInstructionCacheLocalImpl();
         EnsureInstructionConsistency();
-    }
 
-    void StoreEntireCacheForInit() {
+        /* Invalidate entire TLB. */
-        PerformCacheOperationBySetWayLocal<true>(StoreDataCacheLineBySetWayImpl);
+        InvalidateEntireTlb();
-        PerformCacheOperationBySetWayShared<true>(StoreDataCacheLineBySetWayImpl);
-        DataSynchronizationBarrierInnerShareable();
-        InvalidateEntireInstructionCacheForInit();
     }
 
     void FlushEntireDataCache() {

mesosphere/kernel/source/arch/arm64/init/start.s

@@ -383,20 +383,20 @@ _ZN3ams4kern4arch5arm643cpu37FlushEntireDataCacheLocalWithoutStackEv:
     /* const int levels_of_unification = clidr_el1.GetLevelsOfUnification(); */
     ubfx x10, x10, #0x15, 3
 
-    /* int level = levels_of_unification - 1 */
+    /* int level = 0 */
-    sub w9, w10, #1
+    mov x9, xzr
 
-    /* while (level >= 0) { */
+    /* while (level <= levels_of_unification) { */
 begin_flush_cache_local_loop:
-    cmn w9, #1
+    cmp x9, x10
     b.eq done_flush_cache_local_loop
 
     /*     FlushEntireDataCacheImplWithoutStack(level); */
     mov w0, w9
     bl _ZN3ams4kern4arch5arm643cpu36FlushEntireDataCacheImplWithoutStackEv
 
-    /*     level--; */
+    /*     level++; */
-    sub w9, w9, #1
+    add w9, w9, #1
 
     /* } */
     b begin_flush_cache_local_loop
@@ -416,23 +416,23 @@ _ZN3ams4kern4arch5arm643cpu38FlushEntireDataCacheSharedWithoutStackEv:
     /* CacheLineIdAccessor clidr_el1; */
     mrs x10, clidr_el1
     /* const int levels_of_coherency   = clidr_el1.GetLevelsOfCoherency(); */
-    ubfx x9, x10,  #0x18, 3
+    ubfx x9, x10,  #0x15, 3
     /* const int levels_of_unification = clidr_el1.GetLevelsOfUnification(); */
-    ubfx x10, x10, #0x15, 3
+    ubfx x10, x10, #0x18, 3
 
-    /* int level = levels_of_coherency */
+    /* int level = levels_of_unification */
 
-    /* while (level >= levels_of_unification) { */
+    /* while (level <= levels_of_coherency) { */
 begin_flush_cache_shared_loop:
-    cmp w10, w9
+    cmp w9, w10
-    b.gt done_flush_cache_shared_loop
+    b.hi done_flush_cache_shared_loop
 
     /*     FlushEntireDataCacheImplWithoutStack(level); */
     mov w0, w9
     bl _ZN3ams4kern4arch5arm643cpu36FlushEntireDataCacheImplWithoutStackEv
 
-    /*     level--; */
+    /*     level++; */
-    sub w9, w9, #1
+    add w9, w9, #1
 
     /* } */
     b begin_flush_cache_shared_loop

mesosphere/kernel_ldr/source/kern_init_loader.cpp

@@ -59,26 +59,6 @@ namespace ams::kern::init::loader {
             }
         }
 
-        void EnsureEntireDataCacheFlushed() {
-            /* Flush shared cache. */
-            cpu::FlushEntireDataCacheSharedForInit();
-            cpu::DataSynchronizationBarrier();
-
-            /* Flush local cache. */
-            cpu::FlushEntireDataCacheLocalForInit();
-            cpu::DataSynchronizationBarrier();
-
-            /* Flush shared cache. */
-            cpu::FlushEntireDataCacheSharedForInit();
-            cpu::DataSynchronizationBarrier();
-
-            /* Invalidate entire instruction cache. */
-            cpu::InvalidateEntireInstructionCacheForInit();
-
-            /* Invalidate entire TLB. */
-            cpu::InvalidateEntireTlb();
-        }
-
         void SetupInitialIdentityMapping(KInitialPageTable &init_pt, uintptr_t base_address, uintptr_t kernel_size, uintptr_t page_table_region, size_t page_table_region_size, KInitialPageTable::IPageAllocator &allocator) {
             /* Map in an RWX identity mapping for the kernel. */
             constexpr PageTableEntry KernelRWXIdentityAttribute(PageTableEntry::Permission_KernelRWX, PageTableEntry::PageAttribute_NormalMemory, PageTableEntry::Shareable_InnerShareable, PageTableEntry::MappingFlag_Mapped);
@@ -109,7 +89,7 @@ namespace ams::kern::init::loader {
             PerformBoardSpecificSetup();
 
             /* Ensure that the entire cache is flushed. */
-            EnsureEntireDataCacheFlushed();
+            cpu::FlushEntireCacheForInit();
 
             /* Setup SCTLR_EL1. */
             /* TODO: Define these bits properly elsewhere, document exactly what each bit set is doing .*/