GFPAK disk hash cache.

File_Format_Library/FileFormats/Archives/GFPAK.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
@@ -9,6 +9,7 @@ using Toolbox.Library;
 using System.IO;
 using Toolbox.Library.IO;
 using Toolbox.Library.Animations;
+using FirstPlugin.FileFormats.Hashes;
 
 namespace FirstPlugin
 {
@@ -540,6 +541,8 @@ namespace FirstPlugin
             if (Signature != "GFLXPACK")
                 throw new Exception($"Invalid signature {Signature}! Expected GFLXPACK.");
 
+            GFPAKHashCache.EnsureHashCache();
+
             version = reader.ReadInt32();
             uint padding = reader.ReadUInt32();
             uint FileCount = reader.ReadUInt32();
@@ -570,6 +573,8 @@ namespace FirstPlugin
                 hashes.Add(hash);
             }
 
+            GeneratePokeStrings();
+
             reader.Seek((long)FileInfoOffset, SeekOrigin.Begin);
             for (int i = 0; i < FileCount; i++)
             {
@@ -600,84 +605,26 @@ namespace FirstPlugin
                 fileEntry.FileName = GetString(hashes[i], fileEntry.FolderHash, fileEntry.FileData);
                 fileEntry.FilePathHash = hashes[i];
             }
-        }
 
-        private Dictionary<ulong, string> hashList;
+            GFPAKHashCache.WriteCache();
-        public Dictionary<ulong, string> HashList
-        {
-            get
-            {
-                if (hashList == null) {
-                    hashList = new Dictionary<ulong, string>();
-                    GenerateHashList();
-                }
-                return hashList;
-            }
-        }
-
-        private void GenerateHashList()
-        {
-            foreach (string hashStr in Properties.Resources.Pkmn.Split('\n'))
-            {
-                string HashString = hashStr.TrimEnd();
-
-                ulong hash = FNV64A1.Calculate(HashString);
-                if (!hashList.ContainsKey(hash))
-                    hashList.Add(hash, HashString);
-
-                if (HashString.Contains("pm0000") || 
-                    HashString.Contains("poke_XXXX") || 
-                    HashString.Contains("poke_ball_0000") ||
-                    HashString.Contains("poke_face_0000") ||
-                    HashString.Contains("poke_motion_0000"))
-                    GeneratePokeStrings(HashString);
-
-                string[] hashPaths = HashString.Split('/');
-                for (int i = 0; i < hashPaths?.Length; i++)
-                {
-                    hash = FNV64A1.Calculate(hashPaths[i]);
-                    if (!hashList.ContainsKey(hash))
-                        hashList.Add(hash, HashString);
-                }
-            }
         }
 
         private void GenerateAnmCfgStrings(GFBANMCFG cfg)
         {
             foreach (GFBANMCFG.Animation a in cfg.Config.Animations)
             {
-                ulong Hash = FNV64A1.Calculate(a.FileName);
+                GFPAKHashCache.PutHash(a.FileName);
-                if (!HashList.ContainsKey(Hash)) {
-                    HashList.Add(Hash, a.FileName);
-                }
             }
         }
 
-        private void GeneratePokeStrings(string hashStr)
+        private void GeneratePokeStrings()
         {
             //Also check file name just in case
             if (FileName.Contains("pm"))
             {
                 string baseName = FileName.Substring(0, 12);
-                string pokeStrFile = hashStr.Replace("pm0000_00", baseName);
 
-                ulong hash = FNV64A1.Calculate(pokeStrFile);
+                GFPAKHashCache.GeneratePokeStringsFromFile(baseName);
-                if (!hashList.ContainsKey(hash))
-                    hashList.Add(hash, pokeStrFile);
-            }
-
-            for (int i = 0; i < 1000; i++)
-            {
-                string pokeStr = string.Empty;
-                if (hashStr.Contains("pm0000")) pokeStr = hashStr.Replace("pm0000", $"pm{i.ToString("D4")}");
-                else if (hashStr.Contains("poke_XXXX")) pokeStr = hashStr.Replace("poke_XXXX", $"poke_{i.ToString("D4")}");
-                else if (hashStr.Contains("poke_ball_0000")) pokeStr = hashStr.Replace("poke_ball_0000", $"poke_ball_{i.ToString("D4")}");
-                else if (hashStr.Contains("poke_face_0000")) pokeStr = hashStr.Replace("poke_face_0000", $"poke_face_{i.ToString("D4")}");
-                else if (hashStr.Contains("poke_motion_0000")) pokeStr = hashStr.Replace("poke_motion_0000", $"poke_motion_{i.ToString("D4")}");
-
-                ulong hash = FNV64A1.Calculate(pokeStr);
-                if (!hashList.ContainsKey(hash))
-                    hashList.Add(hash, pokeStr);
             }
         }
 
@@ -689,9 +636,10 @@ namespace FirstPlugin
             bool hasFolderHash = false;
 
             string folder = "";
-            if (HashList.ContainsKey(folderHash)) {
+            string folderHashName = GFPAKHashCache.GetHashName(folderHash);
+            if (folderHashName != null) {
                 hasFolderHash = true;
-                folder = $"{HashList[folderHash]}/";
+                folder = $"{folderHashName}/";
             }
 
             if (!hasFolderHash)
@@ -721,12 +669,13 @@ namespace FirstPlugin
             }
             else
             {
-                if (HashList.ContainsKey(fileHash))
+                string fileHashName = GFPAKHashCache.GetHashName(fileHash);
+                if (fileHashName != null)
                 {
                     if (hasFolderHash)
-                        return $"{folder}{HashList[fileHash]}";
+                        return $"{folder}{fileHashName}";
                     else
-                        return $"{folder}{HashList[fileHash]}[FullHash={fullHash.ToString("X")}]{ext}";
+                        return $"{folder}{fileHashName}[FullHash={fullHash.ToString("X")}]{ext}";
                 }
                 else
                     return $"{folder}{fileHash.ToString("X")}[FullHash={fullHash.ToString("X")}]{ext}";

File_Format_Library/FileFormats/Hashes/GFPAKHashCache.cs

@@ -0,0 +1,177 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Toolbox.Library;
+
+namespace FirstPlugin.FileFormats.Hashes
+{
+    public class GFPAKHashCache
+    {
+        private static Dictionary<ulong, string> HashCacheContent;
+
+        private static ulong CurrentVersionHash;
+        private static string HashBinaryPath;
+        private static bool CacheUpdatedFlag;
+        private static List<string> PokeHashTemplates = new List<string>();
+
+        public static void InitializeHashCache()
+        {
+            HashBinaryPath = Path.Combine(Runtime.ExecutableDir, "Hashes", "GFPAKHashCache.bin");
+            string HashExtraPath = Path.Combine(Runtime.ExecutableDir, "Hashes", "GFPAK.txt");
+
+            bool NeedsBaseCacheRebuild = true;
+            CurrentVersionHash = GetToolboxVersionHash();
+
+            if (File.Exists(HashBinaryPath))
+            {
+                using (BinaryReader Reader = new BinaryReader(new FileStream(HashBinaryPath, FileMode.Open)))
+                {
+                    ulong CacheVersionHash = Reader.ReadUInt64();
+                    NeedsBaseCacheRebuild = CacheVersionHash != CurrentVersionHash;
+                    uint Count = Reader.ReadUInt32();
+                    for (uint HashIndex = 0; HashIndex < Count; HashIndex++)
+                    {
+                        ulong HashCode = Reader.ReadUInt64();
+                        string HashName = Reader.ReadString();
+                        PutHash(HashCode, HashName);
+                    }
+                }
+            }
+
+            if (NeedsBaseCacheRebuild)
+            {
+                GenerateBaseHashList();
+            }
+
+            if (File.Exists(HashExtraPath))
+            {
+                string[] UserHashLines = File.ReadAllLines(HashExtraPath);
+                foreach (string Line in UserHashLines){
+                    PutHash(Line);
+                }
+            }
+
+            WriteCache();
+        }
+
+        public static void WriteCache()
+        {
+            if (CacheUpdatedFlag)
+            {
+                using (BinaryWriter writer = new BinaryWriter(new FileStream(HashBinaryPath, FileMode.OpenOrCreate, FileAccess.Write)))
+                {
+                    writer.Write(CurrentVersionHash);
+                    writer.Write(HashCacheContent.Count);
+                    foreach (KeyValuePair<ulong, string> Entry in HashCacheContent)
+                    {
+                        writer.Write(Entry.Key);
+                        writer.Write(Entry.Value);
+                    }
+                }
+                CacheUpdatedFlag = false;
+            }
+        }
+
+        private static void GenerateBaseHashList()
+        {
+            foreach (string hashStr in Properties.Resources.Pkmn.Split('\n'))
+            {
+                string HashString = hashStr.TrimEnd();
+
+                PutHash(HashString);
+
+                if (HashString.Contains("pm0000") ||
+                    HashString.Contains("poke_XXXX") ||
+                    HashString.Contains("poke_ball_0000") ||
+                    HashString.Contains("poke_face_0000") ||
+                    HashString.Contains("poke_motion_0000"))
+                {
+                    GenerateGenericPokeStrings(HashString);
+                    PokeHashTemplates.Add(HashString);
+                }
+            }
+        }
+
+        private static void GenerateGenericPokeStrings(string hashStr)
+        {
+            for (int i = 0; i < 1000; i++)
+            {
+                string pokeStr = string.Empty;
+                if (hashStr.Contains("pm0000")) pokeStr = hashStr.Replace("pm0000", $"pm{i.ToString("D4")}");
+                else if (hashStr.Contains("poke_XXXX")) pokeStr = hashStr.Replace("poke_XXXX", $"poke_{i.ToString("D4")}");
+                else if (hashStr.Contains("poke_ball_0000")) pokeStr = hashStr.Replace("poke_ball_0000", $"poke_ball_{i.ToString("D4")}");
+                else if (hashStr.Contains("poke_face_0000")) pokeStr = hashStr.Replace("poke_face_0000", $"poke_face_{i.ToString("D4")}");
+                else if (hashStr.Contains("poke_motion_0000")) pokeStr = hashStr.Replace("poke_motion_0000", $"poke_motion_{i.ToString("D4")}");
+                ulong hash = FNV64A1.Calculate(pokeStr);
+                if (!HashCacheContent.ContainsKey(hash))
+                    HashCacheContent.Add(hash, pokeStr);
+            }
+
+        }
+
+        public static void GeneratePokeStringsFromFile(string FileName)
+        {
+            foreach (string PokeNameBase in PokeHashTemplates)
+            {
+                string pokeStrFile = PokeNameBase.Replace("pm0000_00", FileName);
+                PutHash(pokeStrFile);
+            }
+        }
+
+        public static void EnsureHashCache()
+        {
+            if (HashCacheContent == null)
+            {
+                HashCacheContent = new Dictionary<ulong, string>();
+                InitializeHashCache();
+            }
+        }
+
+        public static string GetHashName(ulong Hash)
+        {
+            if (HashCacheContent.ContainsKey(Hash))
+            {
+                return HashCacheContent[Hash];
+            }
+            return null;
+        }
+
+        public static void PutHash(string Name)
+        {
+            PutHash(FNV64A1.Calculate(Name), Name);       
+        }
+
+        public static void PutHash(ulong Hash, string Name)
+        {
+            if (!HashCacheContent.ContainsKey(Hash))
+            {
+                CacheUpdatedFlag = true;
+                HashCacheContent.Add(Hash, Name);
+
+                if (Name.Contains('/'))
+                {
+                    string[] HashPaths = Name.Split('/');
+                    for (int i = 0; i < HashPaths.Length; i++)
+                    {
+                        PutHash(HashPaths[i]);
+                    }
+                }
+            }
+        }
+
+        private static ulong GetToolboxVersionHash()
+        {
+            string VersionFilePath = Path.Combine(Runtime.ExecutableDir, "Version.txt");
+            ulong VersionHash = 0;
+            if (File.Exists(VersionFilePath))
+            {
+                byte[] VersionBytes = File.ReadAllBytes(VersionFilePath);
+                VersionHash = FNV64A1.Calculate(VersionBytes);
+            }
+            return VersionHash;
+        }
+    }
+}

File_Format_Library/File_Format_Library.csproj

@@ -249,6 +249,7 @@
     <Compile Include="FileFormats\Font\BXFNT\Images\RevImageBlock.cs" />
     <Compile Include="FileFormats\Font\BXFNT\TGLP.cs" />
     <Compile Include="FileFormats\Font\BXFNT\BxfntYamlConverter.cs" />
+    <Compile Include="FileFormats\Hashes\GFPAKHashCache.cs" />
     <Compile Include="FileFormats\HyruleWarriors\G1M\G1MCommon.cs" />
     <Compile Include="FileFormats\HyruleWarriors\G1M\NUNO.cs" />
     <Compile Include="FileFormats\HyruleWarriors\G1M\NUNV.cs" />