Merge pull request #5 from goaaats/new_uid

Generate deterministic UIDs for official songs + TJA, DLC compatibility

.gitignore

@@ -396,5 +396,6 @@ FodyWeavers.xsd
 
 # JetBrains Rider
 *.sln.iml
+.idea
 
 TakoTako/Executables/TJAConvert.exe

TakoTako/MusicPatch.cs

@@ -372,24 +372,36 @@ public class MusicPatch
                     song.genreNo = 7;
             }
 
-            var instanceId = Guid.NewGuid().ToString();
             song.SongName = song.id;
             song.FolderPath = directory;
-            song.id = instanceId;
 
-            if (uniqueIdToSong.ContainsKey(song.uniqueId) || (song.uniqueId >= 0 && song.uniqueId <= SaveDataMax))
+            // Clip off the last bit of the hash to make sure that the number is positive. This will lead to more collisions, but we should be fine.
+            if (isTjaSong)
             {
-                var uniqueIdTest = unchecked(song.id.GetHashCode() + song.previewPos + song.fumenOffsetPos);
+                // For TJAs, we need to hash the TJA file.
-                while (uniqueIdToSong.ContainsKey(uniqueIdTest) || (uniqueIdTest >= 0 && uniqueIdTest <= SaveDataMax))
+                song.UniqueId = song.tjaFileHash;
-                    uniqueIdTest = unchecked((uniqueIdTest + 1) * (uniqueIdTest + 1));
 
-                song.uniqueId = uniqueIdTest;
+                if (song.UniqueId == 0)
+                    throw new Exception("Converted TJA had no hash.");
+            }
+            else
+            {
+                // For official songs, we can just use the hash of the song internal name.
+                song.UniqueId = (int)(MurmurHash2.Hash(song.id) & 0xFFFF_FFF);
+            }
+
+            if (song.UniqueId <= SaveDataMax)
+                song.UniqueId += SaveDataMax;
+
+            if (uniqueIdToSong.ContainsKey(song.UniqueId))
+            {
+                throw new Exception($"Song \"{song.id}\" has collision with \"{uniqueIdToSong[song.UniqueId].id}\", bailing out...");
             }
 
             customSongsList.Add(song);
             idToSong[song.id] = song;
-            uniqueIdToSong[song.uniqueId] = song;
+            uniqueIdToSong[song.UniqueId] = song;
-            Log.LogInfo($"Added {(isTjaSong ? "TJA" : "")} Song {song.songName.text}");
+            Log.LogInfo($"Added{(isTjaSong ? " TJA" : "")} Song {song.songName.text}({song.UniqueId})");
         }
     }
 
@@ -451,15 +463,15 @@ public class MusicPatch
                     continue;
 
                 musicInfoAccessors.Add(new MusicDataInterface.MusicInfoAccesser(
-                    song.uniqueId,
+                    song.UniqueId, // From SongInstance, as we always recalculate it now
                     song.id,
                     $"song_{song.id}",
                     song.order,
                     song.genreNo,
-                    !Plugin.Instance.ConfigDisableCustomDLCSongs.Value,
+                    true, // We always want to mark songs as DLC, otherwise ranked games will be broken as you are gonna match songs that other people don't have
                     false,
                     0, false,
-                    0,
+                    2, // Always mark custom songs as "both players need to have this song to play it"
                     new[]
                     {
                         song.branchEasy,
@@ -1287,83 +1299,6 @@ public class MusicPatch
 #pragma warning restore Harmony003
     }
 
-    /// <summary>
-    /// Allow for a song id less than 0
-    /// </summary>
-    [HarmonyPatch(typeof(EnsoDataManager), "DecideSetting")]
-    [HarmonyPrefix]
-    public static bool DecideSetting_Prefix(EnsoDataManager __instance)
-    {
-        var ensoSettings = (EnsoData.Settings) typeof(EnsoDataManager).GetField("ensoSettings", BindingFlags.NonPublic | BindingFlags.Instance).GetValue(__instance);
-        // if (ensoSettings.musicuid.Length <= 0)
-        // {
-        // 	MusicDataInterface.MusicInfoAccesser infoByUniqueId = TaikoSingletonMonoBehaviour<CommonObjects>.Instance.MyDataManager.MusicData.GetInfoByUniqueId(ensoSettings.musicUniqueId);
-        // 	if (infoByUniqueId != null)
-        // 	{
-        // 		ensoSettings.musicuid = infoByUniqueId.Id;
-        // 	}
-        // }
-        // else if (ensoSettings.musicUniqueId <= DataConst.InvalidId)
-        // {
-        // 	MusicDataInterface.MusicInfoAccesser infoById = TaikoSingletonMonoBehaviour<CommonObjects>.Instance.MyDataManager.MusicData.GetInfoById(ensoSettings.musicuid);
-        // 	if (infoById != null)
-        // 	{
-        // 		ensoSettings.musicUniqueId = infoById.UniqueId;
-        // 	}
-        // }
-        if (ensoSettings.musicuid.Length <= 0 /* || ensoSettings.musicUniqueId <= DataConst.InvalidId*/)
-        {
-            List<MusicDataInterface.MusicInfoAccesser> musicInfoAccessers = TaikoSingletonMonoBehaviour<CommonObjects>.Instance.MyDataManager.MusicData.musicInfoAccessers;
-            for (int i = 0; i < musicInfoAccessers.Count; i++)
-            {
-                if (!musicInfoAccessers[i].Debug)
-                {
-                    ensoSettings.musicuid = musicInfoAccessers[i].Id;
-                    ensoSettings.musicUniqueId = musicInfoAccessers[i].UniqueId;
-                }
-            }
-        }
-
-        MusicDataInterface.MusicInfoAccesser infoByUniqueId2 = TaikoSingletonMonoBehaviour<CommonObjects>.Instance.MyDataManager.MusicData.GetInfoByUniqueId(ensoSettings.musicUniqueId);
-        if (infoByUniqueId2 != null)
-        {
-            ensoSettings.songFilePath = infoByUniqueId2.SongFileName;
-        }
-
-        __instance.DecidePartsSetting();
-        if (ensoSettings.ensoType == EnsoData.EnsoType.Normal)
-        {
-            int num = 0;
-            int dlcType = 2;
-            if (ensoSettings.rankMatchType == EnsoData.RankMatchType.None)
-            {
-                num = ((ensoSettings.playerNum != 1) ? 1 : 0);
-            }
-            else if (ensoSettings.rankMatchType == EnsoData.RankMatchType.RankMatch)
-            {
-                num = 2;
-                ensoSettings.isRandomSelect = false;
-                ensoSettings.isDailyBonus = false;
-            }
-            else
-            {
-                num = 3;
-                ensoSettings.isRandomSelect = false;
-                ensoSettings.isDailyBonus = false;
-            }
-
-            TaikoSingletonMonoBehaviour<CommonObjects>.Instance.CosmosLib._kpiListCommon._musicKpiInfo.SetMusicSortSettings(num, dlcType, ensoSettings.isRandomSelect, ensoSettings.isDailyBonus);
-        }
-        else
-        {
-            ensoSettings.isRandomSelect = false;
-            ensoSettings.isDailyBonus = false;
-        }
-
-        typeof(EnsoDataManager).GetField("ensoSettings", BindingFlags.NonPublic | BindingFlags.Instance).SetValue(__instance, ensoSettings);
-        return false;
-    }
-
     #endregion
 
     #region Read Fumen
@@ -1965,5 +1900,6 @@ public class MusicPatch
     {
         public string FolderPath;
         public string SongName;
+        public int UniqueId;
     }
 }

TakoTako/Plugin.cs

@@ -20,7 +20,6 @@ namespace TakoTako
         public ConfigEntry<string> ConfigSongDirectory;
         public ConfigEntry<bool> ConfigSaveEnabled;
         public ConfigEntry<string> ConfigSaveDirectory;
-        public ConfigEntry<bool> ConfigDisableCustomDLCSongs;
         public ConfigEntry<string> ConfigOverrideDefaultSongLanguage;
         public ConfigEntry<bool> ConfigApplyGenreOverride;
 
@@ -64,12 +63,6 @@ namespace TakoTako
                 $"{userFolder}/Documents/{typeof(Plugin).Namespace}/saves",
                 "The directory where saves are stored");
 
-            ConfigDisableCustomDLCSongs = Config.Bind("CustomSongs",
-                "DisableCustomDLCSongs",
-                false,
-                "By default, DLC is enabled for custom songs, this is to reduce any hiccups when playing online with other people. " +
-                "Set this to true if you want DLC to be marked as false, be aware that the fact you're playing a custom song will be sent over the internet");
-
             ConfigOverrideDefaultSongLanguage = Config.Bind("CustomSongs",
                 "ConfigOverrideDefaultSongLanguage",
                 string.Empty,

TakoTakoScripts/TJAConvert/Program.cs

@@ -99,9 +99,11 @@ namespace TJAConvert
                         Directory.Delete(tempOutDirectory, true);
                     Directory.CreateDirectory(tempOutDirectory);
 
-                    var passed = await TJAToFumens(metadata, tjaPath, tempOutDirectory);
+                    var originalTjaData = File.ReadAllBytes(tjaPath);
+                    var tjaHash = (int)(MurmurHash2.Hash(originalTjaData) & 0xFFFF_FFF);
 
-                    if (passed >= 0) passed = CreateMusicFile(metadata, tempOutDirectory) ? 0 : -1;
+                    var passed = await TJAToFumens(metadata, tjaPath, tjaHash, tempOutDirectory);
+                    if (passed >= 0) passed = CreateMusicFile(metadata, tjaHash, tempOutDirectory) ? 0 : -1;
 
                     var copyFilePath = Path.Combine(newDirectory, Path.GetFileName(originalAudioPath));
                     File.Copy(originalAudioPath, copyFilePath);
@@ -110,10 +112,10 @@ namespace TJAConvert
                     switch (audioExtension.ToLowerInvariant())
                     {
                         case "wav":
-                            if (passed >= 0) passed = WavToACB(copyFilePath, tempOutDirectory) ? 0 : -1;
+                            if (passed >= 0) passed = WavToACB(copyFilePath, tempOutDirectory, tjaHash) ? 0 : -1;
                             break;
                         case "ogg":
-                            if (passed >= 0) passed = OGGToACB(copyFilePath, tempOutDirectory) ? 0 : -1;
+                            if (passed >= 0) passed = OGGToACB(copyFilePath, tempOutDirectory, tjaHash) ? 0 : -1;
                             break;
                         default:
                             Console.WriteLine($"Do not support {audioExtension} audio files");
@@ -196,14 +198,13 @@ namespace TJAConvert
             acbFile.Save(acbPath, bufferSize);
         }
 
-        private static bool CreateMusicFile(TJAMetadata metadata, string outputPath)
+        private static bool CreateMusicFile(TJAMetadata metadata, int tjaHash, string outputPath)
         {
             try
             {
                 var musicInfo = new CustomSong
                 {
-                    uniqueId = metadata.Title.GetHashCode(),
+                    id = tjaHash.ToString(),
-                    id = metadata.Id,
                     order = 0,
                     genreNo = (int) metadata.Genre,
                     branchEasy = false,
@@ -213,6 +214,7 @@ namespace TJAConvert
                     branchUra = false,
                     previewPos = (int) (metadata.PreviewTime * 1000),
                     fumenOffsetPos = (int) (metadata.Offset * 10),
+                    tjaFileHash = tjaHash,
                     songName = new TextEntry()
                     {
                         text = metadata.Title,
@@ -316,7 +318,7 @@ namespace TJAConvert
             }
         }
 
-        private static async Task<int> TJAToFumens(TJAMetadata metadata, string tjaPath, string outputPath)
+        private static async Task<int> TJAToFumens(TJAMetadata metadata, string tjaPath, int tjaHash, string outputPath)
         {
             var fileName = Path.GetFileName(tjaPath);
             var newPath = Path.Combine(outputPath, fileName);
@@ -332,7 +334,7 @@ namespace TJAConvert
             if (metadata.Courses.Any(x => x.CourseType == CourseType.UraOni))
             {
                 // tja2bin doesn't support Ura Oni, so rip it out and change the course type to oni, then rename the final file
-                passed = await ConvertUraOni(metadata, newPath);
+                passed = await ConvertUraOni(metadata, newPath, tjaHash);
                 if (passed < 0)
                     return passed;
                 // for every .bin in this directory, we can now add the prefix _x
@@ -365,7 +367,7 @@ namespace TJAConvert
             {
                 // will need to create additional files to splice them out
 
-                passed = await SpliceDoubles(metadata, newPath);
+                passed = await SpliceDoubles(metadata, newPath, tjaHash);
                 if (passed < 0)
                     return passed;
 
@@ -380,7 +382,7 @@ namespace TJAConvert
             }
 
             if (metadata.Courses.All(x => x.PlayStyle != TJAMetadata.PlayStyle.Double))
-                passed = await Convert(newPath, outputPath);
+                passed = await Convert(newPath, outputPath, tjaHash);
 
             if (passed < 0)
                 return passed;
@@ -395,7 +397,7 @@ namespace TJAConvert
             return passed;
         }
 
-        private static async Task<int> ConvertUraOni(TJAMetadata metadata, string newPath)
+        private static async Task<int> ConvertUraOni(TJAMetadata metadata, string newPath, int tjaHash)
         {
             var directory = Path.GetDirectoryName(newPath);
             var fileName = Path.GetFileNameWithoutExtension(newPath);
@@ -427,7 +429,7 @@ namespace TJAConvert
                     var path = $"{directory}/{fileName}.tja";
                     File.WriteAllLines(path, file);
 
-                    var passed = await Convert(path, directory);
+                    var passed = await Convert(path, directory, tjaHash);
                     if (passed < 0)
                         return passed;
 
@@ -435,7 +437,7 @@ namespace TJAConvert
                 }
                 else
                 {
-                    var passed = await SplitP1P2(lines, course, directory, fileName, CourseType.Oni);
+                    var passed = await SplitP1P2(lines, course, directory, fileName, tjaHash, CourseType.Oni);
                     if (passed < 0)
                         return passed;
                 }
@@ -449,7 +451,7 @@ namespace TJAConvert
         /// <summary>
         /// This aims to separate P1 and P2 tracks for TJA2BIN to read
         /// </summary>
-        private static async Task<int> SpliceDoubles(TJAMetadata metadata, string newPath)
+        private static async Task<int> SpliceDoubles(TJAMetadata metadata, string newPath, int tjaHash)
         {
             var directory = Path.GetDirectoryName(newPath);
             var fileName = Path.GetFileNameWithoutExtension(newPath);
@@ -482,7 +484,7 @@ namespace TJAConvert
             // remove doubles section
             foreach (var course in doubleCourses)
             {
-                var passed = await SplitP1P2(lines, course, directory, fileName);
+                var passed = await SplitP1P2(lines, course, directory, fileName, tjaHash);
                 if (passed < 0)
                     return passed;
             }
@@ -491,7 +493,7 @@ namespace TJAConvert
             return 0;
         }
 
-        private static async Task<int> SplitP1P2(List<string> lines, TJAMetadata.Course course, string directory, string fileName, CourseType? courseTypeOverride = null)
+        private static async Task<int> SplitP1P2(List<string> lines, TJAMetadata.Course course, string directory, string fileName, int tjaHash, CourseType? courseTypeOverride = null)
         {
             // metadata end
             int courseStartIndex = lines.FindLastIndex(x =>
@@ -528,7 +530,7 @@ namespace TJAConvert
             var path = $"{directory}/{fileName}_1.tja";
             File.WriteAllLines(path, p1File);
 
-            var passed = await Convert(path, directory);
+            var passed = await Convert(path, directory, tjaHash);
             if (passed < 0)
                 return passed;
 
@@ -540,7 +542,7 @@ namespace TJAConvert
             path = $"{directory}/{fileName}_2.tja";
             File.WriteAllLines(path, p2File);
 
-            passed = await Convert(path, directory);
+            passed = await Convert(path, directory, tjaHash);
             if (passed < 0)
                 return passed;
 
@@ -558,9 +560,9 @@ namespace TJAConvert
             }
         }
 
-        private static async Task<int> Convert(string tjaPath, string outputPath)
+        private static async Task<int> Convert(string tjaPath, string outputPath, int tjaHash)
         {
-            var fileName = Path.GetFileNameWithoutExtension(tjaPath);
+            var fileName = tjaHash.ToString();
 
             TJAMetadata metadata;
             try
@@ -572,7 +574,7 @@ namespace TJAConvert
                 return -2;
             }
 
-            var newPath = $"{outputPath}\\{Path.GetFileName(tjaPath)}";
+            var newPath = $"{outputPath}\\{fileName}";
             if (metadata.Courses.Count == 1)
             {
                 var coursePostfix = metadata.Courses[0].CourseType.ToShort();
@@ -986,12 +988,11 @@ namespace TJAConvert
             }
         }
 
-        private static bool OGGToACB(string oggPath, string outDirectory)
+        private static bool OGGToACB(string oggPath, string outDirectory, int tjaHash)
         {
             try
             {
                 var directory = Path.GetDirectoryName(oggPath);
-                var fileName = Path.GetFileNameWithoutExtension(oggPath);
                 var acbPath = $"{directory}/{Guid.NewGuid().ToString()}";
                 Directory.CreateDirectory(acbPath);
 
@@ -1006,10 +1007,10 @@ namespace TJAConvert
 
                 File.WriteAllBytes($"{acbPath}/00000.hca", hca);
                 Pack(acbPath);
-                if (File.Exists($"{outDirectory}/song_{fileName}.bin"))
+                if (File.Exists($"{outDirectory}/song_{tjaHash}.bin"))
-                    File.Delete($"{outDirectory}/song_{fileName}.bin");
+                    File.Delete($"{outDirectory}/song_{tjaHash}.bin");
 
-                File.Move($"{acbPath}.acb", $"{outDirectory}/song_{fileName}.bin");
+                File.Move($"{acbPath}.acb", $"{outDirectory}/song_{tjaHash}.bin");
                 Directory.Delete(acbPath, true);
                 return true;
             }
@@ -1020,12 +1021,11 @@ namespace TJAConvert
             }
         }
 
-        private static bool WavToACB(string wavPath, string outDirectory, bool deleteWav = false)
+        private static bool WavToACB(string wavPath, string outDirectory, int tjaHash, bool deleteWav = false)
         {
             try
             {
                 var directory = Path.GetDirectoryName(wavPath);
-                var fileName = Path.GetFileNameWithoutExtension(wavPath);
                 var acbPath = $"{directory}/{Guid.NewGuid().ToString()}";
                 Directory.CreateDirectory(acbPath);
 
@@ -1037,10 +1037,10 @@ namespace TJAConvert
                 var hca = WavToHca(wavPath);
                 File.WriteAllBytes($"{acbPath}/00000.hca", hca);
                 Pack(acbPath);
-                if (File.Exists($"{outDirectory}/song_{fileName}.bin"))
+                if (File.Exists($"{outDirectory}/song_{tjaHash}.bin"))
-                    File.Delete($"{outDirectory}/song_{fileName}.bin");
+                    File.Delete($"{outDirectory}/song_{tjaHash}.bin");
 
-                File.Move($"{acbPath}.acb", $"{outDirectory}/song_{fileName}.bin");
+                File.Move($"{acbPath}.acb", $"{outDirectory}/song_{tjaHash}.bin");
 
                 if (deleteWav)
                     File.Delete(wavPath);

TakoTakoScripts/TakoTako.Common/CustomSong.cs

@@ -11,7 +11,6 @@ namespace TakoTako.Common
     public class CustomSong
     {
         // Song Details
-        [DataMember] public int uniqueId;
         [DataMember] public string id;
         [DataMember] public int order;
         [DataMember] public int genreNo;
@@ -41,6 +40,9 @@ namespace TakoTako.Common
         [DataMember] public int scoreMania;
         [DataMember] public int scoreUra;
 
+        // Used for UID
+        [DataMember] public int tjaFileHash;
+
         // Preview Details
         [DataMember] public int previewPos;
         [DataMember] public int fumenOffsetPos;

TakoTakoScripts/TakoTako.Common/MurmurHash2.cs

@@ -0,0 +1,85 @@
+using System.Text;
+
+// MIT License
+//
+// Copyright (c) 2017 Jitbit, 2022 TakoTako contributors
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+namespace TakoTako.Common;
+
+public class MurmurHash2
+{
+    public static uint Hash(string data)
+    {
+        return Hash(Encoding.UTF8.GetBytes(data));
+    }
+
+    public static uint Hash(byte[] data)
+    {
+        return Hash(data, 0xc58f1a7a);
+    }
+
+    private const uint m = 0x5bd1e995;
+    private const int r = 24;
+
+    public static uint Hash(byte[] data, uint seed)
+    {
+        var length = data.Length;
+        if (length == 0)
+            return 0;
+        var h = seed ^ (uint)length;
+        var currentIndex = 0;
+        while (length >= 4)
+        {
+            var k = (uint)(data[currentIndex++] | (data[currentIndex++] << 8) | (data[currentIndex++] << 16) |
+                           (data[currentIndex++] << 24));
+            k *= m;
+            k ^= k >> r;
+            k *= m;
+
+            h *= m;
+            h ^= k;
+            length -= 4;
+        }
+
+        switch (length)
+        {
+            case 3:
+                h ^= (ushort)(data[currentIndex++] | (data[currentIndex++] << 8));
+                h ^= (uint)(data[currentIndex] << 16);
+                h *= m;
+                break;
+            case 2:
+                h ^= (ushort)(data[currentIndex++] | (data[currentIndex] << 8));
+                h *= m;
+                break;
+            case 1:
+                h ^= data[currentIndex];
+                h *= m;
+                break;
+        }
+
+        h ^= h >> 13;
+        h *= m;
+        h ^= h >> 15;
+
+        return h;
+    }
+}