From afaf54cfdb32c97ce07851f94a92f8e2a117e44b Mon Sep 17 00:00:00 2001 From: Farewell_ Date: Tue, 21 Nov 2023 14:54:27 +0100 Subject: [PATCH] Updated checkDatatables, better checks for missing songs I used to rely on musicinfo.bin to build the songlist, Now using all files to build an expected song database --- checkDatatables.py | 516 +++++++++++++++++++++++++++++---------------- encryption.py | 2 +- helpers.py | 41 ++-- 3 files changed, 358 insertions(+), 201 deletions(-) diff --git a/checkDatatables.py b/checkDatatables.py index e6b2865..aab4e8d 100644 --- a/checkDatatables.py +++ b/checkDatatables.py @@ -36,13 +36,14 @@ if __name__ == "__main__": exit(1) isChn = isChn() +songs = [] # region Loading files checkFile = {} infos = loadFile(path="./Data/x64/datatable/musicinfo.bin") -usb = loadFile(path="./Data/x64/datatable/music_usbsetting.bin") -order = loadFile(path="./Data/x64/datatable/music_order.bin") +usbs = loadFile(path="./Data/x64/datatable/music_usbsetting.bin") +orders = loadFile(path="./Data/x64/datatable/music_order.bin") attributes = loadFile(path="./Data/x64/datatable/music_attribute.bin") words = loadFile(path="./Data/x64/datatable/wordlist.bin") # endregion @@ -67,38 +68,58 @@ class Genres(Enum): return cls.Unknown -class Song: - id = "" - uniqueId = -1 - genreNo = -1 - name = "" - sub = "" - detail = "" - - def __init__(self, id, uniqueId, genreNo, name, sub, detail): - self.id = id - self.uniqueId = uniqueId - self.genreNo = genreNo - self.name = name - self.sub = sub - self.detail = detail - - def initCheckFile(): global checkFile checkFile = { - "musicinfo.bin": { - "TotalEntries": len(infos), - "MaxId": max(infos, key=lambda ev: ev["uniqueId"])["uniqueId"], + "stats": { + "TotalSongs": 0, + "MaxId": 0, "UniqueIdTooHigh": 0, "UniqueIdTooHighList": [], "UnusedUniqueIds": 0, "UnusedUniqueIdsList": [], + }, + } + + if infos is not None: + checkFile["musicinfo.bin"] = { + "TotalEntries": len(infos), + "Missing": 0, + "MissingList": [], "Doublons": 0, "DoublonsList": [], "GenreNoList": [], - }, - } + } + for song in infos: + name = findKeyInList( + list=words, + key="key", + keyValue="song_" + song["id"], + value=language, + ) + sub = findKeyInList( + list=words, + key="key", + keyValue="song_sub_" + song["id"], + value=language, + ) + detail = findKeyInList( + list=words, + key="key", + keyValue="song_detail_" + song["id"], + value=language, + ) + + songs.append( + { + "id": song["id"], + "uniqueId": song["uniqueId"], + "genreNo": song["genreNo"], + "name": name, + "sub": sub, + "detail": detail, + } + ) if attributes is not None: checkFile["music_attribute.bin"] = { @@ -110,10 +131,50 @@ def initCheckFile(): "Doublons": 0, "DoublonsList": [], } + for song in attributes: + temp = findKeyInList(list=songs, key="id", keyValue=song["id"]) + if temp is not None: + continue - if order is not None: + name = findKeyInList( + list=words, + key="key", + keyValue="song_" + song["id"], + value=language, + ) + sub = findKeyInList( + list=words, + key="key", + keyValue="song_sub_" + song["id"], + value=language, + ) + detail = findKeyInList( + list=words, + key="key", + keyValue="song_detail_" + song["id"], + value=language, + ) + genreNo = findKeyInList( + list=infos, + key="id", + keyValue=song["id"], + value="genreNo", + ) + + songs.append( + { + "id": song["id"], + "uniqueId": song["uniqueId"], + "genreNo": genreNo, + "name": name, + "sub": sub, + "detail": detail, + } + ) + + if orders is not None: checkFile["music_order.bin"] = { - "TotalEntries": len(order), + "TotalEntries": len(orders), "UniqueEntries": 0, "UniqueEntriesList": [], "GenreNoList": [], @@ -122,10 +183,50 @@ def initCheckFile(): "Mismatch": 0, "MismatchList": [], } + for song in orders: + temp = findKeyInList(list=songs, key="id", keyValue=song["id"]) + if temp is not None: + continue - if usb is not None: + name = findKeyInList( + list=words, + key="key", + keyValue="song_" + song["id"], + value=language, + ) + sub = findKeyInList( + list=words, + key="key", + keyValue="song_sub_" + song["id"], + value=language, + ) + detail = findKeyInList( + list=words, + key="key", + keyValue="song_detail_" + song["id"], + value=language, + ) + genreNo = findKeyInList( + list=infos, + key="id", + keyValue=song["id"], + value="genreNo", + ) + + songs.append( + { + "id": song["id"], + "uniqueId": song["uniqueId"], + "genreNo": genreNo, + "name": name, + "sub": sub, + "detail": detail, + } + ) + + if usbs is not None: checkFile["music_usbsetting.bin"] = { - "TotalEntries": len(usb), + "TotalEntries": len(usbs), "Missing": 0, "MissingList": [], "Mismatch": 0, @@ -133,6 +234,46 @@ def initCheckFile(): "Doublons": 0, "DoublonsList": [], } + for song in usbs: + temp = findKeyInList(list=songs, key="id", keyValue=song["id"]) + if temp is not None: + continue + + name = findKeyInList( + list=words, + key="key", + keyValue="song_" + song["id"], + value=language, + ) + sub = findKeyInList( + list=words, + key="key", + keyValue="song_sub_" + song["id"], + value=language, + ) + detail = findKeyInList( + list=words, + key="key", + keyValue="song_detail_" + song["id"], + value=language, + ) + genreNo = findKeyInList( + list=infos, + key="id", + keyValue=song["id"], + value="genreNo", + ) + + songs.append( + { + "id": song["id"], + "uniqueId": song["uniqueId"], + "genreNo": genreNo, + "name": name, + "sub": sub, + "detail": detail, + } + ) if words is not None: checkFile["wordlist.bin"] = { @@ -147,181 +288,183 @@ def initCheckFile(): "DoublonsList": [], } - checkFile.update( - { - "GameFiles": { - "MissingSound": 0, - "MissingSoundList": [], - "MissingFumen": 0, - "MissingFumenList": [], - }, - } - ) + if all([doesPathExist("./Data/x64/fumen"), doesPathExist("./Data/x64/sound")]): + checkFile.update( + { + "GameFiles": { + "MissingSound": 0, + "MissingSoundList": [], + "MissingFumen": 0, + "MissingFumenList": [], + }, + } + ) + + # Update stats + checkFile["stats"]["TotalSongs"] = len(songs) + checkFile["stats"]["MaxId"] = max(songs, key=lambda ev: ev["uniqueId"])["uniqueId"] + + print("Found a total of", str(len(songs)), "songs in datatables") # endregion -# Loading all songs from musicinfo in an array -songs = [] - -for song in infos: - name = findKeyInList( - list=words, - key="key", - keyValue="song_" + song["id"], - value=language, - ) - sub = findKeyInList( - list=words, - key="key", - keyValue="song_sub_" + song["id"], - value=language, - ) - detail = findKeyInList( - list=words, - key="key", - keyValue="song_detail_" + song["id"], - value=language, - ) - - songs.append( - Song( - id=song["id"], - uniqueId=song["uniqueId"], - genreNo=song["genreNo"], - name=name, - sub=sub, - detail=detail, - ) - ) # Preparing the json file containing the results of this checking script initCheckFile() # Checking... for song in songs: + # stats + # Checking for too high of an id + if song["uniqueId"] > 1599: + checkFile["stats"]["UniqueIdTooHigh"] += 1 + checkFile["stats"]["UniqueIdTooHighList"].append( + { + "id": song["id"], + "uniqueId": song["uniqueId"], + } + ) + # musicinfo.bin if infos is not None: - # Checking for too high of an id - if song.uniqueId > 1599: - checkFile["musicinfo.bin"]["UniqueIdTooHigh"] += 1 - checkFile["musicinfo.bin"]["UniqueIdTooHighList"].append( - { - "id": song.id, - "uniqueId": song.uniqueId, - } - ) + # Check for missing uniqueIds or id and uniqueId mismatches + orderOccurences = findAllObjects(list=infos, key="id", keyValue=song["id"]) + if len(orderOccurences) == 0: + checkFile["musicinfo.bin"]["Missing"] += 1 + checkFile["musicinfo.bin"]["MissingList"].append(song["id"]) # Listing genres and counting entries for each genres genre = { - "GenreNo": song.genreNo, - "Name": Genres(song.genreNo).name, + "GenreNo": song["genreNo"], + "Name": Genres(song["genreNo"]).name, "NumberofSongs": 0, } if ( findKeyInList( list=checkFile["musicinfo.bin"]["GenreNoList"], key="GenreNo", - keyValue=song.genreNo, + keyValue=song["genreNo"], ) is None ): genre["NumberofSongs"] = len( - findAllObjects(list=infos, key="genreNo", keyValue=song.genreNo) + findAllObjects(list=infos, key="genreNo", keyValue=song["genreNo"]) ) checkFile["musicinfo.bin"]["GenreNoList"].append(genre) # Search doublons - if findDoubloninList(list=infos, key="id", keyValue=song.id): - if song.id not in checkFile["musicinfo.bin"]["DoublonsList"]: + if findDoubloninList(list=infos, key="id", keyValue=song["id"]): + if song["id"] not in checkFile["musicinfo.bin"]["DoublonsList"]: checkFile["musicinfo.bin"]["Doublons"] += 1 - checkFile["musicinfo.bin"]["DoublonsList"].append(song.id) + checkFile["musicinfo.bin"]["DoublonsList"].append(song["id"]) - if findDoubloninList(list=infos, key="uniqueId", keyValue=song.uniqueId): - if song.id not in checkFile["musicinfo.bin"]["DoublonsList"]: + if findDoubloninList(list=infos, key="uniqueId", keyValue=song["uniqueId"]): + if song["uniqueId"] not in checkFile["musicinfo.bin"]["DoublonsList"]: checkFile["musicinfo.bin"]["Doublons"] += 1 - checkFile["musicinfo.bin"]["DoublonsList"].append(song.uniqueId) - - # music_usbsetting.bin - if usb is not None: - # Check for missing uniqueIds or id and uniqueId mismatches - orderOccurences = findAllObjects(list=usb, key="id", keyValue=song.id) - if len(orderOccurences) == 0: - checkFile["music_usbsetting.bin"]["Missing"] += 1 - checkFile["music_usbsetting.bin"]["MissingList"].append(song.id) - else: - for occurence in orderOccurences: - if not all( - [song.id == occurence["id"], song.uniqueId == occurence["uniqueId"]] - ): - if song.id not in checkFile["music_usbsetting.bin"]["MismatchList"]: - checkFile["music_usbsetting.bin"]["Mismatch"] += 1 - checkFile["music_usbsetting.bin"]["MismatchList"].append( - { - "id": song.id, - "ExpectedUniqueId": song.uniqueId, - "CurrentUniqueId": occurence["uniqueId"], - } - ) - - # Search doublons - if findDoubloninList(list=usb, key="id", keyValue=song.id): - if song.id not in checkFile["music_usbsetting.bin"]["DoublonsList"]: - checkFile["music_usbsetting.bin"]["Doublons"] += 1 - checkFile["music_usbsetting.bin"]["DoublonsList"].append(song.id) - if findDoubloninList(list=usb, key="uniqueId", keyValue=song.uniqueId): - if song.id not in checkFile["musicinfo.bin"]["DoublonsList"]: - checkFile["music_usbsetting.bin"]["Doublons"] += 1 - checkFile["music_usbsetting.bin"]["DoublonsList"].append(song.uniqueId) + checkFile["musicinfo.bin"]["DoublonsList"].append(song["uniqueId"]) # music_attribute.bin if attributes is not None: # Check for missing uniqueIds or id and uniqueId mismatches - orderOccurences = findAllObjects(list=attributes, key="id", keyValue=song.id) + orderOccurences = findAllObjects(list=attributes, key="id", keyValue=song["id"]) if len(orderOccurences) == 0: checkFile["music_attribute.bin"]["Missing"] += 1 - checkFile["music_attribute.bin"]["MissingList"].append(song.id) + checkFile["music_attribute.bin"]["MissingList"].append(song["id"]) else: for occurence in orderOccurences: if not all( - [song.id == occurence["id"], song.uniqueId == occurence["uniqueId"]] + [ + song["id"] == occurence["id"], + song["uniqueId"] == occurence["uniqueId"], + ] ): - if song.id not in checkFile["music_attribute.bin"]["MismatchList"]: + if ( + song["id"] + not in checkFile["music_attribute.bin"]["MismatchList"] + ): checkFile["music_attribute.bin"]["Mismatch"] += 1 checkFile["music_attribute.bin"]["MismatchList"].append( { - "id": song.id, - "ExpectedUniqueId": song.uniqueId, + "id": song["id"], + "ExpectedUniqueId": song["uniqueId"], "CurrentUniqueId": occurence["uniqueId"], } ) - if findDoubloninList(list=attributes, key="id", keyValue=song.id): - if song.id not in checkFile["music_attribute.bin"]["DoublonsList"]: + if findDoubloninList(list=attributes, key="id", keyValue=song["id"]): + if song["id"] not in checkFile["music_attribute.bin"]["DoublonsList"]: checkFile["music_attribute.bin"]["Doublons"] += 1 - checkFile["music_attribute.bin"]["DoublonsList"].append(song.id) - if findDoubloninList(list=attributes, key="uniqueId", keyValue=song.uniqueId): - if song.id not in checkFile["musicinfo.bin"]["DoublonsList"]: + checkFile["music_attribute.bin"]["DoublonsList"].append(song["id"]) + if findDoubloninList( + list=attributes, key="uniqueId", keyValue=song["uniqueId"] + ): + if song["uniqueId"] not in checkFile["musicinfo.bin"]["DoublonsList"]: checkFile["music_attribute.bin"]["Doublons"] += 1 - checkFile["music_attribute.bin"]["DoublonsList"].append(song.uniqueId) + checkFile["music_attribute.bin"]["DoublonsList"].append( + song["uniqueId"] + ) + + # music_usbsetting.bin + if usbs is not None: + # Check for missing uniqueIds or id and uniqueId mismatches + orderOccurences = findAllObjects(list=usbs, key="id", keyValue=song["id"]) + if len(orderOccurences) == 0: + checkFile["music_usbsetting.bin"]["Missing"] += 1 + checkFile["music_usbsetting.bin"]["MissingList"].append(song["id"]) + else: + for occurence in orderOccurences: + if not all( + [ + song["id"] == occurence["id"], + song["uniqueId"] == occurence["uniqueId"], + ] + ): + if ( + song["id"] + not in checkFile["music_usbsetting.bin"]["MismatchList"] + ): + checkFile["music_usbsetting.bin"]["Mismatch"] += 1 + checkFile["music_usbsetting.bin"]["MismatchList"].append( + { + "id": song["id"], + "ExpectedUniqueId": song["uniqueId"], + "CurrentUniqueId": occurence["uniqueId"], + } + ) + + # Search doublons + if findDoubloninList(list=usbs, key="id", keyValue=song["id"]): + if song["id"] not in checkFile["music_usbsetting.bin"]["DoublonsList"]: + checkFile["music_usbsetting.bin"]["Doublons"] += 1 + checkFile["music_usbsetting.bin"]["DoublonsList"].append(song["id"]) + if findDoubloninList(list=usbs, key="uniqueId", keyValue=song["uniqueId"]): + if song["uniqueId"] not in checkFile["musicinfo.bin"]["DoublonsList"]: + checkFile["music_usbsetting.bin"]["Doublons"] += 1 + checkFile["music_usbsetting.bin"]["DoublonsList"].append( + song["uniqueId"] + ) # music_order.bin - if order is not None: + if orders is not None: # Check for missing uniqueIds or id and uniqueId mismatches - orderOccurences = findAllObjects(list=order, key="id", keyValue=song.id) + orderOccurences = findAllObjects(list=orders, key="id", keyValue=song["id"]) if len(orderOccurences) == 0: checkFile["music_order.bin"]["Missing"] += 1 - checkFile["music_order.bin"]["MissingList"].append(song.id) + checkFile["music_order.bin"]["MissingList"].append(song["id"]) else: songGenres = [] for occurence in orderOccurences: songGenres.append(occurence["genreNo"]) if not all( - [song.id == occurence["id"], song.uniqueId == occurence["uniqueId"]] + [ + song["id"] == occurence["id"], + song["uniqueId"] == occurence["uniqueId"], + ] ): - if song.id not in checkFile["music_order.bin"]["MismatchList"]: + if song["id"] not in checkFile["music_order.bin"]["MismatchList"]: checkFile["music_order.bin"]["Mismatch"] += 1 checkFile["music_order.bin"]["MismatchList"].append( { - "id": song.id, - "ExpectedUniqueId": song.uniqueId, + "id": song["id"], + "ExpectedUniqueId": song["uniqueId"], "CurrentUniqueId": occurence["uniqueId"], } ) @@ -330,61 +473,65 @@ for song in songs: checkFile["music_order.bin"]["UniqueEntries"] += 1 checkFile["music_order.bin"]["UniqueEntriesList"].append( { - song.id: songGenres, + song["id"]: songGenres, } ) # wordlist.bin if words is not None: - if song.name == "": + if song["name"] == "": checkFile["wordlist.bin"]["MissingSongName"] += 1 - checkFile["wordlist.bin"]["MissingSongNameList"].append(song.id) - if song.sub == "": + checkFile["wordlist.bin"]["MissingSongNameList"].append(song["id"]) + if song["sub"] == "": checkFile["wordlist.bin"]["MissingSongSub"] += 1 - checkFile["wordlist.bin"]["MissingSongSubList"].append(song.id) - if song.detail == "": + checkFile["wordlist.bin"]["MissingSongSubList"].append(song["id"]) + if song["detail"] == "": checkFile["wordlist.bin"]["MissingSongDetail"] += 1 - checkFile["wordlist.bin"]["MissingSongDetailList"].append(song.id) + checkFile["wordlist.bin"]["MissingSongDetailList"].append(song["id"]) # Gamefiles - if not doesPathExist("./Data/x64/sound/" + "song_" + song.id + ".nus3bank"): - checkFile["GameFiles"]["MissingSound"] += 1 - checkFile["GameFiles"]["MissingSoundList"].append(song.id) - if not doesPathExist("./Data/x64/fumen/" + song.id): - checkFile["GameFiles"]["MissingFumen"] += 1 - checkFile["GameFiles"]["MissingFumenList"].append(song.id) + if all([doesPathExist("./Data/x64/fumen"), doesPathExist("./Data/x64/sound")]): + if not doesPathExist("./Data/x64/sound/" + "song_" + song["id"] + ".nus3bank"): + checkFile["GameFiles"]["MissingSound"] += 1 + checkFile["GameFiles"]["MissingSoundList"].append(song["id"]) + if not doesPathExist("./Data/x64/fumen/" + song["id"]): + checkFile["GameFiles"]["MissingFumen"] += 1 + checkFile["GameFiles"]["MissingFumenList"].append(song["id"]) # Checking for vacant uniqueIds -for i in range(max(checkFile["musicinfo.bin"]["MaxId"], 1600)): - key = findKeyInList(list=infos, key="uniqueId", keyValue=i) +for i in range(max(checkFile["stats"]["MaxId"], 1600)): + key = findKeyInList(list=songs, key="uniqueId", keyValue=i) if key is not None: # Updating GenreNoList of music_order.bin - for song in findAllObjects( - list=order, key="uniqueId", keyValue=key["uniqueId"] - ): - genre = { - "GenreNo": song["genreNo"], - "Name": Genres(song["genreNo"]).name, - "NumberofSongs": 0, - } - if ( - findKeyInList( - list=checkFile["music_order.bin"]["GenreNoList"], - key="GenreNo", - keyValue=song["genreNo"], - ) - is None + if orders is not None: + for song in findAllObjects( + list=orders, key="uniqueId", keyValue=key["uniqueId"] ): - genre["NumberofSongs"] = len( - findAllObjects(list=order, key="genreNo", keyValue=song["genreNo"]) - ) - checkFile["music_order.bin"]["GenreNoList"].append(genre) + genre = { + "GenreNo": song["genreNo"], + "Name": Genres(song["genreNo"]).name, + "NumberofSongs": 0, + } + if ( + findKeyInList( + list=checkFile["music_order.bin"]["GenreNoList"], + key="GenreNo", + keyValue=song["genreNo"], + ) + is None + ): + genre["NumberofSongs"] = len( + findAllObjects( + list=orders, key="genreNo", keyValue=song["genreNo"] + ) + ) + checkFile["music_order.bin"]["GenreNoList"].append(genre) else: # Finding unused Ids bellow 1599 if i < 1600: - checkFile["musicinfo.bin"]["UnusedUniqueIds"] += 1 - checkFile["musicinfo.bin"]["UnusedUniqueIdsList"].append(i) + checkFile["stats"]["UnusedUniqueIds"] += 1 + checkFile["stats"]["UnusedUniqueIdsList"].append(i) # Checking for doublons in wordlist if words is not None: @@ -394,13 +541,16 @@ if words is not None: checkFile["wordlist.bin"]["Doublons"] += 1 checkFile["wordlist.bin"]["DoublonsList"].append(word["key"]) + # Sorting some values for better readability -checkFile["musicinfo.bin"]["GenreNoList"].sort( - key=lambda x: x["GenreNo"], reverse=False -) -checkFile["music_order.bin"]["GenreNoList"].sort( - key=lambda x: x["GenreNo"], reverse=False -) +if infos is not None: + checkFile["musicinfo.bin"]["GenreNoList"].sort( + key=lambda x: str(x["GenreNo"]), reverse=False + ) +if orders is not None: + checkFile["music_order.bin"]["GenreNoList"].sort( + key=lambda x: str(x["GenreNo"]), reverse=False + ) # Writing everything to checks.json json_object = json.dumps(checkFile, ensure_ascii=False, indent="\t") diff --git a/encryption.py b/encryption.py index 02eff6f..5a21181 100644 --- a/encryption.py +++ b/encryption.py @@ -68,7 +68,7 @@ def decrypt_file(input_file, key_type: Keys = Keys(Keys.Datatable)): def isJson(file: bytes): try: - json.loads(file)["items"] + json.loads(file) return True except: return False diff --git a/helpers.py b/helpers.py index 256ecfc..ff0ee97 100644 --- a/helpers.py +++ b/helpers.py @@ -27,10 +27,14 @@ def loadFile(path: str): try: if not isChn(): # Loading files for 08.18 - return json.load(gzip.open(path, "rb"))["items"] + file = json.load(gzip.open(path, "rb"))["items"] + print("Successfully loaded", path) + return file else: # Loading files for 32.09 CHN - return json.loads(decrypt_file(input_file=path))["items"] + file = json.loads(decrypt_file(input_file=path))["items"] + print("Successfully loaded", path) + return file except Exception as error: print(error) print("Couldn't load", path) @@ -40,22 +44,25 @@ def loadFile(path: str): def findKeyInList(list: list, key: str, keyValue, value=None): - for object in list: - try: - if object[key] == keyValue: + try: + for object in list: + try: + if object[key] == keyValue: + if value is not None: + return object[value] + else: + return object + except: if value is not None: - return object[value] - else: - return object - except: - if value is not None: - print( - value - + " doesn't exist in " - + str(object) - + ", are you using the right language ?" - ) - exit(0) + print( + value + + " doesn't exist in " + + str(object) + + ", are you using the right language ?" + ) + exit(0) + except Exception as error: + return None if value is not None: return ""