Updated checkDatatables, better checks for missing songs

I used to rely on musicinfo.bin to build the songlist,
Now using all files to build an expected song database
This commit is contained in:
Farewell_ 2023-11-21 14:54:27 +01:00
parent 2a104a12c7
commit afaf54cfdb
3 changed files with 358 additions and 201 deletions

View File

@ -36,13 +36,14 @@ if __name__ == "__main__":
exit(1)
isChn = isChn()
songs = []
# region Loading files
checkFile = {}
infos = loadFile(path="./Data/x64/datatable/musicinfo.bin")
usb = loadFile(path="./Data/x64/datatable/music_usbsetting.bin")
order = loadFile(path="./Data/x64/datatable/music_order.bin")
usbs = loadFile(path="./Data/x64/datatable/music_usbsetting.bin")
orders = loadFile(path="./Data/x64/datatable/music_order.bin")
attributes = loadFile(path="./Data/x64/datatable/music_attribute.bin")
words = loadFile(path="./Data/x64/datatable/wordlist.bin")
# endregion
@ -67,38 +68,58 @@ class Genres(Enum):
return cls.Unknown
class Song:
id = ""
uniqueId = -1
genreNo = -1
name = ""
sub = ""
detail = ""
def __init__(self, id, uniqueId, genreNo, name, sub, detail):
self.id = id
self.uniqueId = uniqueId
self.genreNo = genreNo
self.name = name
self.sub = sub
self.detail = detail
def initCheckFile():
global checkFile
checkFile = {
"musicinfo.bin": {
"TotalEntries": len(infos),
"MaxId": max(infos, key=lambda ev: ev["uniqueId"])["uniqueId"],
"stats": {
"TotalSongs": 0,
"MaxId": 0,
"UniqueIdTooHigh": 0,
"UniqueIdTooHighList": [],
"UnusedUniqueIds": 0,
"UnusedUniqueIdsList": [],
},
}
if infos is not None:
checkFile["musicinfo.bin"] = {
"TotalEntries": len(infos),
"Missing": 0,
"MissingList": [],
"Doublons": 0,
"DoublonsList": [],
"GenreNoList": [],
},
}
}
for song in infos:
name = findKeyInList(
list=words,
key="key",
keyValue="song_" + song["id"],
value=language,
)
sub = findKeyInList(
list=words,
key="key",
keyValue="song_sub_" + song["id"],
value=language,
)
detail = findKeyInList(
list=words,
key="key",
keyValue="song_detail_" + song["id"],
value=language,
)
songs.append(
{
"id": song["id"],
"uniqueId": song["uniqueId"],
"genreNo": song["genreNo"],
"name": name,
"sub": sub,
"detail": detail,
}
)
if attributes is not None:
checkFile["music_attribute.bin"] = {
@ -110,10 +131,50 @@ def initCheckFile():
"Doublons": 0,
"DoublonsList": [],
}
for song in attributes:
temp = findKeyInList(list=songs, key="id", keyValue=song["id"])
if temp is not None:
continue
if order is not None:
name = findKeyInList(
list=words,
key="key",
keyValue="song_" + song["id"],
value=language,
)
sub = findKeyInList(
list=words,
key="key",
keyValue="song_sub_" + song["id"],
value=language,
)
detail = findKeyInList(
list=words,
key="key",
keyValue="song_detail_" + song["id"],
value=language,
)
genreNo = findKeyInList(
list=infos,
key="id",
keyValue=song["id"],
value="genreNo",
)
songs.append(
{
"id": song["id"],
"uniqueId": song["uniqueId"],
"genreNo": genreNo,
"name": name,
"sub": sub,
"detail": detail,
}
)
if orders is not None:
checkFile["music_order.bin"] = {
"TotalEntries": len(order),
"TotalEntries": len(orders),
"UniqueEntries": 0,
"UniqueEntriesList": [],
"GenreNoList": [],
@ -122,10 +183,50 @@ def initCheckFile():
"Mismatch": 0,
"MismatchList": [],
}
for song in orders:
temp = findKeyInList(list=songs, key="id", keyValue=song["id"])
if temp is not None:
continue
if usb is not None:
name = findKeyInList(
list=words,
key="key",
keyValue="song_" + song["id"],
value=language,
)
sub = findKeyInList(
list=words,
key="key",
keyValue="song_sub_" + song["id"],
value=language,
)
detail = findKeyInList(
list=words,
key="key",
keyValue="song_detail_" + song["id"],
value=language,
)
genreNo = findKeyInList(
list=infos,
key="id",
keyValue=song["id"],
value="genreNo",
)
songs.append(
{
"id": song["id"],
"uniqueId": song["uniqueId"],
"genreNo": genreNo,
"name": name,
"sub": sub,
"detail": detail,
}
)
if usbs is not None:
checkFile["music_usbsetting.bin"] = {
"TotalEntries": len(usb),
"TotalEntries": len(usbs),
"Missing": 0,
"MissingList": [],
"Mismatch": 0,
@ -133,6 +234,46 @@ def initCheckFile():
"Doublons": 0,
"DoublonsList": [],
}
for song in usbs:
temp = findKeyInList(list=songs, key="id", keyValue=song["id"])
if temp is not None:
continue
name = findKeyInList(
list=words,
key="key",
keyValue="song_" + song["id"],
value=language,
)
sub = findKeyInList(
list=words,
key="key",
keyValue="song_sub_" + song["id"],
value=language,
)
detail = findKeyInList(
list=words,
key="key",
keyValue="song_detail_" + song["id"],
value=language,
)
genreNo = findKeyInList(
list=infos,
key="id",
keyValue=song["id"],
value="genreNo",
)
songs.append(
{
"id": song["id"],
"uniqueId": song["uniqueId"],
"genreNo": genreNo,
"name": name,
"sub": sub,
"detail": detail,
}
)
if words is not None:
checkFile["wordlist.bin"] = {
@ -147,181 +288,183 @@ def initCheckFile():
"DoublonsList": [],
}
checkFile.update(
{
"GameFiles": {
"MissingSound": 0,
"MissingSoundList": [],
"MissingFumen": 0,
"MissingFumenList": [],
},
}
)
if all([doesPathExist("./Data/x64/fumen"), doesPathExist("./Data/x64/sound")]):
checkFile.update(
{
"GameFiles": {
"MissingSound": 0,
"MissingSoundList": [],
"MissingFumen": 0,
"MissingFumenList": [],
},
}
)
# Update stats
checkFile["stats"]["TotalSongs"] = len(songs)
checkFile["stats"]["MaxId"] = max(songs, key=lambda ev: ev["uniqueId"])["uniqueId"]
print("Found a total of", str(len(songs)), "songs in datatables")
# endregion
# Loading all songs from musicinfo in an array
songs = []
for song in infos:
name = findKeyInList(
list=words,
key="key",
keyValue="song_" + song["id"],
value=language,
)
sub = findKeyInList(
list=words,
key="key",
keyValue="song_sub_" + song["id"],
value=language,
)
detail = findKeyInList(
list=words,
key="key",
keyValue="song_detail_" + song["id"],
value=language,
)
songs.append(
Song(
id=song["id"],
uniqueId=song["uniqueId"],
genreNo=song["genreNo"],
name=name,
sub=sub,
detail=detail,
)
)
# Preparing the json file containing the results of this checking script
initCheckFile()
# Checking...
for song in songs:
# stats
# Checking for too high of an id
if song["uniqueId"] > 1599:
checkFile["stats"]["UniqueIdTooHigh"] += 1
checkFile["stats"]["UniqueIdTooHighList"].append(
{
"id": song["id"],
"uniqueId": song["uniqueId"],
}
)
# musicinfo.bin
if infos is not None:
# Checking for too high of an id
if song.uniqueId > 1599:
checkFile["musicinfo.bin"]["UniqueIdTooHigh"] += 1
checkFile["musicinfo.bin"]["UniqueIdTooHighList"].append(
{
"id": song.id,
"uniqueId": song.uniqueId,
}
)
# Check for missing uniqueIds or id and uniqueId mismatches
orderOccurences = findAllObjects(list=infos, key="id", keyValue=song["id"])
if len(orderOccurences) == 0:
checkFile["musicinfo.bin"]["Missing"] += 1
checkFile["musicinfo.bin"]["MissingList"].append(song["id"])
# Listing genres and counting entries for each genres
genre = {
"GenreNo": song.genreNo,
"Name": Genres(song.genreNo).name,
"GenreNo": song["genreNo"],
"Name": Genres(song["genreNo"]).name,
"NumberofSongs": 0,
}
if (
findKeyInList(
list=checkFile["musicinfo.bin"]["GenreNoList"],
key="GenreNo",
keyValue=song.genreNo,
keyValue=song["genreNo"],
)
is None
):
genre["NumberofSongs"] = len(
findAllObjects(list=infos, key="genreNo", keyValue=song.genreNo)
findAllObjects(list=infos, key="genreNo", keyValue=song["genreNo"])
)
checkFile["musicinfo.bin"]["GenreNoList"].append(genre)
# Search doublons
if findDoubloninList(list=infos, key="id", keyValue=song.id):
if song.id not in checkFile["musicinfo.bin"]["DoublonsList"]:
if findDoubloninList(list=infos, key="id", keyValue=song["id"]):
if song["id"] not in checkFile["musicinfo.bin"]["DoublonsList"]:
checkFile["musicinfo.bin"]["Doublons"] += 1
checkFile["musicinfo.bin"]["DoublonsList"].append(song.id)
checkFile["musicinfo.bin"]["DoublonsList"].append(song["id"])
if findDoubloninList(list=infos, key="uniqueId", keyValue=song.uniqueId):
if song.id not in checkFile["musicinfo.bin"]["DoublonsList"]:
if findDoubloninList(list=infos, key="uniqueId", keyValue=song["uniqueId"]):
if song["uniqueId"] not in checkFile["musicinfo.bin"]["DoublonsList"]:
checkFile["musicinfo.bin"]["Doublons"] += 1
checkFile["musicinfo.bin"]["DoublonsList"].append(song.uniqueId)
# music_usbsetting.bin
if usb is not None:
# Check for missing uniqueIds or id and uniqueId mismatches
orderOccurences = findAllObjects(list=usb, key="id", keyValue=song.id)
if len(orderOccurences) == 0:
checkFile["music_usbsetting.bin"]["Missing"] += 1
checkFile["music_usbsetting.bin"]["MissingList"].append(song.id)
else:
for occurence in orderOccurences:
if not all(
[song.id == occurence["id"], song.uniqueId == occurence["uniqueId"]]
):
if song.id not in checkFile["music_usbsetting.bin"]["MismatchList"]:
checkFile["music_usbsetting.bin"]["Mismatch"] += 1
checkFile["music_usbsetting.bin"]["MismatchList"].append(
{
"id": song.id,
"ExpectedUniqueId": song.uniqueId,
"CurrentUniqueId": occurence["uniqueId"],
}
)
# Search doublons
if findDoubloninList(list=usb, key="id", keyValue=song.id):
if song.id not in checkFile["music_usbsetting.bin"]["DoublonsList"]:
checkFile["music_usbsetting.bin"]["Doublons"] += 1
checkFile["music_usbsetting.bin"]["DoublonsList"].append(song.id)
if findDoubloninList(list=usb, key="uniqueId", keyValue=song.uniqueId):
if song.id not in checkFile["musicinfo.bin"]["DoublonsList"]:
checkFile["music_usbsetting.bin"]["Doublons"] += 1
checkFile["music_usbsetting.bin"]["DoublonsList"].append(song.uniqueId)
checkFile["musicinfo.bin"]["DoublonsList"].append(song["uniqueId"])
# music_attribute.bin
if attributes is not None:
# Check for missing uniqueIds or id and uniqueId mismatches
orderOccurences = findAllObjects(list=attributes, key="id", keyValue=song.id)
orderOccurences = findAllObjects(list=attributes, key="id", keyValue=song["id"])
if len(orderOccurences) == 0:
checkFile["music_attribute.bin"]["Missing"] += 1
checkFile["music_attribute.bin"]["MissingList"].append(song.id)
checkFile["music_attribute.bin"]["MissingList"].append(song["id"])
else:
for occurence in orderOccurences:
if not all(
[song.id == occurence["id"], song.uniqueId == occurence["uniqueId"]]
[
song["id"] == occurence["id"],
song["uniqueId"] == occurence["uniqueId"],
]
):
if song.id not in checkFile["music_attribute.bin"]["MismatchList"]:
if (
song["id"]
not in checkFile["music_attribute.bin"]["MismatchList"]
):
checkFile["music_attribute.bin"]["Mismatch"] += 1
checkFile["music_attribute.bin"]["MismatchList"].append(
{
"id": song.id,
"ExpectedUniqueId": song.uniqueId,
"id": song["id"],
"ExpectedUniqueId": song["uniqueId"],
"CurrentUniqueId": occurence["uniqueId"],
}
)
if findDoubloninList(list=attributes, key="id", keyValue=song.id):
if song.id not in checkFile["music_attribute.bin"]["DoublonsList"]:
if findDoubloninList(list=attributes, key="id", keyValue=song["id"]):
if song["id"] not in checkFile["music_attribute.bin"]["DoublonsList"]:
checkFile["music_attribute.bin"]["Doublons"] += 1
checkFile["music_attribute.bin"]["DoublonsList"].append(song.id)
if findDoubloninList(list=attributes, key="uniqueId", keyValue=song.uniqueId):
if song.id not in checkFile["musicinfo.bin"]["DoublonsList"]:
checkFile["music_attribute.bin"]["DoublonsList"].append(song["id"])
if findDoubloninList(
list=attributes, key="uniqueId", keyValue=song["uniqueId"]
):
if song["uniqueId"] not in checkFile["musicinfo.bin"]["DoublonsList"]:
checkFile["music_attribute.bin"]["Doublons"] += 1
checkFile["music_attribute.bin"]["DoublonsList"].append(song.uniqueId)
checkFile["music_attribute.bin"]["DoublonsList"].append(
song["uniqueId"]
)
# music_usbsetting.bin
if usbs is not None:
# Check for missing uniqueIds or id and uniqueId mismatches
orderOccurences = findAllObjects(list=usbs, key="id", keyValue=song["id"])
if len(orderOccurences) == 0:
checkFile["music_usbsetting.bin"]["Missing"] += 1
checkFile["music_usbsetting.bin"]["MissingList"].append(song["id"])
else:
for occurence in orderOccurences:
if not all(
[
song["id"] == occurence["id"],
song["uniqueId"] == occurence["uniqueId"],
]
):
if (
song["id"]
not in checkFile["music_usbsetting.bin"]["MismatchList"]
):
checkFile["music_usbsetting.bin"]["Mismatch"] += 1
checkFile["music_usbsetting.bin"]["MismatchList"].append(
{
"id": song["id"],
"ExpectedUniqueId": song["uniqueId"],
"CurrentUniqueId": occurence["uniqueId"],
}
)
# Search doublons
if findDoubloninList(list=usbs, key="id", keyValue=song["id"]):
if song["id"] not in checkFile["music_usbsetting.bin"]["DoublonsList"]:
checkFile["music_usbsetting.bin"]["Doublons"] += 1
checkFile["music_usbsetting.bin"]["DoublonsList"].append(song["id"])
if findDoubloninList(list=usbs, key="uniqueId", keyValue=song["uniqueId"]):
if song["uniqueId"] not in checkFile["musicinfo.bin"]["DoublonsList"]:
checkFile["music_usbsetting.bin"]["Doublons"] += 1
checkFile["music_usbsetting.bin"]["DoublonsList"].append(
song["uniqueId"]
)
# music_order.bin
if order is not None:
if orders is not None:
# Check for missing uniqueIds or id and uniqueId mismatches
orderOccurences = findAllObjects(list=order, key="id", keyValue=song.id)
orderOccurences = findAllObjects(list=orders, key="id", keyValue=song["id"])
if len(orderOccurences) == 0:
checkFile["music_order.bin"]["Missing"] += 1
checkFile["music_order.bin"]["MissingList"].append(song.id)
checkFile["music_order.bin"]["MissingList"].append(song["id"])
else:
songGenres = []
for occurence in orderOccurences:
songGenres.append(occurence["genreNo"])
if not all(
[song.id == occurence["id"], song.uniqueId == occurence["uniqueId"]]
[
song["id"] == occurence["id"],
song["uniqueId"] == occurence["uniqueId"],
]
):
if song.id not in checkFile["music_order.bin"]["MismatchList"]:
if song["id"] not in checkFile["music_order.bin"]["MismatchList"]:
checkFile["music_order.bin"]["Mismatch"] += 1
checkFile["music_order.bin"]["MismatchList"].append(
{
"id": song.id,
"ExpectedUniqueId": song.uniqueId,
"id": song["id"],
"ExpectedUniqueId": song["uniqueId"],
"CurrentUniqueId": occurence["uniqueId"],
}
)
@ -330,61 +473,65 @@ for song in songs:
checkFile["music_order.bin"]["UniqueEntries"] += 1
checkFile["music_order.bin"]["UniqueEntriesList"].append(
{
song.id: songGenres,
song["id"]: songGenres,
}
)
# wordlist.bin
if words is not None:
if song.name == "":
if song["name"] == "":
checkFile["wordlist.bin"]["MissingSongName"] += 1
checkFile["wordlist.bin"]["MissingSongNameList"].append(song.id)
if song.sub == "":
checkFile["wordlist.bin"]["MissingSongNameList"].append(song["id"])
if song["sub"] == "":
checkFile["wordlist.bin"]["MissingSongSub"] += 1
checkFile["wordlist.bin"]["MissingSongSubList"].append(song.id)
if song.detail == "":
checkFile["wordlist.bin"]["MissingSongSubList"].append(song["id"])
if song["detail"] == "":
checkFile["wordlist.bin"]["MissingSongDetail"] += 1
checkFile["wordlist.bin"]["MissingSongDetailList"].append(song.id)
checkFile["wordlist.bin"]["MissingSongDetailList"].append(song["id"])
# Gamefiles
if not doesPathExist("./Data/x64/sound/" + "song_" + song.id + ".nus3bank"):
checkFile["GameFiles"]["MissingSound"] += 1
checkFile["GameFiles"]["MissingSoundList"].append(song.id)
if not doesPathExist("./Data/x64/fumen/" + song.id):
checkFile["GameFiles"]["MissingFumen"] += 1
checkFile["GameFiles"]["MissingFumenList"].append(song.id)
if all([doesPathExist("./Data/x64/fumen"), doesPathExist("./Data/x64/sound")]):
if not doesPathExist("./Data/x64/sound/" + "song_" + song["id"] + ".nus3bank"):
checkFile["GameFiles"]["MissingSound"] += 1
checkFile["GameFiles"]["MissingSoundList"].append(song["id"])
if not doesPathExist("./Data/x64/fumen/" + song["id"]):
checkFile["GameFiles"]["MissingFumen"] += 1
checkFile["GameFiles"]["MissingFumenList"].append(song["id"])
# Checking for vacant uniqueIds
for i in range(max(checkFile["musicinfo.bin"]["MaxId"], 1600)):
key = findKeyInList(list=infos, key="uniqueId", keyValue=i)
for i in range(max(checkFile["stats"]["MaxId"], 1600)):
key = findKeyInList(list=songs, key="uniqueId", keyValue=i)
if key is not None:
# Updating GenreNoList of music_order.bin
for song in findAllObjects(
list=order, key="uniqueId", keyValue=key["uniqueId"]
):
genre = {
"GenreNo": song["genreNo"],
"Name": Genres(song["genreNo"]).name,
"NumberofSongs": 0,
}
if (
findKeyInList(
list=checkFile["music_order.bin"]["GenreNoList"],
key="GenreNo",
keyValue=song["genreNo"],
)
is None
if orders is not None:
for song in findAllObjects(
list=orders, key="uniqueId", keyValue=key["uniqueId"]
):
genre["NumberofSongs"] = len(
findAllObjects(list=order, key="genreNo", keyValue=song["genreNo"])
)
checkFile["music_order.bin"]["GenreNoList"].append(genre)
genre = {
"GenreNo": song["genreNo"],
"Name": Genres(song["genreNo"]).name,
"NumberofSongs": 0,
}
if (
findKeyInList(
list=checkFile["music_order.bin"]["GenreNoList"],
key="GenreNo",
keyValue=song["genreNo"],
)
is None
):
genre["NumberofSongs"] = len(
findAllObjects(
list=orders, key="genreNo", keyValue=song["genreNo"]
)
)
checkFile["music_order.bin"]["GenreNoList"].append(genre)
else:
# Finding unused Ids bellow 1599
if i < 1600:
checkFile["musicinfo.bin"]["UnusedUniqueIds"] += 1
checkFile["musicinfo.bin"]["UnusedUniqueIdsList"].append(i)
checkFile["stats"]["UnusedUniqueIds"] += 1
checkFile["stats"]["UnusedUniqueIdsList"].append(i)
# Checking for doublons in wordlist
if words is not None:
@ -394,13 +541,16 @@ if words is not None:
checkFile["wordlist.bin"]["Doublons"] += 1
checkFile["wordlist.bin"]["DoublonsList"].append(word["key"])
# Sorting some values for better readability
checkFile["musicinfo.bin"]["GenreNoList"].sort(
key=lambda x: x["GenreNo"], reverse=False
)
checkFile["music_order.bin"]["GenreNoList"].sort(
key=lambda x: x["GenreNo"], reverse=False
)
if infos is not None:
checkFile["musicinfo.bin"]["GenreNoList"].sort(
key=lambda x: str(x["GenreNo"]), reverse=False
)
if orders is not None:
checkFile["music_order.bin"]["GenreNoList"].sort(
key=lambda x: str(x["GenreNo"]), reverse=False
)
# Writing everything to checks.json
json_object = json.dumps(checkFile, ensure_ascii=False, indent="\t")

View File

@ -68,7 +68,7 @@ def decrypt_file(input_file, key_type: Keys = Keys(Keys.Datatable)):
def isJson(file: bytes):
try:
json.loads(file)["items"]
json.loads(file)
return True
except:
return False

View File

@ -27,10 +27,14 @@ def loadFile(path: str):
try:
if not isChn():
# Loading files for 08.18
return json.load(gzip.open(path, "rb"))["items"]
file = json.load(gzip.open(path, "rb"))["items"]
print("Successfully loaded", path)
return file
else:
# Loading files for 32.09 CHN
return json.loads(decrypt_file(input_file=path))["items"]
file = json.loads(decrypt_file(input_file=path))["items"]
print("Successfully loaded", path)
return file
except Exception as error:
print(error)
print("Couldn't load", path)
@ -40,22 +44,25 @@ def loadFile(path: str):
def findKeyInList(list: list, key: str, keyValue, value=None):
for object in list:
try:
if object[key] == keyValue:
try:
for object in list:
try:
if object[key] == keyValue:
if value is not None:
return object[value]
else:
return object
except:
if value is not None:
return object[value]
else:
return object
except:
if value is not None:
print(
value
+ " doesn't exist in "
+ str(object)
+ ", are you using the right language ?"
)
exit(0)
print(
value
+ " doesn't exist in "
+ str(object)
+ ", are you using the right language ?"
)
exit(0)
except Exception as error:
return None
if value is not None:
return ""