Updated checkDatatables, encryption converts to JSON

checkDatatables has a better doublons detection (now also detects uniqueID doublons)

encryption.py now automatically changes the extension to json and back to bin for datatable files
This commit is contained in:
Farewell_ 2023-11-21 11:12:49 +01:00
parent beb8feede8
commit 2a104a12c7
4 changed files with 112 additions and 68 deletions

View File

@ -76,8 +76,8 @@ It is meant to be used for basic checks such as:
* Listing the amount of songs in your tables
* Listing all vacant entries bellow 1599 to facilitate adding songs in
* Checking their uniqueIds to make sure they don't exceed 1599
* Listing all missing word entries for your songlist
* Checking for doublons in various files
* Listing all missing word entries from your song list
* Checking for id and uniqueId doublons in various files
* Checking for id and uniqueId mismatches in various files
* Checking for missing sound and fumen files

View File

@ -57,8 +57,8 @@ class Genres(Enum):
Vocaloid = 3
GameMusic = 4
NamcoOriginal = 5
Variety = 6 if not isChn else 7
Classical = 7 if not isChn else 8
Variety = 6 if isChn else 7
Classical = 7 if isChn else 8
if not isChn:
Custom = 9
@ -87,7 +87,7 @@ class Song:
def initCheckFile():
global checkFile
checkFile = {
"musicinfo.json": {
"musicinfo.bin": {
"TotalEntries": len(infos),
"MaxId": max(infos, key=lambda ev: ev["uniqueId"])["uniqueId"],
"UniqueIdTooHigh": 0,
@ -101,7 +101,7 @@ def initCheckFile():
}
if attributes is not None:
checkFile["music_attribute.json"] = {
checkFile["music_attribute.bin"] = {
"TotalEntries": len(attributes),
"Missing": 0,
"MissingList": [],
@ -112,7 +112,7 @@ def initCheckFile():
}
if order is not None:
checkFile["music_order.json"] = {
checkFile["music_order.bin"] = {
"TotalEntries": len(order),
"UniqueEntries": 0,
"UniqueEntriesList": [],
@ -124,7 +124,7 @@ def initCheckFile():
}
if usb is not None:
checkFile["music_usbsetting.json"] = {
checkFile["music_usbsetting.bin"] = {
"TotalEntries": len(usb),
"Missing": 0,
"MissingList": [],
@ -135,7 +135,7 @@ def initCheckFile():
}
if words is not None:
checkFile["wordlist.json"] = {
checkFile["wordlist.bin"] = {
"TotalEntries": len(words),
"MissingSongName": 0,
"MissingSongNameList": [],
@ -200,12 +200,12 @@ initCheckFile()
# Checking...
for song in songs:
# musicinfo.json
# musicinfo.bin
if infos is not None:
# Checking for too high of an id
if song.uniqueId > 1599:
checkFile["musicinfo.json"]["UniqueIdTooHigh"] += 1
checkFile["musicinfo.json"]["UniqueIdTooHighList"].append(
checkFile["musicinfo.bin"]["UniqueIdTooHigh"] += 1
checkFile["musicinfo.bin"]["UniqueIdTooHighList"].append(
{
"id": song.id,
"uniqueId": song.uniqueId,
@ -219,7 +219,7 @@ for song in songs:
}
if (
findKeyInList(
list=checkFile["musicinfo.json"]["GenreNoList"],
list=checkFile["musicinfo.bin"]["GenreNoList"],
key="GenreNo",
keyValue=song.genreNo,
)
@ -228,31 +228,33 @@ for song in songs:
genre["NumberofSongs"] = len(
findAllObjects(list=infos, key="genreNo", keyValue=song.genreNo)
)
checkFile["musicinfo.json"]["GenreNoList"].append(genre)
checkFile["musicinfo.bin"]["GenreNoList"].append(genre)
# Search doublons
if findDoubloninList(list=infos, key="id", keyValue=song.id):
if song.id not in checkFile["musicinfo.json"]["DoublonsList"]:
checkFile["musicinfo.json"]["Doublons"] += 1
checkFile["musicinfo.json"]["DoublonsList"].append(song.id)
if song.id not in checkFile["musicinfo.bin"]["DoublonsList"]:
checkFile["musicinfo.bin"]["Doublons"] += 1
checkFile["musicinfo.bin"]["DoublonsList"].append(song.id)
# music_usbsetting.json
if findDoubloninList(list=infos, key="uniqueId", keyValue=song.uniqueId):
if song.id not in checkFile["musicinfo.bin"]["DoublonsList"]:
checkFile["musicinfo.bin"]["Doublons"] += 1
checkFile["musicinfo.bin"]["DoublonsList"].append(song.uniqueId)
# music_usbsetting.bin
if usb is not None:
# Check for missing uniqueIds or id and uniqueId mismatches
orderOccurences = findAllObjects(list=usb, key="id", keyValue=song.id)
if len(orderOccurences) == 0:
checkFile["music_usbsetting.json"]["Missing"] += 1
checkFile["music_usbsetting.json"]["MissingList"].append(song.id)
checkFile["music_usbsetting.bin"]["Missing"] += 1
checkFile["music_usbsetting.bin"]["MissingList"].append(song.id)
else:
for occurence in orderOccurences:
if not all(
[song.id == occurence["id"], song.uniqueId == occurence["uniqueId"]]
):
if (
song.id
not in checkFile["music_usbsetting.json"]["MismatchList"]
):
checkFile["music_usbsetting.json"]["Mismatch"] += 1
checkFile["music_usbsetting.json"]["MismatchList"].append(
if song.id not in checkFile["music_usbsetting.bin"]["MismatchList"]:
checkFile["music_usbsetting.bin"]["Mismatch"] += 1
checkFile["music_usbsetting.bin"]["MismatchList"].append(
{
"id": song.id,
"ExpectedUniqueId": song.uniqueId,
@ -262,25 +264,29 @@ for song in songs:
# Search doublons
if findDoubloninList(list=usb, key="id", keyValue=song.id):
if song.id not in checkFile["music_usbsetting.json"]["DoublonsList"]:
checkFile["music_usbsetting.json"]["Doublons"] += 1
checkFile["music_usbsetting.json"]["DoublonsList"].append(song.id)
if song.id not in checkFile["music_usbsetting.bin"]["DoublonsList"]:
checkFile["music_usbsetting.bin"]["Doublons"] += 1
checkFile["music_usbsetting.bin"]["DoublonsList"].append(song.id)
if findDoubloninList(list=usb, key="uniqueId", keyValue=song.uniqueId):
if song.id not in checkFile["musicinfo.bin"]["DoublonsList"]:
checkFile["music_usbsetting.bin"]["Doublons"] += 1
checkFile["music_usbsetting.bin"]["DoublonsList"].append(song.uniqueId)
# music_attribute.json
# music_attribute.bin
if attributes is not None:
# Check for missing uniqueIds or id and uniqueId mismatches
orderOccurences = findAllObjects(list=attributes, key="id", keyValue=song.id)
if len(orderOccurences) == 0:
checkFile["music_attribute.json"]["Missing"] += 1
checkFile["music_attribute.json"]["MissingList"].append(song.id)
checkFile["music_attribute.bin"]["Missing"] += 1
checkFile["music_attribute.bin"]["MissingList"].append(song.id)
else:
for occurence in orderOccurences:
if not all(
[song.id == occurence["id"], song.uniqueId == occurence["uniqueId"]]
):
if song.id not in checkFile["music_attribute.json"]["MismatchList"]:
checkFile["music_attribute.json"]["Mismatch"] += 1
checkFile["music_attribute.json"]["MismatchList"].append(
if song.id not in checkFile["music_attribute.bin"]["MismatchList"]:
checkFile["music_attribute.bin"]["Mismatch"] += 1
checkFile["music_attribute.bin"]["MismatchList"].append(
{
"id": song.id,
"ExpectedUniqueId": song.uniqueId,
@ -288,17 +294,21 @@ for song in songs:
}
)
if findDoubloninList(list=attributes, key="id", keyValue=song.id):
if song.id not in checkFile["music_attribute.json"]["DoublonsList"]:
checkFile["music_attribute.json"]["Doublons"] += 1
checkFile["music_attribute.json"]["DoublonsList"].append(song.id)
if song.id not in checkFile["music_attribute.bin"]["DoublonsList"]:
checkFile["music_attribute.bin"]["Doublons"] += 1
checkFile["music_attribute.bin"]["DoublonsList"].append(song.id)
if findDoubloninList(list=attributes, key="uniqueId", keyValue=song.uniqueId):
if song.id not in checkFile["musicinfo.bin"]["DoublonsList"]:
checkFile["music_attribute.bin"]["Doublons"] += 1
checkFile["music_attribute.bin"]["DoublonsList"].append(song.uniqueId)
# music_order.json
# music_order.bin
if order is not None:
# Check for missing uniqueIds or id and uniqueId mismatches
orderOccurences = findAllObjects(list=order, key="id", keyValue=song.id)
if len(orderOccurences) == 0:
checkFile["music_order.json"]["Missing"] += 1
checkFile["music_order.json"]["MissingList"].append(song.id)
checkFile["music_order.bin"]["Missing"] += 1
checkFile["music_order.bin"]["MissingList"].append(song.id)
else:
songGenres = []
for occurence in orderOccurences:
@ -306,9 +316,9 @@ for song in songs:
if not all(
[song.id == occurence["id"], song.uniqueId == occurence["uniqueId"]]
):
if song.id not in checkFile["music_order.json"]["MismatchList"]:
checkFile["music_order.json"]["Mismatch"] += 1
checkFile["music_order.json"]["MismatchList"].append(
if song.id not in checkFile["music_order.bin"]["MismatchList"]:
checkFile["music_order.bin"]["Mismatch"] += 1
checkFile["music_order.bin"]["MismatchList"].append(
{
"id": song.id,
"ExpectedUniqueId": song.uniqueId,
@ -317,24 +327,24 @@ for song in songs:
)
# Counting unique entries
checkFile["music_order.json"]["UniqueEntries"] += 1
checkFile["music_order.json"]["UniqueEntriesList"].append(
checkFile["music_order.bin"]["UniqueEntries"] += 1
checkFile["music_order.bin"]["UniqueEntriesList"].append(
{
song.id: songGenres,
}
)
# wordlist.json
# wordlist.bin
if words is not None:
if song.name == "":
checkFile["wordlist.json"]["MissingSongName"] += 1
checkFile["wordlist.json"]["MissingSongNameList"].append(song.id)
checkFile["wordlist.bin"]["MissingSongName"] += 1
checkFile["wordlist.bin"]["MissingSongNameList"].append(song.id)
if song.sub == "":
checkFile["wordlist.json"]["MissingSongSub"] += 1
checkFile["wordlist.json"]["MissingSongSubList"].append(song.id)
checkFile["wordlist.bin"]["MissingSongSub"] += 1
checkFile["wordlist.bin"]["MissingSongSubList"].append(song.id)
if song.detail == "":
checkFile["wordlist.json"]["MissingSongDetail"] += 1
checkFile["wordlist.json"]["MissingSongDetailList"].append(song.id)
checkFile["wordlist.bin"]["MissingSongDetail"] += 1
checkFile["wordlist.bin"]["MissingSongDetailList"].append(song.id)
# Gamefiles
if not doesPathExist("./Data/x64/sound/" + "song_" + song.id + ".nus3bank"):
@ -345,11 +355,11 @@ for song in songs:
checkFile["GameFiles"]["MissingFumenList"].append(song.id)
# Checking for vacant uniqueIds
for i in range(max(checkFile["musicinfo.json"]["MaxId"], 1600)):
for i in range(max(checkFile["musicinfo.bin"]["MaxId"], 1600)):
key = findKeyInList(list=infos, key="uniqueId", keyValue=i)
if key is not None:
# Updating GenreNoList of music_order.json
# Updating GenreNoList of music_order.bin
for song in findAllObjects(
list=order, key="uniqueId", keyValue=key["uniqueId"]
):
@ -360,7 +370,7 @@ for i in range(max(checkFile["musicinfo.json"]["MaxId"], 1600)):
}
if (
findKeyInList(
list=checkFile["music_order.json"]["GenreNoList"],
list=checkFile["music_order.bin"]["GenreNoList"],
key="GenreNo",
keyValue=song["genreNo"],
)
@ -369,26 +379,26 @@ for i in range(max(checkFile["musicinfo.json"]["MaxId"], 1600)):
genre["NumberofSongs"] = len(
findAllObjects(list=order, key="genreNo", keyValue=song["genreNo"])
)
checkFile["music_order.json"]["GenreNoList"].append(genre)
checkFile["music_order.bin"]["GenreNoList"].append(genre)
else:
# Finding unused Ids bellow 1599
if i < 1600:
checkFile["musicinfo.json"]["UnusedUniqueIds"] += 1
checkFile["musicinfo.json"]["UnusedUniqueIdsList"].append(i)
checkFile["musicinfo.bin"]["UnusedUniqueIds"] += 1
checkFile["musicinfo.bin"]["UnusedUniqueIdsList"].append(i)
# Checking for doublons in wordlist
if words is not None:
for word in words:
if findDoubloninList(list=words, key="key", keyValue=word["key"]):
if word["key"] not in checkFile["wordlist.json"]["DoublonsList"]:
checkFile["wordlist.json"]["Doublons"] += 1
checkFile["wordlist.json"]["DoublonsList"].append(word["key"])
if word["key"] not in checkFile["wordlist.bin"]["DoublonsList"]:
checkFile["wordlist.bin"]["Doublons"] += 1
checkFile["wordlist.bin"]["DoublonsList"].append(word["key"])
# Sorting some values for better readability
checkFile["musicinfo.json"]["GenreNoList"].sort(
checkFile["musicinfo.bin"]["GenreNoList"].sort(
key=lambda x: x["GenreNo"], reverse=False
)
checkFile["music_order.json"]["GenreNoList"].sort(
checkFile["music_order.bin"]["GenreNoList"].sort(
key=lambda x: x["GenreNo"], reverse=False
)

View File

@ -1,4 +1,5 @@
import gzip
import json
import os
from pathlib import Path
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
@ -39,9 +40,14 @@ def decrypt_file(input_file, key_type: Keys = Keys(Keys.Datatable)):
# Read the IV from the first 16 bytes of the input file
iv = read_iv_from_file(input_file)
# Create an AES cipher object with CBC mode
cipher = Cipher(algorithms.AES(key), modes.CBC(iv), backend=default_backend())
decryptor = cipher.decryptor()
try:
# Create an AES cipher object with CBC mode
cipher = Cipher(algorithms.AES(key), modes.CBC(iv), backend=default_backend())
decryptor = cipher.decryptor()
except Exception as error:
print(error)
print("You need to set the right AES keys in the encryption.py file")
exit(0)
with open(input_file, "rb") as infile:
# Skip the IV in the input file
@ -60,6 +66,14 @@ def decrypt_file(input_file, key_type: Keys = Keys(Keys.Datatable)):
return decompressed_data
def isJson(file: bytes):
try:
json.loads(file)["items"]
return True
except:
return False
def encrypt_file(input_file, key_type: Keys = Keys(Keys.Datatable)):
# Convert the key from hex to bytes
key = binascii.unhexlify(Keys(key_type.value).value)
@ -99,6 +113,13 @@ def save_file(file: bytes, outdir: str, encrypt: bool):
else encrypt_file(input_file=file, key_type=type)
)
if isJson(fileContent):
base = os.path.splitext(outdir)[0]
outdir = base + ".json"
else:
base = os.path.splitext(outdir)[0]
outdir = base + ".bin"
print("Decrypting" if not encrypt else "Encrypting", file, "to", outdir)
with open(outdir, "wb") as outfile:

View File

@ -6,7 +6,20 @@ from encryption import decrypt_file
def isChn():
return os.path.exists("./Data/x64/datatable/music_usbsetting.bin")
try:
try:
# Trying to load file for 08.18
json.load(gzip.open("./Data/x64/datatable/musicinfo.bin", "rb"))["items"]
return False
except Exception as error:
# Trying to load file for 32.09 CHN
json.loads(decrypt_file(input_file="./Data/x64/datatable/musicinfo.bin"))[
"items"
]
return True
except Exception as error:
print(error)
print("Couldn't find musicinfo.bin, assuming 08.18")
def loadFile(path: str):