AkaiiKitsune/nijiiro-toolset
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Updated checkDatatables, encryption converts to JSON

Browse Source 
checkDatatables has a better doublons detection (now also detects uniqueID doublons)

encryption.py now automatically changes the extension to json and back to bin for datatable files
This commit is contained in:
Farewell_ 2023-11-21 11:12:49 +01:00
parent beb8feede8
commit 2a104a12c7
4 changed files with 112 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@ -76,8 +76,8 @@ It is meant to be used for basic checks such as:
* Listing the amount of songs in your tables * Listing the amount of songs in your tables
* Listing all vacant entries bellow 1599 to facilitate adding songs in * Listing all vacant entries bellow 1599 to facilitate adding songs in
* Checking their uniqueIds to make sure they don't exceed 1599 * Checking their uniqueIds to make sure they don't exceed 1599
* Listing all missing word entries for your songlist * Listing all missing word entries from your song list
* Checking for doublons in various files * Checking for id and uniqueId doublons in various files
* Checking for id and uniqueId mismatches in various files * Checking for id and uniqueId mismatches in various files
* Checking for missing sound and fumen files * Checking for missing sound and fumen files

134
checkDatatables.py
View File

@ -57,8 +57,8 @@ class Genres(Enum):
Vocaloid = 3 Vocaloid = 3
GameMusic = 4 GameMusic = 4
NamcoOriginal = 5 NamcoOriginal = 5
Variety = 6 if not isChn else 7 Variety = 6 if isChn else 7
Classical = 7 if not isChn else 8 Classical = 7 if isChn else 8
if not isChn: if not isChn:
Custom = 9 Custom = 9
@ -87,7 +87,7 @@ class Song:
def initCheckFile(): def initCheckFile():
global checkFile global checkFile
checkFile = { checkFile = {
"musicinfo.json": { "musicinfo.bin": {
"TotalEntries": len(infos), "TotalEntries": len(infos),
"MaxId": max(infos, key=lambda ev: ev["uniqueId"])["uniqueId"], "MaxId": max(infos, key=lambda ev: ev["uniqueId"])["uniqueId"],
"UniqueIdTooHigh": 0, "UniqueIdTooHigh": 0,
@ -101,7 +101,7 @@ def initCheckFile():
} }
if attributes is not None: if attributes is not None:
checkFile["music_attribute.json"] = { checkFile["music_attribute.bin"] = {
"TotalEntries": len(attributes), "TotalEntries": len(attributes),
"Missing": 0, "Missing": 0,
"MissingList": [], "MissingList": [],
@ -112,7 +112,7 @@ def initCheckFile():
} }
if order is not None: if order is not None:
checkFile["music_order.json"] = { checkFile["music_order.bin"] = {
"TotalEntries": len(order), "TotalEntries": len(order),
"UniqueEntries": 0, "UniqueEntries": 0,
"UniqueEntriesList": [], "UniqueEntriesList": [],
@ -124,7 +124,7 @@ def initCheckFile():
} }
if usb is not None: if usb is not None:
checkFile["music_usbsetting.json"] = { checkFile["music_usbsetting.bin"] = {
"TotalEntries": len(usb), "TotalEntries": len(usb),
"Missing": 0, "Missing": 0,
"MissingList": [], "MissingList": [],
@ -135,7 +135,7 @@ def initCheckFile():
} }
if words is not None: if words is not None:
checkFile["wordlist.json"] = { checkFile["wordlist.bin"] = {
"TotalEntries": len(words), "TotalEntries": len(words),
"MissingSongName": 0, "MissingSongName": 0,
"MissingSongNameList": [], "MissingSongNameList": [],
@ -200,12 +200,12 @@ initCheckFile()
# Checking... # Checking...
for song in songs: for song in songs:
# musicinfo.json # musicinfo.bin
if infos is not None: if infos is not None:
# Checking for too high of an id # Checking for too high of an id
if song.uniqueId > 1599: if song.uniqueId > 1599:
checkFile["musicinfo.json"]["UniqueIdTooHigh"] += 1 checkFile["musicinfo.bin"]["UniqueIdTooHigh"] += 1
checkFile["musicinfo.json"]["UniqueIdTooHighList"].append( checkFile["musicinfo.bin"]["UniqueIdTooHighList"].append(
{ {
"id": song.id, "id": song.id,
"uniqueId": song.uniqueId, "uniqueId": song.uniqueId,
@ -219,7 +219,7 @@ for song in songs:
} }
if ( if (
findKeyInList( findKeyInList(
list=checkFile["musicinfo.json"]["GenreNoList"], list=checkFile["musicinfo.bin"]["GenreNoList"],
key="GenreNo", key="GenreNo",
keyValue=song.genreNo, keyValue=song.genreNo,
) )
@ -228,31 +228,33 @@ for song in songs:
genre["NumberofSongs"] = len( genre["NumberofSongs"] = len(
findAllObjects(list=infos, key="genreNo", keyValue=song.genreNo) findAllObjects(list=infos, key="genreNo", keyValue=song.genreNo)
) )
checkFile["musicinfo.json"]["GenreNoList"].append(genre) checkFile["musicinfo.bin"]["GenreNoList"].append(genre)
# Search doublons # Search doublons
if findDoubloninList(list=infos, key="id", keyValue=song.id): if findDoubloninList(list=infos, key="id", keyValue=song.id):
if song.id not in checkFile["musicinfo.json"]["DoublonsList"]: if song.id not in checkFile["musicinfo.bin"]["DoublonsList"]:
checkFile["musicinfo.json"]["Doublons"] += 1 checkFile["musicinfo.bin"]["Doublons"] += 1
checkFile["musicinfo.json"]["DoublonsList"].append(song.id) checkFile["musicinfo.bin"]["DoublonsList"].append(song.id)
# music_usbsetting.json if findDoubloninList(list=infos, key="uniqueId", keyValue=song.uniqueId):
if song.id not in checkFile["musicinfo.bin"]["DoublonsList"]:
checkFile["musicinfo.bin"]["Doublons"] += 1
checkFile["musicinfo.bin"]["DoublonsList"].append(song.uniqueId)
# music_usbsetting.bin
if usb is not None: if usb is not None:
# Check for missing uniqueIds or id and uniqueId mismatches # Check for missing uniqueIds or id and uniqueId mismatches
orderOccurences = findAllObjects(list=usb, key="id", keyValue=song.id) orderOccurences = findAllObjects(list=usb, key="id", keyValue=song.id)
if len(orderOccurences) == 0: if len(orderOccurences) == 0:
checkFile["music_usbsetting.json"]["Missing"] += 1 checkFile["music_usbsetting.bin"]["Missing"] += 1
checkFile["music_usbsetting.json"]["MissingList"].append(song.id) checkFile["music_usbsetting.bin"]["MissingList"].append(song.id)
else: else:
for occurence in orderOccurences: for occurence in orderOccurences:
if not all( if not all(
[song.id == occurence["id"], song.uniqueId == occurence["uniqueId"]] [song.id == occurence["id"], song.uniqueId == occurence["uniqueId"]]
): ):
if ( if song.id not in checkFile["music_usbsetting.bin"]["MismatchList"]:
song.id checkFile["music_usbsetting.bin"]["Mismatch"] += 1
not in checkFile["music_usbsetting.json"]["MismatchList"] checkFile["music_usbsetting.bin"]["MismatchList"].append(
):
checkFile["music_usbsetting.json"]["Mismatch"] += 1
checkFile["music_usbsetting.json"]["MismatchList"].append(
{ {
"id": song.id, "id": song.id,
"ExpectedUniqueId": song.uniqueId, "ExpectedUniqueId": song.uniqueId,
@ -262,25 +264,29 @@ for song in songs:
# Search doublons # Search doublons
if findDoubloninList(list=usb, key="id", keyValue=song.id): if findDoubloninList(list=usb, key="id", keyValue=song.id):
if song.id not in checkFile["music_usbsetting.json"]["DoublonsList"]: if song.id not in checkFile["music_usbsetting.bin"]["DoublonsList"]:
checkFile["music_usbsetting.json"]["Doublons"] += 1 checkFile["music_usbsetting.bin"]["Doublons"] += 1
checkFile["music_usbsetting.json"]["DoublonsList"].append(song.id) checkFile["music_usbsetting.bin"]["DoublonsList"].append(song.id)
if findDoubloninList(list=usb, key="uniqueId", keyValue=song.uniqueId):
if song.id not in checkFile["musicinfo.bin"]["DoublonsList"]:
checkFile["music_usbsetting.bin"]["Doublons"] += 1
checkFile["music_usbsetting.bin"]["DoublonsList"].append(song.uniqueId)
# music_attribute.json # music_attribute.bin
if attributes is not None: if attributes is not None:
# Check for missing uniqueIds or id and uniqueId mismatches # Check for missing uniqueIds or id and uniqueId mismatches
orderOccurences = findAllObjects(list=attributes, key="id", keyValue=song.id) orderOccurences = findAllObjects(list=attributes, key="id", keyValue=song.id)
if len(orderOccurences) == 0: if len(orderOccurences) == 0:
checkFile["music_attribute.json"]["Missing"] += 1 checkFile["music_attribute.bin"]["Missing"] += 1
checkFile["music_attribute.json"]["MissingList"].append(song.id) checkFile["music_attribute.bin"]["MissingList"].append(song.id)
else: else:
for occurence in orderOccurences: for occurence in orderOccurences:
if not all( if not all(
[song.id == occurence["id"], song.uniqueId == occurence["uniqueId"]] [song.id == occurence["id"], song.uniqueId == occurence["uniqueId"]]
): ):
if song.id not in checkFile["music_attribute.json"]["MismatchList"]: if song.id not in checkFile["music_attribute.bin"]["MismatchList"]:
checkFile["music_attribute.json"]["Mismatch"] += 1 checkFile["music_attribute.bin"]["Mismatch"] += 1
checkFile["music_attribute.json"]["MismatchList"].append( checkFile["music_attribute.bin"]["MismatchList"].append(
{ {
"id": song.id, "id": song.id,
"ExpectedUniqueId": song.uniqueId, "ExpectedUniqueId": song.uniqueId,
@ -288,17 +294,21 @@ for song in songs:
} }
) )
if findDoubloninList(list=attributes, key="id", keyValue=song.id): if findDoubloninList(list=attributes, key="id", keyValue=song.id):
if song.id not in checkFile["music_attribute.json"]["DoublonsList"]: if song.id not in checkFile["music_attribute.bin"]["DoublonsList"]:
checkFile["music_attribute.json"]["Doublons"] += 1 checkFile["music_attribute.bin"]["Doublons"] += 1
checkFile["music_attribute.json"]["DoublonsList"].append(song.id) checkFile["music_attribute.bin"]["DoublonsList"].append(song.id)
if findDoubloninList(list=attributes, key="uniqueId", keyValue=song.uniqueId):
if song.id not in checkFile["musicinfo.bin"]["DoublonsList"]:
checkFile["music_attribute.bin"]["Doublons"] += 1
checkFile["music_attribute.bin"]["DoublonsList"].append(song.uniqueId)
# music_order.json # music_order.bin
if order is not None: if order is not None:
# Check for missing uniqueIds or id and uniqueId mismatches # Check for missing uniqueIds or id and uniqueId mismatches
orderOccurences = findAllObjects(list=order, key="id", keyValue=song.id) orderOccurences = findAllObjects(list=order, key="id", keyValue=song.id)
if len(orderOccurences) == 0: if len(orderOccurences) == 0:
checkFile["music_order.json"]["Missing"] += 1 checkFile["music_order.bin"]["Missing"] += 1
checkFile["music_order.json"]["MissingList"].append(song.id) checkFile["music_order.bin"]["MissingList"].append(song.id)
else: else:
songGenres = [] songGenres = []
for occurence in orderOccurences: for occurence in orderOccurences:
@ -306,9 +316,9 @@ for song in songs:
if not all( if not all(
[song.id == occurence["id"], song.uniqueId == occurence["uniqueId"]] [song.id == occurence["id"], song.uniqueId == occurence["uniqueId"]]
): ):
if song.id not in checkFile["music_order.json"]["MismatchList"]: if song.id not in checkFile["music_order.bin"]["MismatchList"]:
checkFile["music_order.json"]["Mismatch"] += 1 checkFile["music_order.bin"]["Mismatch"] += 1
checkFile["music_order.json"]["MismatchList"].append( checkFile["music_order.bin"]["MismatchList"].append(
{ {
"id": song.id, "id": song.id,
"ExpectedUniqueId": song.uniqueId, "ExpectedUniqueId": song.uniqueId,
@ -317,24 +327,24 @@ for song in songs:
) )
# Counting unique entries # Counting unique entries
checkFile["music_order.json"]["UniqueEntries"] += 1 checkFile["music_order.bin"]["UniqueEntries"] += 1
checkFile["music_order.json"]["UniqueEntriesList"].append( checkFile["music_order.bin"]["UniqueEntriesList"].append(
{ {
song.id: songGenres, song.id: songGenres,
} }
) )
# wordlist.json # wordlist.bin
if words is not None: if words is not None:
if song.name == "": if song.name == "":
checkFile["wordlist.json"]["MissingSongName"] += 1 checkFile["wordlist.bin"]["MissingSongName"] += 1
checkFile["wordlist.json"]["MissingSongNameList"].append(song.id) checkFile["wordlist.bin"]["MissingSongNameList"].append(song.id)
if song.sub == "": if song.sub == "":
checkFile["wordlist.json"]["MissingSongSub"] += 1 checkFile["wordlist.bin"]["MissingSongSub"] += 1
checkFile["wordlist.json"]["MissingSongSubList"].append(song.id) checkFile["wordlist.bin"]["MissingSongSubList"].append(song.id)
if song.detail == "": if song.detail == "":
checkFile["wordlist.json"]["MissingSongDetail"] += 1 checkFile["wordlist.bin"]["MissingSongDetail"] += 1
checkFile["wordlist.json"]["MissingSongDetailList"].append(song.id) checkFile["wordlist.bin"]["MissingSongDetailList"].append(song.id)
# Gamefiles # Gamefiles
if not doesPathExist("./Data/x64/sound/" + "song_" + song.id + ".nus3bank"): if not doesPathExist("./Data/x64/sound/" + "song_" + song.id + ".nus3bank"):
@ -345,11 +355,11 @@ for song in songs:
checkFile["GameFiles"]["MissingFumenList"].append(song.id) checkFile["GameFiles"]["MissingFumenList"].append(song.id)
# Checking for vacant uniqueIds # Checking for vacant uniqueIds
for i in range(max(checkFile["musicinfo.json"]["MaxId"], 1600)): for i in range(max(checkFile["musicinfo.bin"]["MaxId"], 1600)):
key = findKeyInList(list=infos, key="uniqueId", keyValue=i) key = findKeyInList(list=infos, key="uniqueId", keyValue=i)
if key is not None: if key is not None:
# Updating GenreNoList of music_order.json # Updating GenreNoList of music_order.bin
for song in findAllObjects( for song in findAllObjects(
list=order, key="uniqueId", keyValue=key["uniqueId"] list=order, key="uniqueId", keyValue=key["uniqueId"]
): ):
@ -360,7 +370,7 @@ for i in range(max(checkFile["musicinfo.json"]["MaxId"], 1600)):
} }
if ( if (
findKeyInList( findKeyInList(
list=checkFile["music_order.json"]["GenreNoList"], list=checkFile["music_order.bin"]["GenreNoList"],
key="GenreNo", key="GenreNo",
keyValue=song["genreNo"], keyValue=song["genreNo"],
) )
@ -369,26 +379,26 @@ for i in range(max(checkFile["musicinfo.json"]["MaxId"], 1600)):
genre["NumberofSongs"] = len( genre["NumberofSongs"] = len(
findAllObjects(list=order, key="genreNo", keyValue=song["genreNo"]) findAllObjects(list=order, key="genreNo", keyValue=song["genreNo"])
) )
checkFile["music_order.json"]["GenreNoList"].append(genre) checkFile["music_order.bin"]["GenreNoList"].append(genre)
else: else:
# Finding unused Ids bellow 1599 # Finding unused Ids bellow 1599
if i < 1600: if i < 1600:
checkFile["musicinfo.json"]["UnusedUniqueIds"] += 1 checkFile["musicinfo.bin"]["UnusedUniqueIds"] += 1
checkFile["musicinfo.json"]["UnusedUniqueIdsList"].append(i) checkFile["musicinfo.bin"]["UnusedUniqueIdsList"].append(i)
# Checking for doublons in wordlist # Checking for doublons in wordlist
if words is not None: if words is not None:
for word in words: for word in words:
if findDoubloninList(list=words, key="key", keyValue=word["key"]): if findDoubloninList(list=words, key="key", keyValue=word["key"]):
if word["key"] not in checkFile["wordlist.json"]["DoublonsList"]: if word["key"] not in checkFile["wordlist.bin"]["DoublonsList"]:
checkFile["wordlist.json"]["Doublons"] += 1 checkFile["wordlist.bin"]["Doublons"] += 1
checkFile["wordlist.json"]["DoublonsList"].append(word["key"]) checkFile["wordlist.bin"]["DoublonsList"].append(word["key"])
# Sorting some values for better readability # Sorting some values for better readability
checkFile["musicinfo.json"]["GenreNoList"].sort( checkFile["musicinfo.bin"]["GenreNoList"].sort(
key=lambda x: x["GenreNo"], reverse=False key=lambda x: x["GenreNo"], reverse=False
) )
checkFile["music_order.json"]["GenreNoList"].sort( checkFile["music_order.bin"]["GenreNoList"].sort(
key=lambda x: x["GenreNo"], reverse=False key=lambda x: x["GenreNo"], reverse=False
) )

21
encryption.py
View File

@ -1,4 +1,5 @@
import gzip import gzip
import json
import os import os
from pathlib import Path from pathlib import Path
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
@ -39,9 +40,14 @@ def decrypt_file(input_file, key_type: Keys = Keys(Keys.Datatable)):
# Read the IV from the first 16 bytes of the input file # Read the IV from the first 16 bytes of the input file
iv = read_iv_from_file(input_file) iv = read_iv_from_file(input_file)
try:
# Create an AES cipher object with CBC mode # Create an AES cipher object with CBC mode
cipher = Cipher(algorithms.AES(key), modes.CBC(iv), backend=default_backend()) cipher = Cipher(algorithms.AES(key), modes.CBC(iv), backend=default_backend())
decryptor = cipher.decryptor() decryptor = cipher.decryptor()
except Exception as error:
print(error)
print("You need to set the right AES keys in the encryption.py file")
exit(0)
with open(input_file, "rb") as infile: with open(input_file, "rb") as infile:
# Skip the IV in the input file # Skip the IV in the input file
@ -60,6 +66,14 @@ def decrypt_file(input_file, key_type: Keys = Keys(Keys.Datatable)):
return decompressed_data return decompressed_data
def isJson(file: bytes):
try:
json.loads(file)["items"]
return True
except:
return False
def encrypt_file(input_file, key_type: Keys = Keys(Keys.Datatable)): def encrypt_file(input_file, key_type: Keys = Keys(Keys.Datatable)):
# Convert the key from hex to bytes # Convert the key from hex to bytes
key = binascii.unhexlify(Keys(key_type.value).value) key = binascii.unhexlify(Keys(key_type.value).value)
@ -99,6 +113,13 @@ def save_file(file: bytes, outdir: str, encrypt: bool):
else encrypt_file(input_file=file, key_type=type) else encrypt_file(input_file=file, key_type=type)
) )
if isJson(fileContent):
base = os.path.splitext(outdir)[0]
outdir = base + ".json"
else:
base = os.path.splitext(outdir)[0]
outdir = base + ".bin"
print("Decrypting" if not encrypt else "Encrypting", file, "to", outdir) print("Decrypting" if not encrypt else "Encrypting", file, "to", outdir)
with open(outdir, "wb") as outfile: with open(outdir, "wb") as outfile:

15
helpers.py
View File

@ -6,7 +6,20 @@ from encryption import decrypt_file
def isChn(): def isChn():
return os.path.exists("./Data/x64/datatable/music_usbsetting.bin") try:
try:
# Trying to load file for 08.18
json.load(gzip.open("./Data/x64/datatable/musicinfo.bin", "rb"))["items"]
return False
except Exception as error:
# Trying to load file for 32.09 CHN
json.loads(decrypt_file(input_file="./Data/x64/datatable/musicinfo.bin"))[
"items"
]
return True
except Exception as error:
print(error)
print("Couldn't find musicinfo.bin, assuming 08.18")
def loadFile(path: str): def loadFile(path: str):