Merge branch 'main' into tests

README.md

@@ -28,20 +28,38 @@ class Keys(Enum):
 you also need to install the pip module `cryptography`:
 > pip install cryptography
 
-here's some examples :
+here's some usage examples :
 
 ```py
 # Display the help message
-py .\encryption.py --help
+py encryption.py --help
 
 # Decrypting a datatable :
-py .\encryption.py --input "data.bin" --output "data.json"
+py encryption.py --input "data.bin" --output "data.json"
 
 # Encrypting a datatable :
-py .\encryption.py --enc --input "data.json" --output "data.bin" 
+py encryption.py --enc --input "data.json" --output "data.bin" 
 
 # Encrypting a fumen for use in CHN :
-py .\encryption.py --enc --fumen --input "data_e.bin" --output "data_e.bin" 
+py encryption.py --enc --fumen --input "data_e.bin" --output "data_e.bin" 
+```
+
+## sortAlphabetically.py
+
+This script generates an alphabetically sorted music_order.bin file for a given language.
+Possible languages are : japaneseText, englishUsText, chineseTText, chineseSText and koreanText
+
+Here's some usage examples :
+
+```py
+# Display the help message
+py sortAlphabetically.py --help
+
+# Sort file by english name
+py sortAlphabetically.py --language "englishUsText"
+
+# Restore a backup of the original music_order file
+py sortAlphabetically.py --restore
 ```
 
 ## checkDatatables.py
@@ -58,6 +76,6 @@ This script generates a comprehensive list of various informations regarding you
 
 To run this one you simply need to call it like so:
 
-> py .\checkDatatables.py
+> py checkDatatables.py
 
 The output will be written in a file named `checks.json`

checkDatatables.py

@@ -4,84 +4,38 @@ from encryption import decrypt_file
 import json
 import os
 
-from helpers import doesPathExist, findAllObjects, findDoubloninList, findKeyInList
+from helpers import (
+    doesPathExist,
+    findAllObjects,
+    findDoubloninList,
+    findKeyInList,
+    isChn,
+    loadFile,
+)
 
 # "japaneseText"
 # "englishUsText"
 # "chineseTText"
 # "koreanText"
 # "chineseSText"
-language = "japaneseText"
+language = "englishUsText"
 
-isCHN = False
+isChn = isChn()
 
 # region Loading files
 checkFile = {}
 
-# Loading musicinfo.bin
+infos = loadFile(path="./Data/x64/datatable/musicinfo.bin")
-try:
+usb = loadFile(path="./Data/x64/datatable/music_usbsetting.bin")
-    infos = json.load(gzip.open("./Data/x64/datatable/musicinfo.bin", "rb"))["items"]
+order = loadFile(path="./Data/x64/datatable/music_order.bin")
-except:
+attributes = loadFile(path="./Data/x64/datatable/music_attribute.bin")
-    try:
+words = loadFile(path="./Data/x64/datatable/wordlist.bin")
-        infos = json.loads(
-            decrypt_file(input_file="./Data/x64/datatable/musicinfo.bin")
-        )["items"]
-        isCHN = True
-    except:
-        print("Couldn't load musicinfo.bin, exiting.")
-        exit(0)
-
-# Loading music_usbsetting.bin
-try:
-    usb = (
-        json.loads(
-            decrypt_file(input_file="./Data/x64/datatable/music_usbsetting.bin")
-        )["items"]
-        if isCHN
-        else None
-    )
-except:
-    usb = None
-
-# Loading music_order.bin
-try:
-    order = (
-        json.loads(decrypt_file(input_file="./Data/x64/datatable/music_order.bin"))[
-            "items"
-        ]
-        if isCHN
-        else json.load(gzip.open("./Data/x64/datatable/music_order.bin", "rb"))["items"]
-    )
-except:
-    order = None
-
-# Loading music_attribute.bin
-try:
-    attributes = (
-        json.loads(decrypt_file(input_file="./Data/x64/datatable/music_attribute.bin"))[
-            "items"
-        ]
-        if isCHN
-        else json.load(gzip.open("./Data/x64/datatable/music_attribute.bin", "rb"))[
-            "items"
-        ]
-    )
-except:
-    attributes = None
-
-# Loading wordlist.bin
-try:
-    words = (
-        json.loads(decrypt_file(input_file="./Data/x64/datatable/wordlist.bin"))[
-            "items"
-        ]
-        if isCHN
-        else json.load(gzip.open("./Data/x64/datatable/wordlist.bin", "rb"))["items"]
-    )
-except:
-    words = None
 # endregion
 
+# Forcing japanese language on 08.18 as this is what is usually used for omnimix.
+if isCHN:
+    language = "japaneseText"
+
 
 # region Classes And Methods
 class Genres(Enum):
@@ -92,9 +46,9 @@ class Genres(Enum):
     Vocaloid = 3
     GameMusic = 4
     NamcoOriginal = 5
-    Variety = 6 if not isCHN else 7
+    Variety = 6 if not isChn else 7
-    Classical = 7 if not isCHN else 8
+    Classical = 7 if not isChn else 8
-    if not isCHN:
+    if not isChn:
         Custom = 9
 
     @classmethod
@@ -102,6 +56,44 @@ class Genres(Enum):
         return cls.Unknown
 
 
+def findKeyInList(list: list, key: str, keyValue, value=None):
+    for object in list:
+        if object[key] == keyValue:
+            if value is not None:
+                return object[value]
+            else:
+                return object
+
+    if value is not None:
+        return ""
+    else:
+        return None
+
+
+def findAllObjects(list: list, key: str, keyValue):
+    templist = []
+    templist.append(list)
+    objects = []
+
+    for element in templist[0]:
+        if element[key] == keyValue:
+            objects.append(element)
+
+    return objects
+
+
+def findDoubloninList(list: list, key: str, keyValue):
+    if len(findAllObjects(list=list, key=key, keyValue=keyValue)) > 1:
+        return True
+    return False
+
+
+def doesPathExist(path: str):
+    if os.path.exists(path):
+        return True
+    return False
+
+
 def initCheckFile():
     global checkFile
     checkFile = {

helpers.py

@@ -1,5 +1,30 @@
+import gzip
+import json
 import os
 
+from encryption import decrypt_file
+
+
+def isChn():
+    return os.path.exists("./Data/x64/datatable/music_usbsetting.bin")
+
+
+def loadFile(path: str):
+    if doesPathExist(path):
+        try:
+            if not isChn():
+                # Loading files for 08.18
+                return json.load(gzip.open(path, "rb"))["items"]
+            else:
+                # Loading files for 32.09 CHN
+                return json.loads(decrypt_file(input_file=path))["items"]
+        except Exception as error:
+            print(error)
+            print("Couldn't load", path)
+            return None
+    else:
+        print(path, "doesn't exist")
+
 
 def findKeyInList(list: list, key: str, keyValue, value=None):
     for object in list:

sortAlphabetically.py

@@ -1,36 +1,30 @@
+from argparse import ArgumentParser
 import gzip
+from io import StringIO
 import json
+import os
 import shutil
 
-from helpers import doesPathExist, findAllObjects, findKeyInList
+from helpers import doesPathExist, isChn, findKeyInList, loadFile
 
-from encryption import decrypt_file
+from encryption import encrypt_file
 
-# "japaneseText"
+language = ""
-# "englishUsText"
-# "chineseTText"
-# "koreanText"
-# "chineseSText"
-language = "japaneseText"
-isChn = False
 
-# Loading musicinfo.bin, music_order.bin and wordlist.bin
+order = None
-try:
+words = None
-    info = json.load(gzip.open("./Data/x64/datatable/musicinfo.bin", "rb"))["items"]
+
-    order = json.load(gzip.open("./Data/x64/datatable/music_order.bin", "rb"))["items"]
+
-    words = json.load(gzip.open("./Data/x64/datatable/wordlist.bin", "rb"))["items"]
+def loadFiles():
-except:
+    global order, words
-    try:
+    # Loading music_order.bin and wordlist.bin
-        info = json.loads(
+    order = loadFile(path="./Data/x64/datatable/music_order.bin")
-            decrypt_file(input_file="./Data/x64/datatable/musicinfo.bin")
+    words = loadFile(path="./Data/x64/datatable/wordlist.bin")
-        )["items"]
+
-        order = json.loads(
+    if any([order == None, words == None]):
-            decrypt_file(input_file="./Data/x64/datatable/music_order.bin")
+        print(
-        )["items"]
+            "Couldn't load files. Consider restoring your music_order file using the --restore flag."
-        words = json.load(gzip.open("./Data/x64/datatable/wordlist.bin", "rb"))["items"]
+        )
-        isChn = True
-    except:
-        print("Couldn't load files, exiting.")
         exit(0)
 
 
@@ -58,9 +52,10 @@ class Entry:
         }
 
 
-# Adding all the existing songs in song_order in an array
+def sort(reverse: bool):
-entries = []
+    # Adding all the existing songs in song_order in an array
-for entry in order:
+    entries = []
+    for entry in order:
         name = findKeyInList(
             list=words,
             key="key",
@@ -81,55 +76,99 @@ for entry in order:
             )
         )
 
-# # Adding all the missing songs in song_order in an array
+    print("Sorting", str(len(entries)), "entries")
-# for entry in info:
-#     alreadyIn = False
-#     occurences = findAllObjects(list=order, key="id", keyValue=entry["id"])
-#     if len(occurences) > 0:
-#         for occurence in occurences:
-#             if entry["genreNo"] == occurence["genreNo"]:
-#                 alreadyIn = True
-#                 break
 
-#     if not alreadyIn:
+    # Sorting names alphabetically.
-#         name = findKeyInList(
+    if reverse:
-#             list=words,
+        print("Reversed sorting order!")
-#             key="key",
+    entries.sort(key=lambda x: x.name, reverse=reverse)
-#             keyValue="song_" + entry["id"],
-#             value=language,
-#         )
 
-#         if name == "":
+    # Backing up the original order file
-#             print(entry["id"] + " is missing a name")
+    if not doesPathExist(path="./Data/x64/datatable/music_order.bin.bak"):
-#             continue
-
-#         print("Adding " + entry["id"])
-#         entries.append(
-#             Entry(
-#                 name=name,
-#                 genreNo=entry["genreNo"],
-#                 id=entry["id"],
-#                 uniqueId=entry["uniqueId"],
-#                 closeDispType=0,
-#             )
-#         )
-
-# Sorting names alphabetically.
-entries.sort(key=lambda x: x.name, reverse=False)
-
-# Backing up the original order file
-if not doesPathExist(path="./Data/x64/datatable/music_order.bin.bak"):
         print("Backed up music_order")
-    dest = shutil.move(
+        shutil.move(
             "./Data/x64/datatable/music_order.bin",
             "./Data/x64/datatable/music_order.bin.bak",
         )
 
-file = []
+    file = {"items": []}
-for entry in entries:
+    for entry in entries:
-    file.append(entry.toJson())
+        file["items"].append(entry.toJson())
-# Writing song_order.bin
+    # Writing song_order.bin
-json_object = json.dumps(file, ensure_ascii=False, indent="\t")
+    json_object = json.dumps(file, ensure_ascii=False, indent="\t")
-if not isChn:
+    if not isChn():
-    with open("./Data/x64/datatable/music_order.bin", "w", encoding="utf8") as outfile:
+        # Saving compressed bin file for 08.18
-        outfile.write(gzip.compress(json_object))
+        with open("./Data/x64/datatable/music_order.bin", "wb") as outfile:
+            outfile.write(gzip.compress(bytes(json_object, encoding="utf-8")))
+    else:
+        # Saving encrypted compressed bin file for 32.09 CHN
+        # This is terrible but it works fine :))
+        with open(
+            "./Data/x64/datatable/music_order.json", "w", encoding="utf-8"
+        ) as outfile:
+            outfile.write(json_object)
+        encrypted_object = encrypt_file(
+            input_file="./Data/x64/datatable/music_order.json"
+        )
+        os.remove("./Data/x64/datatable/music_order.json")
+        with open("./Data/x64/datatable/music_order.bin", "wb") as outfile:
+            outfile.write(encrypted_object)
+
+
+def restore():
+    # Checking if we have a backup
+    if doesPathExist(path="./Data/x64/datatable/music_order.bin.bak"):
+        print("Restoring music_order")
+        # Removing current music_order file
+        if os.path.isfile("./Data/x64/datatable/music_order.bin"):
+            os.remove("./Data/x64/datatable/music_order.bin")
+        # Restoring backup
+        shutil.move(
+            "./Data/x64/datatable/music_order.bin.bak",
+            "./Data/x64/datatable/music_order.bin",
+        )
+    else:
+        print("There is no backup to restore.")
+
+
+if __name__ == "__main__":
+    parser = ArgumentParser()
+    parser.add_argument(
+        "--restore",
+        action="store_true",
+        default=False,
+        help="Use this flag to restore a backup of the original file",
+    )
+    parser.add_argument(
+        "--reverse",
+        action="store_true",
+        default=False,
+        help="Revert sorting order",
+    )
+    parser.add_argument(
+        "-l",
+        "--language",
+        default="englishUsText",
+        help="This sets the language used for sorting the files. Possible values are : japaneseText, englishUsText, chineseTText, chineseSText and koreanText",
+    )
+
+    args = parser.parse_args()
+    language = args.language
+
+    if language not in [
+        "japaneseText",
+        "englishUsText",
+        "chineseTText",
+        "chineseSText",
+        "koreanText",
+    ]:
+        print(
+            "Invalid language, Possible values are : japaneseText, englishUsText, chineseTText, chineseSText and koreanText"
+        )
+        exit(1)
+
+    if not args.restore:
+        loadFiles()
+        sort(args.reverse)
+    else:
+        restore()