DDR score import

utils/db/README.md

@@ -0,0 +1,36 @@
+# Database Utilities
+
+**Backup db.json before shrinking or importing**
+
+## Shrink DB
+
+### [trim_monkey_db.py](trim_monkey_db.py) 
+
+This deletes unused Gitadora and IIDX non-best scores, which can drastically reduce the size of db.json in a multiuser environment
+
+Example:
+`python utils\db\trim_monkey_db.py`
+
+## Score Import
+
+### [import_ddr_spice_automap.py](import_ddr_spice_automap.py)
+
+Instructions:
+
+1. Enable `EA Automap` and `EA Netdump` in spicecfg
+
+1. Boot the game on the source network to export
+
+1. Card in on the source profile to export (all the way to music select menu)
+
+1. Exit the game
+
+1. Disable `EA Automap` and `EA Netdump` in spicecfg
+
+1. Run the import script
+
+Example: `python utils\db\import_ddr_spice_automap.py --automap_xml automap_0.xml --version 19 --monkey_db db.json --ddr_id 12345678`
+
+- `--version` {19 for A20P or 20 for A3}
+
+- `--ddr_id` {destination profile in db.json}

utils/db/import_ddr_spice_automap.py

@@ -0,0 +1,140 @@
+import argparse
+import xml.etree.ElementTree as ET
+
+from tinydb import TinyDB, where
+from tinydb.middlewares import CachingMiddleware
+from tinydb.storages import JSONStorage
+
+
+def main(automap_xml, version, monkey_db, ddr_id):
+    storage = CachingMiddleware(JSONStorage)
+    storage.WRITE_CACHE_SIZE = 5000
+
+    db = TinyDB(
+        monkey_db,
+        indent=2,
+        encoding="utf-8",
+        ensure_ascii=False,
+        storage=storage,
+    )
+
+    ddr_id = int(ddr_id.replace("-", ""))
+
+    profile = db.table("ddr_profile").get(where("ddr_id") == ddr_id)
+    if profile == None:
+        raise SystemExit(f"ERROR: DDR profile {ddr_id} not in {monkey_db}")
+
+    game_version = 19
+    if profile["version"].get(str(game_version), None) == None:
+        raise SystemExit(
+            f"ERROR: DDR profile {ddr_id} version {game_version} not in {monkey_db}"
+        )
+
+    scores = []
+
+    with open(automap_xml, "rb") as fp:
+        automap_0 = fp.read().split(b"\n\n")
+
+        if version == 19:
+            playerdata = "playerdata"
+        else:
+            playerdata = "playerdata_2"
+
+        scores_xml = False
+        for xml in automap_0:
+            tree = ET.ElementTree(ET.fromstring(xml.decode(encoding="shift-jis")))
+            root = tree.getroot()
+            if scores_xml:
+                for music in root.findall(f"{playerdata}/music"):
+                    mcode = int(music.find("mcode").text)
+                    for difficulty, chart in enumerate(music.findall("note")):
+                        if int(chart.find("count").text) > 0:
+                            rank = int(chart.find("rank").text)
+                            clearkind = int(chart.find("clearkind").text)
+                            score = int(chart.find("score").text)
+                            scores.append([mcode, difficulty, rank, clearkind, score])
+                break
+            else:
+                try:
+                    if root.find(f"{playerdata}/data/mode").text == "userload":
+                        if len(root.find(f"{playerdata}/data/refid").text) == 16:
+                            scores_xml = True
+                except AttributeError:
+                    continue
+
+        total_count = len(scores)
+
+        if total_count == 0:
+            raise SystemExit("ERROR: No scores to import")
+
+        for s in scores:
+            mcode = s[0]
+            difficulty = s[1]
+            rank = s[2]
+            lamp = s[3]
+            score = s[4]
+            exscore = 0
+
+            print(
+                f"mcode: {mcode}, difficulty: {difficulty}, rank: {rank}, score: {score}, lamp: {lamp}"
+            )
+
+            best = db.table("ddr_scores_best").get(
+                (where("ddr_id") == ddr_id)
+                & (where("game_version") == game_version)
+                & (where("mcode") == mcode)
+                & (where("difficulty") == difficulty)
+            )
+            best = {} if best is None else best
+
+            best_score_data = {
+                "game_version": game_version,
+                "ddr_id": ddr_id,
+                "playstyle": 0 if difficulty < 5 else 1,
+                "mcode": mcode,
+                "difficulty": difficulty,
+                "rank": min(rank, best.get("rank", rank)),
+                "lamp": max(lamp, best.get("lamp", lamp)),
+                "score": max(score, best.get("score", score)),
+                "exscore": max(exscore, best.get("exscore", exscore)),
+            }
+
+            ghostid = db.table("ddr_scores").get(
+                (where("ddr_id") == ddr_id)
+                & (where("game_version") == game_version)
+                & (where("mcode") == mcode)
+                & (where("difficulty") == difficulty)
+                & (where("exscore") == best.get("exscore", exscore))
+            )
+            if ghostid:
+                best_score_data["ghostid"] = ghostid.doc_id
+            else:
+                best_score_data["ghostid"] = -1
+
+            db.table("ddr_scores_best").upsert(
+                best_score_data,
+                (where("ddr_id") == ddr_id)
+                & (where("game_version") == game_version)
+                & (where("mcode") == mcode)
+                & (where("difficulty") == difficulty),
+            )
+
+    db.close()
+    print()
+    print(f"{total_count} scores imported to DDR profile {ddr_id} in {monkey_db}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--automap_xml", help="Input xml file", required=True)
+    parser.add_argument(
+        "--version",
+        help="19 is A20P, 20 is A3",
+        default=19,
+        type=int,
+    )
+    parser.add_argument("--monkey_db", help="Output json file", required=True)
+    parser.add_argument("--ddr_id", help="12345678", required=True)
+    args = parser.parse_args()
+
+    main(args.automap_xml, args.version, args.monkey_db, args.ddr_id)

utils/db/trim_monkey_db.py

@@ -0,0 +1,36 @@
+import time
+from os import stat
+from shutil import copy
+
+from tinydb import TinyDB
+from tinydb.middlewares import CachingMiddleware
+from tinydb.storages import JSONStorage
+
+storage = CachingMiddleware(JSONStorage)
+storage.WRITE_CACHE_SIZE = 5000
+
+infile = "db.json"
+outfile = f"db_{round(time.time())}.json"
+
+copy(infile, outfile)
+
+db = TinyDB(
+    infile,
+    indent=2,
+    encoding="utf-8",
+    ensure_ascii=False,
+    storage=storage,
+)
+
+start_size = stat(infile).st_size
+
+# Non-best tables for GITADORA and IIDX are not used in game
+for table in ("guitarfreaks_scores", "drummania_scores", "iidx_scores"):
+    db.drop_table(table)
+    print("Dropped", table)
+
+db.close()
+
+end_size = stat(infile).st_size
+
+print(f"{infile} {round((start_size - end_size) / 1024 / 1024, 2)} MiB trimmed")