Cool_Tools/bemaniutils
1
0
Fork 0
mirror of synced 2024-12-19 09:45:53 +01:00
Code Releases Activity
d13eb62b30
bemaniutils/bemani/utils/read.py

4275 lines
158 KiB
Python
Raw Normal View History

Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# vim: set fileencoding=utf-8
Update scripts to remove incorrect bash invocation and make them execute from any directory.
2020-05-12 23:01:16 +02:00
Update to stricter type checking, get rid of a bunch of unnecessary ignore statements.
2021-05-31 20:07:03 +02:00
import csv
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
import argparse
import copy
import io
import jaconv # type: ignore
import json
import os
import struct
import xml.etree.ElementTree as ET
Fix error on startup caused by incorrect types and new SqlAlchemy.
2021-03-19 01:47:37 +01:00
from sqlalchemy.engine import CursorResult # type: ignore
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
from sqlalchemy.orm import sessionmaker # type: ignore
from sqlalchemy.sql import text # type: ignore
from sqlalchemy.exc import IntegrityError # type: ignore
from typing import Any, Dict, List, Optional, Tuple
Move PE operations into common library, fix 64-bit struct pointer dereferencing.
2021-08-15 02:41:37 +02:00
from bemani.common import GameConstants, VersionConstants, DBConstants, PEFile, Time
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
from bemani.format import ARC, IFS, IIDXChart, IIDXMusicDB
Make Config a real class instead of a Dict, for safer access and defaults.
2021-08-20 06:43:59 +02:00
from bemani.data import Config, Server, Song
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
from bemani.data.interfaces import APIProviderInterface
from bemani.data.api.music import GlobalMusicData
from bemani.data.api.game import GlobalGameData
from bemani.data.mysql.music import MusicData
from bemani.data.mysql.user import UserData
Unify config loading for remaining utilities that read the same server.yaml.
2021-08-20 06:43:36 +02:00
from bemani.utils.config import load_config
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
class ReadAPI(APIProviderInterface):
def __init__(self, server: str, token: str) -> None:
self.__server = server
self.__token = token
def get_all_servers(self) -> List[Server]:
return [
Server(
0,
Time.now(),
self.__server,
self.__token,
False,
False,
)
]
class ImportBase:
def __init__(
self,
Make Config a real class instead of a Dict, for safer access and defaults.
2021-08-20 06:43:59 +02:00
config: Config,
Switch to using an enum for GameConstants.
2021-08-19 21:21:22 +02:00
game: GameConstants,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
version: Optional[int],
no_combine: bool,
update: bool,
) -> None:
self.game = game
self.version = version
self.update = update
self.no_combine = no_combine
self.__config = config
Make Config a real class instead of a Dict, for safer access and defaults.
2021-08-20 06:43:59 +02:00
self.__engine = self.__config.database.engine
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
self.__sessionmanager = sessionmaker(self.__engine)
self.__conn = self.__engine.connect()
self.__session = self.__sessionmanager(bind=self.__conn)
self.__batch = False
def start_batch(self) -> None:
self.__batch = True
def finish_batch(self) -> None:
self.__session.commit()
self.__batch = False
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
def execute(
self, sql: str, params: Optional[Dict[str, Any]] = None
) -> CursorResult:
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if not self.__batch:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
raise Exception("Logic error, cannot execute outside of a batch!")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
Make Config a real class instead of a Dict, for safer access and defaults.
2021-08-20 06:43:59 +02:00
if self.__config.database.read_only:
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# See if this is an insert/update/delete
for write_statement in [
"insert into ",
"update ",
"delete from ",
]:
if write_statement in sql.lower():
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
raise Exception("Read-only mode is active!")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
return self.__session.execute(text(sql), params if params is not None else {})
def remote_music(self, server: str, token: str) -> GlobalMusicData:
api = ReadAPI(server, token)
user = UserData(self.__config, self.__session)
music = MusicData(self.__config, self.__session)
return GlobalMusicData(api, user, music)
def remote_game(self, server: str, token: str) -> GlobalGameData:
api = ReadAPI(server, token)
return GlobalGameData(api)
def get_next_music_id(self) -> int:
cursor = self.execute("SELECT MAX(id) AS next_id FROM `music`")
result = cursor.fetchone()
try:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
return result["next_id"] + 1
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
except TypeError:
# Nothing in DB
return 1
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
def get_music_id_for_song(
self, songid: int, chart: int, version: Optional[int] = None
) -> Optional[int]:
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if version is None:
# Normal lookup
if self.version is None:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
raise Exception(
"Cannot get music ID for song when operating on all versions!"
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
version = self.version
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
sql = "SELECT id FROM `music` WHERE songid = :songid AND chart = :chart AND game = :game AND version != :version"
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
else:
# Specific version lookup
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
sql = "SELECT id FROM `music` WHERE songid = :songid AND chart = :chart AND game = :game AND version = :version"
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
cursor = self.execute(
sql,
{
"songid": songid,
"chart": chart,
"game": self.game.value,
"version": version,
},
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if cursor.rowcount != 0:
result = cursor.fetchone()
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
return result["id"]
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
else:
return None
def get_music_id_for_song_data(
self,
title: Optional[str],
artist: Optional[str],
genre: Optional[str],
chart: int,
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
version: Optional[int] = None,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
) -> Optional[int]:
frags = []
if title is not None:
frags.append("name = :title")
if artist is not None:
frags.append("artist = :artist")
if genre is not None:
frags.append("genre = :genre")
frags.append("chart = :chart")
frags.append("game = :game")
if version is None:
# Normal lookup
if self.version is None:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
raise Exception(
"Cannot get music ID for song when operating on all versions!"
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
version = self.version
frags.append("version != :version")
else:
frags.append("version = :version")
sql = "SELECT id FROM `music` WHERE " + " AND ".join(frags)
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
cursor = self.execute(
sql,
{
"title": title,
"artist": artist,
"genre": genre,
"chart": chart,
"game": self.game.value,
"version": version,
},
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if cursor.rowcount != 0:
result = cursor.fetchone()
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
return result["id"]
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
else:
return None
def insert_music_id_for_song(
self,
musicid: int,
songid: int,
chart: int,
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
name: Optional[str] = None,
artist: Optional[str] = None,
genre: Optional[str] = None,
data: Optional[Dict[str, Any]] = None,
version: Optional[int] = None,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
) -> None:
version = version if version is not None else self.version
if version is None:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
raise Exception(
"Cannot get insert new song when operating on all versions!"
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if data is None:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
jsondata = "{}"
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
else:
jsondata = json.dumps(data)
try:
sql = (
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"INSERT INTO `music` (id, songid, chart, game, version, name, artist, genre, data) "
+ "VALUES (:id, :songid, :chart, :game, :version, :name, :artist, :genre, :data)"
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
)
self.execute(
sql,
{
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"id": musicid,
"songid": songid,
"chart": chart,
"game": self.game.value,
"version": version,
"name": name,
"artist": artist,
"genre": genre,
"data": jsondata,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
)
except IntegrityError:
if self.update:
print("Entry already existed, so updating information!")
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
self.update_metadata_for_song(
songid, chart, name, artist, genre, data, version
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
else:
print("Entry already existed, so skip creating a second one!")
def update_metadata_for_song(
self,
songid: int,
chart: int,
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
name: Optional[str] = None,
artist: Optional[str] = None,
genre: Optional[str] = None,
data: Optional[Dict[str, Any]] = None,
version: Optional[int] = None,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
) -> None:
if data is None:
jsondata = None
else:
jsondata = json.dumps(data)
version = version if version is not None else self.version
updates = []
if jsondata is not None:
updates.append("data = :data")
if name is not None:
updates.append("name = :name")
if artist is not None:
updates.append("artist = :artist")
if genre is not None:
updates.append("genre = :genre")
if len(updates) == 0:
return
Convert most of the format() string calls to f-strings using libcst. Exact commands run were: python3 -m libcst.tool codemod convert_format_to_fstring.ConvertFormatStringCommand . --no-format python3 setup.py build_ext --inplace
2020-01-07 22:29:07 +01:00
sql = f"UPDATE `music` SET {', '.join(updates)} WHERE songid = :songid AND chart = :chart AND game = :game"
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if version is not None:
sql = sql + " AND version = :version"
self.execute(
sql,
{
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"songid": songid,
"chart": chart,
"game": self.game.value,
"version": version,
"name": name,
"artist": artist,
"genre": genre,
"data": jsondata,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
)
def update_metadata_for_music_id(
self,
musicid: int,
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
name: Optional[str] = None,
artist: Optional[str] = None,
genre: Optional[str] = None,
data: Optional[Dict[str, Any]] = None,
version: Optional[int] = None,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
) -> None:
if data is None:
jsondata = None
else:
jsondata = json.dumps(data)
version = version if version is not None else self.version
updates = []
if jsondata is not None:
updates.append("data = :data")
if name is not None:
updates.append("name = :name")
if artist is not None:
updates.append("artist = :artist")
if genre is not None:
updates.append("genre = :genre")
if len(updates) == 0:
return
Convert most of the format() string calls to f-strings using libcst. Exact commands run were: python3 -m libcst.tool codemod convert_format_to_fstring.ConvertFormatStringCommand . --no-format python3 setup.py build_ext --inplace
2020-01-07 22:29:07 +01:00
sql = f"UPDATE `music` SET {', '.join(updates)} WHERE id = :musicid AND game = :game"
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if version is not None:
sql = sql + " AND version = :version"
self.execute(
sql,
{
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"musicid": musicid,
"game": self.game.value,
"version": version,
"name": name,
"artist": artist,
"genre": genre,
"data": jsondata,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
)
def insert_catalog_entry(
self,
cattype: str,
catid: int,
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
data: Optional[Dict[str, Any]] = None,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
) -> None:
if data is None:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
jsondata = "{}"
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
else:
jsondata = json.dumps(data)
try:
sql = (
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"INSERT INTO `catalog` (game, version, type, id, data) "
+ "VALUES (:game, :version, :type, :id, :data)"
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
)
self.execute(
sql,
{
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"id": catid,
"type": cattype,
"game": self.game.value,
"version": self.version,
"data": jsondata,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
)
except IntegrityError:
if self.update:
print("Entry already existed, so updating information!")
sql = (
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"UPDATE `catalog` SET data = :data WHERE "
+ "game = :game AND version = :version AND type = :type AND id = :id"
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
)
self.execute(
sql,
{
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"id": catid,
"type": cattype,
"game": self.game.value,
"version": self.version,
"data": jsondata,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
)
else:
print("Entry already existed, so skip creating a second one!")
def close(self) -> None:
"""
Close any open data connection.
"""
# Make sure we don't leak connections after finising insertion.
if self.__batch:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
raise Exception("Logic error, opened a batch without closing!")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if self.__session is not None:
self.__session.close()
if self.__conn is not None:
self.__conn.close()
self.__conn = None
if self.__engine is not None:
self.__engine.dispose()
self.__engine = None
class ImportPopn(ImportBase):
def __init__(
self,
Make Config a real class instead of a Dict, for safer access and defaults.
2021-08-20 06:43:59 +02:00
config: Config,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
version: str,
no_combine: bool,
update: bool,
) -> None:
actual_version = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"19": VersionConstants.POPN_MUSIC_TUNE_STREET,
"20": VersionConstants.POPN_MUSIC_FANTASIA,
"21": VersionConstants.POPN_MUSIC_SUNNY_PARK,
"22": VersionConstants.POPN_MUSIC_LAPISTORIA,
"23": VersionConstants.POPN_MUSIC_ECLALE,
"24": VersionConstants.POPN_MUSIC_USANEKO,
"25": VersionConstants.POPN_MUSIC_PEACE,
"26": VersionConstants.POPN_MUSIC_KAIMEI_RIDDLES,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
}.get(version, -1)
if actual_version == VersionConstants.POPN_MUSIC_TUNE_STREET:
# Pop'n 19 has extra charts for old play modes (challenge and enjoy mode).
# Cho challenge is analogous to regular mode in newer games, but Pop'n
# 19 doesn't have easy charts, just 5 button charts.
self.charts = [1, 2, 3, 4, 5, 6, 7, 8, 9]
elif actual_version >= VersionConstants.POPN_MUSIC_FANTASIA:
# Newer pop'n has charts for easy, normal, hyper, another
self.charts = [0, 1, 2, 3]
else:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
raise Exception(
"Unsupported Pop'n Music version, expected one of the following: 19, 20, 21, 22, 23, 24, 25, 26!"
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
super().__init__(
config, GameConstants.POPN_MUSIC, actual_version, no_combine, update
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
def scrape(self, infile: str) -> List[Dict[str, Any]]:
with open(infile, mode="rb") as myfile:
data = myfile.read()
myfile.close()
Move PE operations into common library, fix 64-bit struct pointer dereferencing.
2021-08-15 02:41:37 +02:00
pe = PEFile(data)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if self.version == VersionConstants.POPN_MUSIC_TUNE_STREET:
# Based on K39:J:A:A:2010122200
# Normal offset for music DB, size
offset = 0x1F68E8
step = 72
length = 1048
# Offset and step of file DB
file_offset = 0x2D6888
file_step = 24
# Standard lookups
genre_offset = 0
title_offset = 1
artist_offset = 2
comment_offset = 3
english_title_offset = -1
english_artist_offset = -1
extended_genre_offset = -1
charts_offset = 6
folder_offset = 7
# Offsets for normal chart difficulties
easy_offset = 12
normal_offset = 10
hyper_offset = 11
ex_offset = 13
# Offsets for battle chart difficulties
battle_normal_offset = 14
battle_hyper_offset = 15
# Offsets into which offset to seek to for file lookups
easy_file_offset = 18
normal_file_offset = 16
hyper_file_offset = 17
ex_file_offset = 19
battle_normal_file_offset = 20
battle_hyper_file_offset = 21
packedfmt = (
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"<"
"I" # Genre
"I" # Title
"I" # Artist
"I" # Comment
"H" # ??
"H" # ??
"I" # Available charts mask
"I" # Folder
"I" # Event flags?
"B" # Event flags?
"B" # Normal difficulty
"B" # Hyper difficulty
"B" # Easy difficulty
"B" # EX difficulty
"B" # Battle normal difficulty
"B" # Battle hyper difficulty
"x" # ??
"x" # ??
"x" # ??
"H" # Normal chart pointer
"H" # Hyper chart pointer
"H" # Easy chart pointer
"H" # EX chart pointer
"H" # Battle normal pointer
"H" # Battle hyper pointer
"xxxxxxxxxxxxxxxxxx"
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
)
# Offsets into file DB for finding file and folder.
file_folder_offset = 0
file_name_offset = 1
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
filefmt = "<" "I" "I" "I" "I" "I" "I" # Folder # Filename
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Decoding function for chart masks
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
def available_charts(
mask: int,
) -> Tuple[bool, bool, bool, bool, bool, bool]:
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
return (
True, # Always an easy chart
True, # Always a normal chart
mask & 0x1000000 > 0, # Hyper chart bit
mask & 0x2000000 > 0, # Ex chart bit
True, # Always a battle normal chart
mask & 0x4000000 > 0, # Battle hyper chart bit
)
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
elif self.version == VersionConstants.POPN_MUSIC_FANTASIA:
# Based on L39:J:A:A:2012091900
# Normal offset for music DB, size
offset = 0x1AE240
step = 160
length = 1122
# Offset and step of file DB
file_offset = 0x273768
file_step = 24
# Standard lookups
genre_offset = 0
title_offset = 1
artist_offset = 2
comment_offset = 3
english_title_offset = -1
english_artist_offset = -1
extended_genre_offset = -1
charts_offset = 6
folder_offset = 7
# Offsets for normal chart difficulties
easy_offset = 12
normal_offset = 10
hyper_offset = 11
ex_offset = 13
# Offsets for battle chart difficulties
battle_normal_offset = 14
battle_hyper_offset = 15
# Offsets into which offset to seek to for file lookups
easy_file_offset = 18
normal_file_offset = 16
hyper_file_offset = 17
ex_file_offset = 19
battle_normal_file_offset = 20
battle_hyper_file_offset = 21
packedfmt = (
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"<"
"I" # Genre
"I" # Title
"I" # Artist
"I" # Comment
"H" # ??
"H" # ??
"I" # Available charts mask
"I" # Folder
"I" # Event flags?
"B" # Event flags?
"B" # Normal difficulty
"B" # Hyper difficulty
"B" # Easy difficulty
"B" # EX difficulty
"B" # Battle normal difficulty
"B" # Battle hyper difficulty
"x" # ??
"x" # ??
"x" # ??
"H" # Normal chart pointer
"H" # Hyper chart pointer
"H" # Easy chart pointer
"H" # EX chart pointer
"H" # Battle normal pointer
"H" # Battle hyper pointer
"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
)
# Offsets into file DB for finding file and folder.
file_folder_offset = 0
file_name_offset = 1
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
filefmt = "<" "I" "I" "I" "I" "I" "I" # Folder # Filename
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Decoding function for chart masks
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
def available_charts(
mask: int,
) -> Tuple[bool, bool, bool, bool, bool, bool]:
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
return (
True, # Always an easy chart
True, # Always a normal chart
mask & 0x1000000 > 0, # Hyper chart bit
mask & 0x2000000 > 0, # Ex chart bit
True, # Always a battle normal chart
mask & 0x4000000 > 0, # Battle hyper chart bit
)
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
elif self.version == VersionConstants.POPN_MUSIC_SUNNY_PARK:
# Based on M39:J:A:A:2014061900
# Normal offset for music DB, size
offset = 0x1FB640
step = 164
length = 1280
# Offset and step of file DB
file_offset = 0x2E0D20
file_step = 28
# Standard lookups
genre_offset = 0
title_offset = 1
artist_offset = 2
comment_offset = 3
english_title_offset = 4
english_artist_offset = 5
extended_genre_offset = 6
charts_offset = 9
folder_offset = 10
# Offsets for normal chart difficulties
easy_offset = 13
normal_offset = 14
hyper_offset = 15
ex_offset = 16
# Offsets for battle chart difficulties
battle_normal_offset = 17
battle_hyper_offset = 18
# Offsets into which offset to seek to for file lookups
easy_file_offset = 19
normal_file_offset = 20
hyper_file_offset = 21
ex_file_offset = 22
battle_normal_file_offset = 21
battle_hyper_file_offset = 22
packedfmt = (
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"<"
"I" # Genre
"I" # Title
"I" # Artist
"I" # Comment
"I" # English Title
"I" # English Artist
"I" # Extended genre?
"H" # ??
"H" # ??
"I" # Available charts mask
"I" # Folder
"I" # Event unlocks?
"H" # Event unlocks?
"B" # Easy difficulty
"B" # Normal difficulty
"B" # Hyper difficulty
"B" # EX difficulty
"B" # Battle normal difficulty
"B" # Battle hyper difficulty
"H" # Easy chart pointer
"H" # Normal chart pointer
"H" # Hyper chart pointer
"H" # EX chart pointer
"H" # Battle normal pointer
"H" # Battle hyper pointer
"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
)
# Offsets into file DB for finding file and folder.
file_folder_offset = 0
file_name_offset = 1
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
filefmt = "<" "I" "I" "I" "I" "I" "I" "I" # Folder # Filename
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Decoding function for chart masks
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
def available_charts(
mask: int,
) -> Tuple[bool, bool, bool, bool, bool, bool]:
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
return (
mask & 0x0080000 > 0, # Easy chart bit
True, # Always a normal chart
mask & 0x1000000 > 0, # Hyper chart bit
mask & 0x2000000 > 0, # Ex chart bit
True, # Always a battle normal chart
mask & 0x4000000 > 0, # Battle hyper chart bit
)
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
elif self.version == VersionConstants.POPN_MUSIC_LAPISTORIA:
# Based on M39:J:A:A:2015081900
# Normal offset for music DB, size
offset = 0x3124B0
step = 160
length = 1423
# Offset and step of file DB
file_offset = 0x472130
file_step = 28
# Standard lookups
genre_offset = 0
title_offset = 1
artist_offset = 2
comment_offset = 3
english_title_offset = 4
english_artist_offset = 5
extended_genre_offset = -1
charts_offset = 8
folder_offset = 9
# Offsets for normal chart difficulties
easy_offset = 12
normal_offset = 13
hyper_offset = 14
ex_offset = 15
# Offsets for battle chart difficulties
battle_normal_offset = 16
battle_hyper_offset = 17
# Offsets into which offset to seek to for file lookups
easy_file_offset = 18
normal_file_offset = 19
hyper_file_offset = 20
ex_file_offset = 21
battle_normal_file_offset = 22
battle_hyper_file_offset = 23
packedfmt = (
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"<"
"I" # Genre
"I" # Title
"I" # Artist
"I" # Comment
"I" # English Title
"I" # English Artist
"H" # ??
"H" # ??
"I" # Available charts mask
"I" # Folder
"I" # Event unlocks?
"H" # Event unlocks?
"B" # Easy difficulty
"B" # Normal difficulty
"B" # Hyper difficulty
"B" # EX difficulty
"B" # Battle normal difficulty
"B" # Battle hyper difficulty
"H" # Easy chart pointer
"H" # Normal chart pointer
"H" # Hyper chart pointer
"H" # EX chart pointer
"H" # Battle normal pointer
"H" # Battle hyper pointer
"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
)
# Offsets into file DB for finding file and folder.
file_folder_offset = 0
file_name_offset = 1
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
filefmt = "<" "I" "I" "I" "I" "I" "I" "I" # Folder # Filename
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Decoding function for chart masks
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
def available_charts(
mask: int,
) -> Tuple[bool, bool, bool, bool, bool, bool]:
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
return (
mask & 0x0080000 > 0, # Easy chart bit
True, # Always a normal chart
mask & 0x1000000 > 0, # Hyper chart bit
mask & 0x2000000 > 0, # Ex chart bit
True, # Always a battle normal chart
mask & 0x4000000 > 0, # Battle hyper chart bit
)
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
elif self.version == VersionConstants.POPN_MUSIC_ECLALE:
# Based on M39:J:A:A:2016100500
# Normal offset for music DB, size
offset = 0x2DE5C8
step = 160
length = 1551
# Offset and step of file DB
file_offset = 0x2D1948
file_step = 32
# Standard lookups
genre_offset = 0
title_offset = 1
artist_offset = 2
comment_offset = 3
english_title_offset = 4
english_artist_offset = 5
extended_genre_offset = -1
charts_offset = 8
folder_offset = 9
# Offsets for normal chart difficulties
easy_offset = 12
normal_offset = 13
hyper_offset = 14
ex_offset = 15
# Offsets for battle chart difficulties
battle_normal_offset = 16
battle_hyper_offset = 17
# Offsets into which offset to seek to for file lookups
easy_file_offset = 18
normal_file_offset = 19
hyper_file_offset = 20
ex_file_offset = 21
battle_normal_file_offset = 22
battle_hyper_file_offset = 23
packedfmt = (
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"<"
"I" # Genre
"I" # Title
"I" # Artist
"I" # Comment
"I" # English Title
"I" # English Artist
"H" # ??
"H" # ??
"I" # Available charts mask
"I" # Folder
"I" # Event unlocks?
"H" # Event unlocks?
"B" # Easy difficulty
"B" # Normal difficulty
"B" # Hyper difficulty
"B" # EX difficulty
"B" # Battle normal difficulty
"B" # Battle hyper difficulty
"H" # Easy chart pointer
"H" # Normal chart pointer
"H" # Hyper chart pointer
"H" # EX chart pointer
"H" # Battle normal pointer
"H" # Battle hyper pointer
"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
)
# Offsets into file DB for finding file and folder.
file_folder_offset = 0
file_name_offset = 1
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
filefmt = "<" "I" "I" "I" "I" "I" "I" "I" "I" # Folder # Filename
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Decoding function for chart masks
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
def available_charts(
mask: int,
) -> Tuple[bool, bool, bool, bool, bool, bool]:
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
return (
mask & 0x0080000 > 0, # Easy chart bit
True, # Always a normal chart
mask & 0x1000000 > 0, # Hyper chart bit
mask & 0x2000000 > 0, # Ex chart bit
True, # Always a battle normal chart
mask & 0x4000000 > 0, # Battle hyper chart bit
)
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
elif self.version == VersionConstants.POPN_MUSIC_USANEKO:
# Based on M39:J:A:A:2018101500
# Normal offset for music DB, size
offset = 0x299210
step = 172
length = 1704
# Offset and step of file DB
file_offset = 0x28AF08
file_step = 32
# Standard lookups
genre_offset = 0
title_offset = 1
artist_offset = 2
comment_offset = 3
english_title_offset = 4
english_artist_offset = 5
extended_genre_offset = -1
charts_offset = 8
folder_offset = 9
# Offsets for normal chart difficulties
easy_offset = 12
normal_offset = 13
hyper_offset = 14
ex_offset = 15
# Offsets for battle chart difficulties
battle_normal_offset = 16
battle_hyper_offset = 17
# Offsets into which offset to seek to for file lookups
easy_file_offset = 18
normal_file_offset = 19
hyper_file_offset = 20
ex_file_offset = 21
battle_normal_file_offset = 22
battle_hyper_file_offset = 23
Better Pop'n Music Peace support
2021-09-04 18:21:27 +02:00
packedfmt = (
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"<"
"I" # Genre
"I" # Title
"I" # Artist
"I" # Comment
"I" # English Title
"I" # English Artist
"H" # ??
"H" # ??
"I" # Available charts mask
"I" # Folder
"I" # Event unlocks?
"I" # Event unlocks?
"B" # Easy difficulty
"B" # Normal difficulty
"B" # Hyper difficulty
"B" # EX difficulty
"B" # Battle normal difficulty
"B" # Battle hyper difficulty
"xx" # Unknown pointer
"H" # Easy chart pointer
"H" # Normal chart pointer
"H" # Hyper chart pointer
"H" # EX chart pointer
"H" # Battle normal pointer
"H" # Battle hyper pointer
"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
Better Pop'n Music Peace support
2021-09-04 18:21:27 +02:00
)
# Offsets into file DB for finding file and folder.
file_folder_offset = 0
file_name_offset = 1
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
filefmt = "<" "I" "I" "I" "I" "I" "I" "I" "I" # Folder # Filename
Better Pop'n Music Peace support
2021-09-04 18:21:27 +02:00
# Decoding function for chart masks
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
def available_charts(
mask: int,
) -> Tuple[bool, bool, bool, bool, bool, bool]:
Better Pop'n Music Peace support
2021-09-04 18:21:27 +02:00
return (
mask & 0x0080000 > 0, # Easy chart bit
True, # Always a normal chart
mask & 0x1000000 > 0, # Hyper chart bit
mask & 0x2000000 > 0, # Ex chart bit
True, # Always a battle normal chart
mask & 0x4000000 > 0, # Battle hyper chart bit
)
Fix user name check for MGA in frontend Use "for key, value in" when reading usaneko phases Lint fixes
2021-09-06 02:16:06 +02:00
Better Pop'n Music Peace support
2021-09-04 18:21:27 +02:00
elif self.version == VersionConstants.POPN_MUSIC_PEACE:
# Based on M39:J:A:A:2020092800
# Normal offset for music DB, size
offset = 0x2C7C78
step = 172
length = 1877
# Offset and step of file DB
file_offset = 0x2B8010
file_step = 32
# Standard lookups
genre_offset = 0
title_offset = 1
artist_offset = 2
comment_offset = 3
Add Pnm Kaimei riddles support
2022-09-19 00:36:42 +02:00
english_title_offset = 4
english_artist_offset = 5
extended_genre_offset = -1
charts_offset = 8
folder_offset = 9
# Offsets for normal chart difficulties
easy_offset = 12
normal_offset = 13
hyper_offset = 14
ex_offset = 15
# Offsets for battle chart difficulties
battle_normal_offset = 16
battle_hyper_offset = 17
# Offsets into which offset to seek to for file lookups
easy_file_offset = 18
normal_file_offset = 19
hyper_file_offset = 20
ex_file_offset = 21
battle_normal_file_offset = 22
battle_hyper_file_offset = 23
packedfmt = (
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"<"
"I" # Genre
"I" # Title
"I" # Artist
"I" # Comment
"I" # English Title
"I" # English Artist
"H" # ??
"H" # ??
"I" # Available charts mask
"I" # Folder
"I" # Event unlocks?
"I" # Event unlocks?
"B" # Easy difficulty
"B" # Normal difficulty
"B" # Hyper difficulty
"B" # EX difficulty
"B" # Battle normal difficulty
"B" # Battle hyper difficulty
"xx" # Unknown pointer
"H" # Easy chart pointer
"H" # Normal chart pointer
"H" # Hyper chart pointer
"H" # EX chart pointer
"H" # Battle normal pointer
"H" # Battle hyper pointer
"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
Add Pnm Kaimei riddles support
2022-09-19 00:36:42 +02:00
)
# Offsets into file DB for finding file and folder.
file_folder_offset = 0
file_name_offset = 1
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
filefmt = "<" "I" "I" "I" "I" "I" "I" "I" "I" # Folder # Filename
Add Pnm Kaimei riddles support
2022-09-19 00:36:42 +02:00
# Decoding function for chart masks
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
def available_charts(
mask: int,
) -> Tuple[bool, bool, bool, bool, bool, bool]:
Add Pnm Kaimei riddles support
2022-09-19 00:36:42 +02:00
return (
mask & 0x0080000 > 0, # Easy chart bit
True, # Always a normal chart
mask & 0x1000000 > 0, # Hyper chart bit
mask & 0x2000000 > 0, # Ex chart bit
True, # Always a battle normal chart
mask & 0x4000000 > 0, # Battle hyper chart bit
)
elif self.version == VersionConstants.POPN_MUSIC_KAIMEI_RIDDLES:
# Based on M39:J:A:A:2022061300
# Normal offset for music DB, size
offset = 0x2DEA68
step = 172
length = 2019
# Offset and step of file DB
file_offset = 0x2CDB00
file_step = 32
# Standard lookups
genre_offset = 0
title_offset = 1
artist_offset = 2
comment_offset = 3
Better Pop'n Music Peace support
2021-09-04 18:21:27 +02:00
english_title_offset = 4
english_artist_offset = 5
extended_genre_offset = -1
charts_offset = 8
folder_offset = 9
# Offsets for normal chart difficulties
easy_offset = 12
normal_offset = 13
hyper_offset = 14
ex_offset = 15
# Offsets for battle chart difficulties
battle_normal_offset = 16
battle_hyper_offset = 17
# Offsets into which offset to seek to for file lookups
easy_file_offset = 18
normal_file_offset = 19
hyper_file_offset = 20
ex_file_offset = 21
battle_normal_file_offset = 22
battle_hyper_file_offset = 23
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
packedfmt = (
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"<"
"I" # Genre
"I" # Title
"I" # Artist
"I" # Comment
"I" # English Title
"I" # English Artist
"H" # ??
"H" # ??
"I" # Available charts mask
"I" # Folder
"I" # Event unlocks?
"I" # Event unlocks?
"B" # Easy difficulty
"B" # Normal difficulty
"B" # Hyper difficulty
"B" # EX difficulty
"B" # Battle normal difficulty
"B" # Battle hyper difficulty
"xx" # Unknown pointer
"H" # Easy chart pointer
"H" # Normal chart pointer
"H" # Hyper chart pointer
"H" # EX chart pointer
"H" # Battle normal pointer
"H" # Battle hyper pointer
"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
)
# Offsets into file DB for finding file and folder.
file_folder_offset = 0
file_name_offset = 1
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
filefmt = "<" "I" "I" "I" "I" "I" "I" "I" "I" # Folder # Filename
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Decoding function for chart masks
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
def available_charts(
mask: int,
) -> Tuple[bool, bool, bool, bool, bool, bool]:
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
return (
mask & 0x0080000 > 0, # Easy chart bit
True, # Always a normal chart
mask & 0x1000000 > 0, # Hyper chart bit
mask & 0x2000000 > 0, # Ex chart bit
True, # Always a battle normal chart
mask & 0x4000000 > 0, # Battle hyper chart bit
)
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
else:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
raise Exception(f"Unsupported version {self.version}")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
def read_string(offset: int) -> str:
# First, translate load offset in memory to disk offset
Move PE operations into common library, fix 64-bit struct pointer dereferencing.
2021-08-15 02:41:37 +02:00
offset = pe.virtual_to_physical(offset)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Now, grab bytes until we're null-terminated
bytestring = []
while data[offset] != 0:
bytestring.append(data[offset])
offset = offset + 1
# Its shift-jis encoded, so decode it now
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
return bytes(bytestring).decode("shift_jisx0213")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
def file_chunk(offset: int) -> Tuple[Any, ...]:
fileoffset = file_offset + (file_step * offset)
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
filedata = data[fileoffset : (fileoffset + file_step)]
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
return struct.unpack(filefmt, filedata)
def file_handle(offset: int) -> str:
chunk = file_chunk(offset)
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
return (
read_string(chunk[file_folder_offset])
+ "/"
+ read_string(chunk[file_name_offset])
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
songs = []
for songid in range(length):
chunkoffset = offset + (step * songid)
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
chunkdata = data[chunkoffset : (chunkoffset + step)]
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
unpacked = struct.unpack(packedfmt, chunkdata)
valid_charts = available_charts(unpacked[charts_offset])
songinfo = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"id": songid,
"title": read_string(unpacked[title_offset]),
"artist": read_string(unpacked[artist_offset]),
"genre": read_string(unpacked[genre_offset]),
"comment": read_string(unpacked[comment_offset]),
"title_en": read_string(unpacked[english_title_offset])
if english_title_offset > 0
else "",
"artist_en": read_string(unpacked[english_artist_offset])
if english_artist_offset > 0
else "",
"long_genre": read_string(unpacked[extended_genre_offset])
if extended_genre_offset > 0
else "",
"folder": unpacked[folder_offset],
"difficulty": {
"standard": {
"easy": unpacked[easy_offset] if valid_charts[0] else 0,
"normal": unpacked[normal_offset] if valid_charts[1] else 0,
"hyper": unpacked[hyper_offset] if valid_charts[2] else 0,
"ex": unpacked[ex_offset] if valid_charts[3] else 0,
},
"battle": {
"normal": unpacked[battle_normal_offset]
if valid_charts[4]
else 0,
"hyper": unpacked[battle_hyper_offset]
if valid_charts[5]
else 0,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
},
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"file": {
"standard": {
"easy": file_handle(unpacked[easy_file_offset])
if valid_charts[0]
else "",
"normal": file_handle(unpacked[normal_file_offset])
if valid_charts[1]
else "",
"hyper": file_handle(unpacked[hyper_file_offset])
if valid_charts[2]
else "",
"ex": file_handle(unpacked[ex_file_offset])
if valid_charts[3]
else "",
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"battle": {
"normal": file_handle(unpacked[battle_normal_file_offset])
if valid_charts[4]
else "",
"hyper": file_handle(unpacked[battle_hyper_file_offset])
if valid_charts[5]
else "",
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
},
}
if (
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
songinfo["title"] in ["-", "‐"]
and songinfo["genre"] in ["-", "‐"]
and songinfo["artist"] in ["-", "‐"]
and songinfo["comment"] in ["-", "‐"]
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
):
# This is a removed song
continue
if (
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
songinfo["title"] == "ＤＵＭＭＹ"
and songinfo["artist"] == "ＤＵＭＭＹ"
and songinfo["genre"] == "ＤＵＭＭＹ"
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
):
# This is a song the intern left in
continue
# Fix accent issues with title/artist
accent_lut: Dict[str, str] = {
"鵝": "7",
"圄": "à",
"圉": "ä",
"鵤": "Ä",
"鵑": "👁",
"鶤": "©",
"圈": "é",
"鵐": "ê",
"鵙": "Ə",
"鵲": "ë",
"！": "!",
"囿": "♥",
"鶚": "㊙",
"鶉": "ó",
"鶇": "ö",
"鶲": "Ⓟ",
"鶫": "²",
"圍": "@",
"圖": "ţ",
"鵺": "Ü",
"囎": ":",
"囂": "♡",
"釁": "🐾",
}
for orig, rep in accent_lut.items():
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
songinfo["title"] = songinfo["title"].replace(orig, rep)
songinfo["artist"] = songinfo["artist"].replace(orig, rep)
songinfo["title_en"] = songinfo["title_en"].replace(orig, rep)
songinfo["artist_en"] = songinfo["artist_en"].replace(orig, rep)
songinfo["genre"] = songinfo["genre"].replace(orig, rep)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
songs.append(songinfo)
return songs
def lookup(self, server: str, token: str) -> List[Dict[str, Any]]:
# Grab music info from remote server
music = self.remote_music(server, token)
songs = music.get_all_songs(self.game, self.version)
lut: Dict[int, Dict[str, Any]] = {}
chart_map = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
0: "easy",
1: "normal",
2: "hyper",
3: "ex",
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
}
# Format it the way we expect
for song in songs:
if song.chart not in chart_map:
# Ignore charts on songs we don't support/care about.
continue
if song.id not in lut:
lut[song.id] = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"id": song.id,
"title": song.name,
"artist": song.artist,
"genre": song.genre,
"comment": "",
"title_en": "",
"artist_en": "",
"long_genre": "",
"folder": song.data.get_str("category"),
"difficulty": {
"standard": {
"easy": 0,
"normal": 0,
"hyper": 0,
"ex": 0,
},
"battle": {
"normal": 0,
"hyper": 0,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
},
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"file": {
"standard": {
"easy": "",
"normal": "",
"hyper": "",
"ex": "",
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"battle": {
"normal": "",
"hyper": "",
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
},
}
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
lut[song.id]["difficulty"]["standard"][
chart_map[song.chart]
] = song.data.get_int("difficulty")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Return the reassembled data
return [val for _, val in lut.items()]
def import_music_db(self, songs: List[Dict[str, Any]]) -> None:
chart_map = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
0: "easy",
1: "normal",
2: "hyper",
3: "ex",
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
}
for song in songs:
self.start_batch()
for chart in self.charts:
# First, try to find in the DB from another version
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
old_id = self.get_music_id_for_song(song["id"], chart)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Now, look up metadata
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
title = song["title_en"] if len(song["title_en"]) > 0 else song["title"]
artist = (
song["artist_en"] if len(song["artist_en"]) > 0 else song["artist"]
)
genre = song["genre"]
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# We only care about easy/normal/hyper/ex, so only provide mappings there
if chart in chart_map:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
difficulty = song["difficulty"]["standard"][chart_map[chart]]
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
else:
difficulty = 0
if self.no_combine or old_id is None:
# Insert original
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
print(
f"New entry for {artist} {title} ({song['id']} chart {chart})"
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
next_id = self.get_next_music_id()
else:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
print(
f"Reused entry for {artist} {title} ({song['id']} chart {chart})"
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
next_id = old_id
self.insert_music_id_for_song(
next_id,
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
song["id"],
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
chart,
title,
artist,
genre,
{
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"category": str(song["folder"]),
"difficulty": difficulty,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
)
self.finish_batch()
class ImportJubeat(ImportBase):
def __init__(
self,
Make Config a real class instead of a Dict, for safer access and defaults.
2021-08-20 06:43:59 +02:00
config: Config,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
version: str,
no_combine: bool,
update: bool,
) -> None:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
if version in ["saucer", "saucer-fulfill", "prop", "qubell", "clan", "festo"]:
update read script
2022-08-17 01:44:34 +02:00
actual_version = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"saucer": VersionConstants.JUBEAT_SAUCER,
"saucer-fulfill": VersionConstants.JUBEAT_SAUCER_FULFILL,
"prop": VersionConstants.JUBEAT_PROP,
"qubell": VersionConstants.JUBEAT_QUBELL,
"clan": VersionConstants.JUBEAT_CLAN,
"festo": VersionConstants.JUBEAT_FESTO,
update read script
2022-08-17 01:44:34 +02:00
}.get(version, -1)
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
elif version in ["omni-prop", "omni-qubell", "omni-clan", "omni-festo"]:
update read script
2022-08-17 01:44:34 +02:00
actual_version = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"omni-prop": VersionConstants.JUBEAT_PROP,
"omni-qubell": VersionConstants.JUBEAT_QUBELL,
"omni-clan": VersionConstants.JUBEAT_CLAN,
"omni-festo": VersionConstants.JUBEAT_FESTO,
update read script
2022-08-17 01:44:34 +02:00
}.get(version, -1) + DBConstants.OMNIMIX_VERSION_BUMP
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
elif version == "all":
update read script
2022-08-17 01:44:34 +02:00
actual_version = None
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if actual_version in [
None,
update read script
2022-08-17 01:44:34 +02:00
VersionConstants.JUBEAT_FESTO,
VersionConstants.JUBEAT_FESTO + DBConstants.OMNIMIX_VERSION_BUMP,
]:
# jubeat festo adds in separation of normal and hard mode scores.
# This adds a duplicate of each chart so that we show separated scores.
self.charts = [0, 1, 2, 3, 4, 5]
elif actual_version in [
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
VersionConstants.JUBEAT_SAUCER,
VersionConstants.JUBEAT_SAUCER_FULFILL,
VersionConstants.JUBEAT_PROP,
VersionConstants.JUBEAT_QUBELL,
VersionConstants.JUBEAT_CLAN,
update read script
2022-08-17 01:44:34 +02:00
VersionConstants.JUBEAT_PROP + DBConstants.OMNIMIX_VERSION_BUMP,
VersionConstants.JUBEAT_QUBELL + DBConstants.OMNIMIX_VERSION_BUMP,
VersionConstants.JUBEAT_CLAN + DBConstants.OMNIMIX_VERSION_BUMP,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
]:
self.charts = [0, 1, 2]
update read script
2022-08-17 01:44:34 +02:00
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
else:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
raise Exception(
"Unsupported Jubeat version, expected one of the following: saucer, saucer-fulfill, prop, omni-prop, qubell, omni-qubell, clan, omni-clan, festo, omni-festo!"
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
super().__init__(
config, GameConstants.JUBEAT, actual_version, no_combine, update
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
Teach network how to import/export Jubeat emblem catalog.
2020-04-24 21:20:27 +02:00
def scrape(self, xmlfile: str) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if self.version is None:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
raise Exception("Can't scrape Jubeat database for 'all' version!")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
Teach network how to import/export Jubeat emblem catalog.
2020-04-24 21:20:27 +02:00
try:
# Probably UTF-8 music DB
tree = ET.parse(xmlfile)
root = tree.getroot()
except ValueError:
# Probably shift-jis emblems
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
with open(xmlfile, "rb") as xmlhandle:
xmldata = xmlhandle.read().decode("shift_jisx0213")
Teach network how to import/export Jubeat emblem catalog.
2020-04-24 21:20:27 +02:00
root = ET.fromstring(xmldata)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
songs: List[Dict[str, Any]] = []
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
for music_entry in root.find("body") or []:
songid = int(music_entry.find("music_id").text)
bpm_min = float(music_entry.find("bpm_min").text)
bpm_max = float(music_entry.find("bpm_max").text)
jubeat: Update read script to import version
2022-08-16 06:56:09 +02:00
earliest_version = 0
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
version = int(music_entry.find("version").text.strip(), 16)
jubeat: Update read script to import version
2022-08-16 06:56:09 +02:00
while not version & 1:
version >>= 1
earliest_version += 1
# Since this is actually 1 to 1, I'm only adding this for clarity rather than necessity
version_to_db_constant = {
1: VersionConstants.JUBEAT,
2: VersionConstants.JUBEAT_RIPPLES,
3: VersionConstants.JUBEAT_RIPPLES_APPEND,
4: VersionConstants.JUBEAT_KNIT,
5: VersionConstants.JUBEAT_KNIT_APPEND,
6: VersionConstants.JUBEAT_COPIOUS,
7: VersionConstants.JUBEAT_COPIOUS_APPEND,
8: VersionConstants.JUBEAT_SAUCER,
9: VersionConstants.JUBEAT_SAUCER_FULFILL,
10: VersionConstants.JUBEAT_PROP,
11: VersionConstants.JUBEAT_QUBELL,
12: VersionConstants.JUBEAT_CLAN,
13: VersionConstants.JUBEAT_FESTO,
}
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if bpm_max > 0 and bpm_min < 0:
bpm_min = bpm_max
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
if music_entry.find("detail_level_bsc") is not None:
update read script
2022-08-17 01:44:34 +02:00
difficulties = [
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
float(music_entry.find("detail_level_bsc").text),
float(music_entry.find("detail_level_adv").text),
float(music_entry.find("detail_level_ext").text),
update read script
2022-08-17 01:44:34 +02:00
]
else:
difficulties = [
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
float(music_entry.find("level_bsc").text),
float(music_entry.find("level_adv").text),
float(music_entry.find("level_ext").text),
update read script
2022-08-17 01:44:34 +02:00
]
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
genre = "other"
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
if (
music_entry.find("genre") is not None
): # Qubell extend music_info doesn't have this field
for possible_genre in music_entry.find("genre"):
update read script
2022-08-17 01:44:34 +02:00
if int(possible_genre.text) != 0:
genre = str(possible_genre.tag)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
songs.append(
{
"id": songid,
# Title/artist aren't in the music data for Jubeat and must be manually populated.
# This is why there is a separate "import_metadata" and data file.
"title": None,
"artist": None,
"genre": genre,
"version": version_to_db_constant.get(earliest_version),
"bpm_min": bpm_min,
"bpm_max": bpm_max,
"difficulty": {
"basic": difficulties[0],
"advanced": difficulties[1],
"extreme": difficulties[2],
},
}
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
Teach network how to import/export Jubeat emblem catalog.
2020-04-24 21:20:27 +02:00
emblems: List[Dict[str, Any]] = []
if self.version in {
VersionConstants.JUBEAT_PROP,
VersionConstants.JUBEAT_QUBELL,
VersionConstants.JUBEAT_CLAN,
update read script
2022-08-17 01:44:34 +02:00
VersionConstants.JUBEAT_FESTO,
Teach network how to import/export Jubeat emblem catalog.
2020-04-24 21:20:27 +02:00
}:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
for emblem_entry in root.find("emblem_list") or []:
Teach network how to import/export Jubeat emblem catalog.
2020-04-24 21:20:27 +02:00
print(emblem_entry)
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
index = int(emblem_entry.find("index").text)
layer = int(emblem_entry.find("layer").text)
music_id = int(emblem_entry.find("music_id").text)
evolved = int(emblem_entry.find("evolved").text)
rarity = int(emblem_entry.find("rarity").text)
name = emblem_entry.find("name").text
emblems.append(
{
"id": index,
"layer": layer,
"music_id": music_id,
"evolved": evolved,
"rarity": rarity,
"name": name,
}
)
Teach network how to import/export Jubeat emblem catalog.
2020-04-24 21:20:27 +02:00
return songs, emblems
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
def lookup(
self, server: str, token: str
) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if self.version is None:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
raise Exception("Can't look up Jubeat database for 'all' version!")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Grab music info from remote server
music = self.remote_music(server, token)
songs = music.get_all_songs(self.game, self.version)
lut: Dict[int, Dict[str, Any]] = {}
chart_map = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
0: "basic",
1: "advanced",
2: "extreme",
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
}
# Format it the way we expect
for song in songs:
if song.chart not in chart_map:
# Ignore charts on songs we don't support/care about.
continue
if song.id not in lut:
lut[song.id] = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"id": song.id,
"title": song.name,
"artist": song.artist,
"genre": song.genre,
"version": song.data.get_int("version"),
"bpm_min": song.data.get_float("bpm_min"),
"bpm_max": song.data.get_float("bpm_max"),
"difficulty": {
"basic": 0.0,
"advanced": 0.0,
"extreme": 0.0,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
}
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
lut[song.id]["difficulty"][chart_map[song.chart]] = song.data.get_float(
"difficulty"
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
Teach network how to import/export Jubeat emblem catalog.
2020-04-24 21:20:27 +02:00
# Reassemble the data
reassembled_songs = [val for _, val in lut.items()]
emblems: List[Dict[str, Any]] = []
if self.version in {
VersionConstants.JUBEAT_PROP,
VersionConstants.JUBEAT_QUBELL,
VersionConstants.JUBEAT_CLAN,
update read script
2022-08-17 01:44:34 +02:00
VersionConstants.JUBEAT_FESTO,
Teach network how to import/export Jubeat emblem catalog.
2020-04-24 21:20:27 +02:00
}:
game = self.remote_game(server, token)
for item in game.get_items(self.game, self.version):
if item.type == "emblem":
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
emblems.append(
{
"id": item.id,
"layer": item.data.get_int("layer"),
"music_id": item.data.get_int("music_id"),
"evolved": item.data.get_int("evolved"),
"rarity": item.data.get_int("rarity"),
"name": item.data.get_str("name"),
}
)
Teach network how to import/export Jubeat emblem catalog.
2020-04-24 21:20:27 +02:00
return reassembled_songs, emblems
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
update read script
2022-08-17 01:44:34 +02:00
def __revivals(self, songid: int, chart: int) -> Optional[int]:
old_id = self.get_music_id_for_song(songid, chart)
if old_id is not None:
return old_id
# In qubell and clan omnimix, PPAP and Bonjour the world are placed
# at this arbitrary songid since they weren't assigned one originally
# In jubeat festo, these songs were given proper songids so we need to account for this
legacy_to_modern_map = {
71000001: 70000124, # PPAP
71000002: 70000154, # Bonjour the world
50000020: 80000037, # 千本桜 was removed and then revived in clan
60000063: 70000100, # Khamen break sdvx had the first id for prop(never released officially)
}
modern_to_legacy_map = {v: k for k, v in legacy_to_modern_map.items()}
legacy_songid = legacy_to_modern_map.get(songid)
if legacy_songid is not None:
old_id = self.get_music_id_for_song(legacy_songid, chart)
if old_id is not None:
return old_id
modern_songid = modern_to_legacy_map.get(songid)
if modern_songid is not None:
old_id = self.get_music_id_for_song(modern_songid, chart)
if old_id is not None:
return old_id
# Failed, so create a new one
return None
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
def import_music_db(self, songs: List[Dict[str, Any]]) -> None:
if self.version is None:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
raise Exception("Can't import Jubeat database for 'all' version!")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
chart_map: Dict[int, str] = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
0: "basic",
1: "advanced",
2: "extreme",
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
}
for song in songs:
# Skip over duplicate songs for the "play five different versions of this song
# across different prefectures" event. The song ID range is 8000301-8000347, so
# we arbitrarily choose to keep only the first one.
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
songid = song["id"]
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if songid in set(range(80000302, 80000348)):
continue
self.start_batch()
for chart in self.charts:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
if chart <= 2:
update read script
2022-08-17 01:44:34 +02:00
# First, try to find in the DB from another version
old_id = self.__revivals(songid, chart)
if self.no_combine or old_id is None:
# Insert original
print(f"New entry for {songid} chart {chart}")
next_id = self.get_next_music_id()
else:
# Insert pointing at same ID so scores transfer
print(f"Reused entry for {songid} chart {chart}")
next_id = old_id
data = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"difficulty": song["difficulty"][chart_map[chart]],
"bpm_min": song["bpm_min"],
"bpm_max": song["bpm_max"],
"version": song["version"],
update read script
2022-08-17 01:44:34 +02:00
}
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
else:
update read script
2022-08-17 01:44:34 +02:00
# First, try to find in the DB from another version
old_id = self.__revivals(songid, chart)
if self.no_combine or old_id is None:
# Insert original
print(f"New entry for {songid} chart {chart}")
next_id = self.get_next_music_id()
else:
# Insert pointing at same ID so scores transfer
print(f"Reused entry for {songid} chart {chart}")
next_id = old_id
data = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"difficulty": song["difficulty"][chart_map[chart - 3]],
"bpm_min": song["bpm_min"],
"bpm_max": song["bpm_max"],
"version": song["version"],
update read script
2022-08-17 01:44:34 +02:00
}
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
self.insert_music_id_for_song(
next_id,
songid,
chart,
song["title"],
song["artist"],
song["genre"],
data,
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
self.finish_batch()
Teach network how to import/export Jubeat emblem catalog.
2020-04-24 21:20:27 +02:00
def import_emblems(self, emblems: List[Dict[str, Any]]) -> None:
if self.version is None:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
raise Exception("Can't import Jubeat database for 'all' version!")
Teach network how to import/export Jubeat emblem catalog.
2020-04-24 21:20:27 +02:00
self.start_batch()
for i, emblem in enumerate(emblems):
# Make importing faster but still do it in chunks
if (i % 16) == 15:
self.finish_batch()
self.start_batch()
print(f"New catalog entry for {emblem['music_id']}")
self.insert_catalog_entry(
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"emblem",
emblem["id"],
Teach network how to import/export Jubeat emblem catalog.
2020-04-24 21:20:27 +02:00
{
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"layer": emblem["layer"],
"music_id": emblem["music_id"],
"evolved": emblem["evolved"],
"rarity": emblem["rarity"],
"name": emblem["name"],
Teach network how to import/export Jubeat emblem catalog.
2020-04-24 21:20:27 +02:00
},
)
self.finish_batch()
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
def import_metadata(self, tsvfile: str) -> None:
if self.version is not None:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
raise Exception(
"Unsupported Jubeat version, expected one of the following: all"
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
with open(tsvfile, newline="") as tsvhandle:
jubeatreader = csv.reader(tsvhandle, delimiter="\t", quotechar='"')
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
for row in jubeatreader:
songid = int(row[0])
name = row[1]
artist = row[2]
Convert most of the format() string calls to f-strings using libcst. Exact commands run were: python3 -m libcst.tool codemod convert_format_to_fstring.ConvertFormatStringCommand . --no-format python3 setup.py build_ext --inplace
2020-01-07 22:29:07 +01:00
print(f"Setting name/artist for {songid} all charts")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
self.start_batch()
for chart in self.charts:
self.update_metadata_for_song(songid, chart, name, artist)
self.finish_batch()
class ImportIIDX(ImportBase):
# Tutorial charts that shouldn't be on the UI
BANNED_CHARTS = [
16070,
16071,
16072,
16080,
16081,
16082,
]
def __init__(
self,
Make Config a real class instead of a Dict, for safer access and defaults.
2021-08-20 06:43:59 +02:00
config: Config,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
version: str,
no_combine: bool,
update: bool,
) -> None:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
if version in ["20", "21", "22", "23", "24", "25", "26"]:
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
actual_version = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"20": VersionConstants.IIDX_TRICORO,
"21": VersionConstants.IIDX_SPADA,
"22": VersionConstants.IIDX_PENDUAL,
"23": VersionConstants.IIDX_COPULA,
"24": VersionConstants.IIDX_SINOBUZ,
"25": VersionConstants.IIDX_CANNON_BALLERS,
"26": VersionConstants.IIDX_ROOTAGE,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
}[version]
self.charts = [0, 1, 2, 3, 4, 5, 6]
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
elif version in [
"omni-20",
"omni-21",
"omni-22",
"omni-23",
"omni-24",
"omni-25",
"omni-26",
]:
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
actual_version = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"omni-20": VersionConstants.IIDX_TRICORO,
"omni-21": VersionConstants.IIDX_SPADA,
"omni-22": VersionConstants.IIDX_PENDUAL,
"omni-23": VersionConstants.IIDX_COPULA,
"omni-24": VersionConstants.IIDX_SINOBUZ,
"omni-25": VersionConstants.IIDX_CANNON_BALLERS,
"omni-26": VersionConstants.IIDX_ROOTAGE,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
}[version] + DBConstants.OMNIMIX_VERSION_BUMP
self.charts = [0, 1, 2, 3, 4, 5, 6]
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
elif version == "all":
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
actual_version = None
self.charts = [0, 1, 2, 3, 4, 5, 6]
else:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
raise Exception(
"Unsupported IIDX version, expected one of the following: 20, 21, 22, 23, 24, 25, 26, omni-20, omni-21, omni-22, omni-23, omni-24, omni-25, omni-26!"
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
super().__init__(config, GameConstants.IIDX, actual_version, no_combine, update)
def __gather_sound_files(self, directory: str) -> Dict[int, str]:
files = {}
for (dirpath, dirnames, filenames) in os.walk(directory):
for filename in filenames:
songid, extension = os.path.splitext(filename)
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
if extension == ".1" or extension == ".ifs":
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
try:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
files[int(songid)] = os.path.join(
directory, os.path.join(dirpath, filename)
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
except ValueError:
# Invalid file
pass
for dirname in dirnames:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
files.update(
self.__gather_sound_files(os.path.join(directory, dirname))
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
return files
def __revivals(self, songid: int, chart: int) -> Optional[int]:
old_id = self.get_music_id_for_song(songid, chart)
if old_id is not None:
return old_id
# For revivals from older games, these show up as their respective old IDs
# in Spada Omnimix, but in Pendual Omnimix they're in the Pendual category.
legacy_to_modern_map = {
4213: 23066,
9203: 22068,
10203: 22052,
12201: 22039,
12204: 21201,
12206: 21064,
13215: 23077,
14202: 22025,
14210: 21068,
14211: 22069,
14214: 23070,
15202: 23069,
15204: 21063,
15205: 21065,
15207: 22028,
15208: 22049,
15209: 22043,
15211: 23060,
15215: 21062,
16207: 21067,
16209: 23062,
16212: 21066,
22096: 23030,
22097: 23051,
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
21214: 11101,
21221: 14101,
21225: 15104,
21226: 15102,
21231: 15101,
21237: 15103,
21240: 16105,
21242: 16104,
21253: 16103,
21258: 16102,
21262: 16101,
21220: 14100,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
}
# Some charts were changed, and others kept the same on these
if chart in [0, 1, 2]:
legacy_to_modern_map[9206] = 23065
legacy_songid = legacy_to_modern_map.get(songid)
if legacy_songid is not None:
old_id = self.get_music_id_for_song(legacy_songid, chart)
if old_id is not None:
return old_id
modern_to_legacy_map = {
23066: 4213,
22068: 9203,
22052: 10203,
22039: 12201,
21201: 12204,
21064: 12206,
23077: 13215,
22025: 14202,
21068: 14210,
22069: 14211,
23070: 14214,
23069: 15202,
21063: 15204,
21065: 15205,
22028: 15207,
22049: 15208,
22043: 15209,
23060: 15211,
21062: 15215,
21067: 16207,
23062: 16209,
21066: 16212,
23030: 22096,
23051: 22097,
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
11101: 21214,
14101: 21221,
15104: 21225,
15102: 21226,
15101: 21231,
15103: 21237,
16105: 21240,
16104: 21242,
16103: 21253,
16102: 21258,
16101: 21262,
14100: 21220,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
}
# Some charts were changed, and others kept the same on tehse
if chart in [0, 1, 2]:
modern_to_legacy_map[23065] = 9206
modern_songid = modern_to_legacy_map.get(songid)
if modern_songid is not None:
old_id = self.get_music_id_for_song(modern_songid, chart)
if old_id is not None:
return old_id
# Failed, so create a new one
return None
def __charts(self, songid: int, chart: int) -> int:
# Scripted connection long was set as a hyper in Tricoro omnimix, we
# need to map the charts to the another in every other version.
if songid == 12204:
if chart == 1:
return 2
if chart == 2:
return 1
return chart
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
def scrape(
self, binfile: str, assets_dir: Optional[str]
) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if self.version is None:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
raise Exception("Can't import IIDX database for 'all' version!")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if assets_dir is not None:
sound_files = self.__gather_sound_files(os.path.abspath(assets_dir))
else:
sound_files = None
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
bh = open(binfile, "rb")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
try:
binarydata = bh.read()
finally:
bh.close()
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
import_qpros = True # by default, try to import qpros
try:
Move PE operations into common library, fix 64-bit struct pointer dereferencing.
2021-08-15 02:41:37 +02:00
pe = PEFile(binarydata)
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
except BaseException:
import_qpros = False # if it failed then we're reading a music db file, not the executable
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
songs: List[Dict[str, Any]] = []
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
if not import_qpros:
musicdb = IIDXMusicDB(binarydata)
for song in musicdb.songs:
bpm = (0, 0)
notecounts = [0, 0, 0, 0, 0, 0]
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
if song.id in self.BANNED_CHARTS:
continue
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
if sound_files is not None:
if song.id in sound_files:
# Look up chart info!
filename = sound_files[song.id]
_, extension = os.path.splitext(filename)
data = None
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
if extension == ".1":
fp = open(filename, "rb")
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
data = fp.read()
fp.close()
else:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
fp = open(filename, "rb")
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
ifsdata = fp.read()
fp.close()
ifs = IFS(ifsdata)
for fn in ifs.filenames:
_, extension = os.path.splitext(fn)
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
if extension == ".1":
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
data = ifs.read_file(fn)
if data is not None:
iidxchart = IIDXChart(data)
bpm_min, bpm_max = iidxchart.bpm
bpm = (bpm_min, bpm_max)
notecounts = iidxchart.notecounts
else:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
print(
f"Could not find chart information for song {song.id}!"
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
else:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
print(
f"No chart information because chart for song {song.id} is missing!"
)
songs.append(
{
"id": song.id,
"title": song.title,
"artist": song.artist,
"genre": song.genre,
"bpm_min": bpm[0],
"bpm_max": bpm[1],
"difficulty": {
"spn": song.difficulties[0],
"sph": song.difficulties[1],
"spa": song.difficulties[2],
"dpn": song.difficulties[3],
"dph": song.difficulties[4],
"dpa": song.difficulties[5],
},
"notecount": {
"spn": notecounts[0],
"sph": notecounts[1],
"spa": notecounts[2],
"dpn": notecounts[3],
"dph": notecounts[4],
"dpa": notecounts[5],
},
}
)
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
qpros: List[Dict[str, Any]] = []
if self.version == VersionConstants.IIDX_TRICORO:
iidx: Add comments on read script denoting game version for qpro import
2021-08-02 22:23:47 +02:00
# Based on LDJ:J:A:A:2013090900
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
stride = 4
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
qp_head_offset = 0x1CCB18 # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in
qp_head_length = 79 # the qpro id numbers, it's best to store them as separate types in the catalog as well.
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
qp_hair_offset = 0x1CCC58
qp_hair_length = 103
qp_face_offset = 0x1CCDF8
qp_face_length = 50
qp_hand_offset = 0x1CCEC0
qp_hand_length = 103
qp_body_offset = 0x1CD060
qp_body_length = 106
filename_offset = 0
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
packedfmt = "I" # filename
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
if self.version == VersionConstants.IIDX_SPADA:
iidx: Add comments on read script denoting game version for qpro import
2021-08-02 22:23:47 +02:00
# Based on LDJ:J:A:A:2014071600
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
stride = 4
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
qp_head_offset = 0x213B50 # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in
qp_head_length = 125 # the qpro id numbers, it's best to store them as separate types in the catalog as well.
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
qp_hair_offset = 0x213D48
qp_hair_length = 126
qp_face_offset = 0x213F40
qp_face_length = 72
qp_hand_offset = 0x214060
qp_hand_length = 135
qp_body_offset = 0x214280
qp_body_length = 135
filename_offset = 0
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
packedfmt = "I" # filename
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
if self.version == VersionConstants.IIDX_PENDUAL:
iidx: Add comments on read script denoting game version for qpro import
2021-08-02 22:23:47 +02:00
# Based on LDJ:J:A:A:2015080500
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
stride = 4
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
qp_head_offset = 0x1D5228 # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in
qp_head_length = 163 # the qpro id numbers, it's best to store them as separate types in the catalog as well.
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
qp_hair_offset = 0x1D54B8
qp_hair_length = 182
qp_face_offset = 0x1D5790
qp_face_length = 106
qp_hand_offset = 0x1D5938
qp_hand_length = 184
qp_body_offset = 0x1D5C18
qp_body_length = 191
filename_offset = 0
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
packedfmt = "I" # filename
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
if self.version == VersionConstants.IIDX_COPULA:
iidx: Add comments on read script denoting game version for qpro import
2021-08-02 22:23:47 +02:00
# Based on LDJ:J:A:A:2016083100
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
stride = 8
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
qp_head_offset = 0x12F9D8 # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in
qp_head_length = 186 # the qpro id numbers, it's best to store them as separate types in the catalog as well.
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
qp_hair_offset = 0x12FFA8
qp_hair_length = 202
qp_face_offset = 0x1305F8
qp_face_length = 126
qp_hand_offset = 0x1309E8
qp_hand_length = 206
qp_body_offset = 0x131058
qp_body_length = 211
filename_offset = 0
qpro_id_offset = 1
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
packedfmt = "I" "I" # filename # string containing id and name of the part
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
if self.version == VersionConstants.IIDX_SINOBUZ:
iidx: Add comments on read script denoting game version for qpro import
2021-08-02 22:23:47 +02:00
# Based on LDJ:J:A:A:2017082800
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
stride = 8
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
qp_head_offset = 0x149F88 # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in
qp_head_length = 211 # the qpro id numbers, it's best to store them as separate types in the catalog as well.
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
qp_hair_offset = 0x14A620
qp_hair_length = 245
qp_face_offset = 0x14ADC8
qp_face_length = 152
qp_hand_offset = 0x14B288
qp_hand_length = 236
qp_body_offset = 0x14B9E8
qp_body_length = 256
filename_offset = 0
qpro_id_offset = 1
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
packedfmt = "I" "I" # filename # string containing id and name of the part
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
if self.version == VersionConstants.IIDX_CANNON_BALLERS:
iidx: Add comments on read script denoting game version for qpro import
2021-08-02 22:23:47 +02:00
# Based on LDJ:J:A:A:2018091900
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
stride = 16
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
qp_head_offset = 0x2339E0 # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in
qp_head_length = 231 # the qpro id numbers, it's best to store them as separate types in the catalog as well.
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
qp_hair_offset = 0x234850
qp_hair_length = 267
qp_face_offset = 0x235900
qp_face_length = 173
qp_hand_offset = 0x2363D0
qp_hand_length = 261
qp_body_offset = 0x237420
qp_body_length = 282
filename_offset = 0
qpro_id_offset = 1
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
packedfmt = "Q" "Q" # filename # string containing id and name of the part
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
if self.version == VersionConstants.IIDX_ROOTAGE:
iidx: Add comments on read script denoting game version for qpro import
2021-08-02 22:23:47 +02:00
# Based on LDJ:J:A:A:2019090200
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
stride = 16
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
qp_head_offset = 0x5065F0 # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in
qp_head_length = 259 # the qpro id numbers, it's best to store them as separate types in the catalog as well.
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
qp_hair_offset = 0x507620
qp_hair_length = 288
qp_face_offset = 0x508820
qp_face_length = 193
qp_hand_offset = 0x509430
qp_hand_length = 287
qp_body_offset = 0x50A620
qp_body_length = 304
filename_offset = 0
qpro_id_offset = 1
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
packedfmt = "Q" "Q" # filename # string containing id and name of the part
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
def read_string(offset: int) -> str:
# First, translate load offset in memory to disk offset
Move PE operations into common library, fix 64-bit struct pointer dereferencing.
2021-08-15 02:41:37 +02:00
offset = pe.virtual_to_physical(offset)
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
# Now, grab bytes until we're null-terminated
bytestring = []
while binarydata[offset] != 0:
bytestring.append(binarydata[offset])
offset = offset + 1
# Its shift-jis encoded, so decode it now
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
return bytes(bytestring).decode("shift_jisx0213")
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
def read_qpro_db(offset: int, length: int, qp_type: str) -> None:
for qpro_id in range(length):
chunkoffset = offset + (stride * qpro_id)
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
chunkdata = binarydata[chunkoffset : (chunkoffset + stride)]
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
unpacked = struct.unpack(packedfmt, chunkdata)
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
filename = read_string(unpacked[filename_offset]).replace("qp_", "")
remove = f"_{qp_type}.ifs"
filename = (
filename.replace(remove, "")
.replace("_head1.ifs", "")
.replace("_head2.ifs", "")
)
if self.version in [
VersionConstants.IIDX_TRICORO,
VersionConstants.IIDX_SPADA,
VersionConstants.IIDX_PENDUAL,
]:
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
name = filename # qpro names are not stored in these 3 games so use the identifier instead
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
else:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
name = read_string(unpacked[qpro_id_offset])[
4:
] # qpro name is stored in second string of form "000:name"
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
qproinfo = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"identifier": filename,
"id": qpro_id,
"name": name,
"type": qp_type,
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
}
qpros.append(qproinfo)
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
if import_qpros:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
read_qpro_db(qp_head_offset, qp_head_length, "head")
read_qpro_db(qp_hair_offset, qp_hair_length, "hair")
read_qpro_db(qp_face_offset, qp_face_length, "face")
read_qpro_db(qp_hand_offset, qp_hand_length, "hand")
read_qpro_db(qp_body_offset, qp_body_length, "body")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
return songs, qpros
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
def lookup(
self, server: str, token: str
) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if self.version is None:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
raise Exception("Can't look up IIDX database for 'all' version!")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Grab music info from remote server
music = self.remote_music(server, token)
songs = music.get_all_songs(self.game, self.version)
lut: Dict[int, Dict[str, Any]] = {}
chart_map = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
0: "spn",
1: "sph",
2: "spa",
3: "dpn",
4: "dph",
5: "dpa",
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
}
# Format it the way we expect
for song in songs:
if song.id in self.BANNED_CHARTS:
continue
if song.chart not in chart_map:
# Ignore charts on songs we don't support/care about.
continue
if song.id not in lut:
lut[song.id] = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"id": song.id,
"title": song.name,
"artist": song.artist,
"genre": song.genre,
"bpm_min": song.data.get_int("bpm_min"),
"bpm_max": song.data.get_int("bpm_max"),
"difficulty": {
"spn": 0,
"sph": 0,
"spa": 0,
"dpn": 0,
"dph": 0,
"dpa": 0,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"notecount": {
"spn": 0,
"sph": 0,
"spa": 0,
"dpn": 0,
"dph": 0,
"dpa": 0,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
}
if song.chart in chart_map:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
lut[song.id]["difficulty"][chart_map[song.chart]] = song.data.get_int(
"difficulty"
)
lut[song.id]["notecount"][chart_map[song.chart]] = song.data.get_int(
"notecount"
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Return the reassembled data
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
qpros: List[Dict[str, Any]] = []
game = self.remote_game(server, token)
for item in game.get_items(self.game, self.version):
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
if "qp_" in item.type:
qpros.append(
{
"identifier": item.data.get_str("identifier"),
"id": item.id,
"name": item.data.get_str("name"),
"type": item.data.get_str("type"),
}
)
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
return [val for _, val in lut.items()], qpros
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
def import_music_db(self, songs: List[Dict[str, Any]]) -> None:
if self.version is None:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
raise Exception("Can't import IIDX database for 'all' version!")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Import each song into our DB
chart_map = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
0: "spn",
1: "sph",
2: "spa",
3: "dpn",
4: "dph",
5: "dpa",
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
}
for song in songs:
self.start_batch()
for chart in self.charts:
if chart == 6:
# Beginner chart
songdata: Dict[str, Any] = {}
else:
songdata = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"difficulty": song["difficulty"][chart_map[chart]],
"bpm_min": song["bpm_min"],
"bpm_max": song["bpm_max"],
"notecount": song["notecount"][chart_map[chart]],
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
}
# First, try to find in the DB from another version
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
old_id = self.__revivals(song["id"], self.__charts(song["id"], chart))
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if self.no_combine or old_id is None:
# Insert original
Convert most of the format() string calls to f-strings using libcst. Exact commands run were: python3 -m libcst.tool codemod convert_format_to_fstring.ConvertFormatStringCommand . --no-format python3 setup.py build_ext --inplace
2020-01-07 22:29:07 +01:00
print(f"New entry for {song['id']} chart {chart}")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
next_id = self.get_next_music_id()
else:
# Insert pointing at same ID so scores transfer
Convert most of the format() string calls to f-strings using libcst. Exact commands run were: python3 -m libcst.tool codemod convert_format_to_fstring.ConvertFormatStringCommand . --no-format python3 setup.py build_ext --inplace
2020-01-07 22:29:07 +01:00
print(f"Reused entry for {song['id']} chart {chart}")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
next_id = old_id
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
self.insert_music_id_for_song(
next_id,
song["id"],
chart,
song["title"],
song["artist"],
song["genre"],
songdata,
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
self.finish_batch()
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
def import_qpros(self, qpros: List[Dict[str, Any]]) -> None:
if self.version is None:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
raise Exception("Can't import IIDX database for 'all' version!")
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
self.start_batch()
for i, qpro in enumerate(qpros):
# Make importing faster but still do it in chunks
if (i % 16) == 15:
self.finish_batch()
self.start_batch()
print(f"New catalog entry for {qpro['id']}")
self.insert_catalog_entry(
f"qp_{qpro['type']}",
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
qpro["id"],
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
{
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"name": qpro["name"],
"identifier": qpro["identifier"],
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
},
)
self.finish_batch()
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
def import_metadata(self, tsvfile: str) -> None:
if self.version is not None:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
raise Exception(
"Unsupported IIDX version, expected one of the following: all"
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
with open(tsvfile, newline="") as tsvhandle:
iidxreader = csv.reader(tsvhandle, delimiter="\t", quotechar='"')
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
for row in iidxreader:
songid = int(row[0])
name = row[1]
artist = row[2]
genre = row[3]
if len(name) == 0:
name = None
if len(artist) == 0:
artist = None
if len(genre) == 0:
genre = None
Convert most of the format() string calls to f-strings using libcst. Exact commands run were: python3 -m libcst.tool codemod convert_format_to_fstring.ConvertFormatStringCommand . --no-format python3 setup.py build_ext --inplace
2020-01-07 22:29:07 +01:00
print(f"Setting name/artist/genre for {songid} all charts")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
self.start_batch()
for chart in self.charts:
self.update_metadata_for_song(songid, chart, name, artist, genre)
self.finish_batch()
class ImportDDR(ImportBase):
def __init__(
self,
Make Config a real class instead of a Dict, for safer access and defaults.
2021-08-20 06:43:59 +02:00
config: Config,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
version: str,
no_combine: bool,
update: bool,
) -> None:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
if version in ["12", "13", "14", "15", "16"]:
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
actual_version = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"12": VersionConstants.DDR_X2,
"13": VersionConstants.DDR_X3_VS_2NDMIX,
"14": VersionConstants.DDR_2013,
"15": VersionConstants.DDR_2014,
"16": VersionConstants.DDR_ACE,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
}[version]
self.charts = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
else:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
raise Exception(
"Unsupported DDR version, expected one of the following: 12, 13, 14, 15, 16"
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
super().__init__(config, GameConstants.DDR, actual_version, no_combine, update)
def scrape(self, infile: str) -> List[Dict[str, Any]]:
with open(infile, mode="rb") as myfile:
data = myfile.read()
myfile.close()
if self.version == VersionConstants.DDR_X2:
# Based on JDX:J:A:A:2010111000
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
offset = 0x254FC0
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
size = 0x14C
length = 894
# Basic stuff like ID, bpm, chart difficulties
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
unpackfmt = "<xxxxxxxxHHxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxHHBBBBBBBBBB"
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Groove radar
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
unpackfmt += "HHHHHHHHH" * 5
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if len(unpackfmt) < size:
# Skew is because I'm too lazy to count the Hs above
skew = 3 + (9 * 5)
# Just pad it for ease of construction
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
unpackfmt = unpackfmt + ("x" * (size - len(unpackfmt) - skew))
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Basic offsets
id_offset = 1
edit_offset = 0
bpm_min_offset = 3
bpm_max_offset = 2
folder_offset = 24 # This is a byte offset into the raw field
# Single/double difficulty array offsets
single_difficulties = 4
double_difficulties = 9
# Groove gauge offsets
groove_single_beginner = 22
groove_single_basic = 14
groove_single_difficult = 15
groove_single_expert = 16
groove_single_challenge = 17
groove_double_basic = 18
groove_double_difficult = 19
groove_double_expert = 20
groove_double_challenge = 21
# Relative offsets for each groove gauge value
voltage = 0
stream = 9
air = 18
chaos = 27
freeze = 36
# Folder start version
folder_start = 12
elif self.version == VersionConstants.DDR_X3_VS_2NDMIX:
# Based on KDX:J:A:A:2012112600
offset = 0x27A4C8
size = 0x150
length = 1062
# Basic stuff like ID, bpm, chart difficulties
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
unpackfmt = (
"<xxxxxxxxHHxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxHHBBBBBBBBBB"
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Groove radar
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
unpackfmt += "HHHHHHHHH" * 5
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if len(unpackfmt) < size:
# Skew is because I'm too lazy to count the Hs above
skew = 3 + (9 * 5)
# Just pad it for ease of construction
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
unpackfmt = unpackfmt + ("x" * (size - len(unpackfmt) - skew))
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Basic offsets
id_offset = 1
edit_offset = 0
bpm_min_offset = 3
bpm_max_offset = 2
folder_offset = 24 # This is a byte offset into the raw field
# Single/double difficulty array offsets
single_difficulties = 4
double_difficulties = 9
# Groove gauge offsets
groove_single_beginner = 22
groove_single_basic = 14
groove_single_difficult = 15
groove_single_expert = 16
groove_single_challenge = 17
groove_double_basic = 18
groove_double_difficult = 19
groove_double_expert = 20
groove_double_challenge = 21
# Relative offsets for each groove gauge value
voltage = 0
stream = 9
air = 18
chaos = 27
freeze = 36
# Folder start version
folder_start = 13
elif self.version == VersionConstants.DDR_2013:
# Based on MDX:J:A:A:2014032700
offset = 0x2663D8
size = 0x1D0
length = 1238
# Basic stuff like ID, bpm, chart difficulties
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
unpackfmt = (
"<xxxxxxxxHHxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxHHBBBBBBBBBB"
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Groove radar
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
unpackfmt += "HHHHHHHHH" * 5
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if len(unpackfmt) < size:
# Skew is because I'm too lazy to count the Hs above
skew = 3 + (9 * 5)
# Just pad it for ease of construction
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
unpackfmt = unpackfmt + ("x" * (size - len(unpackfmt) - skew))
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Basic offsets
id_offset = 1
edit_offset = 0
bpm_min_offset = 3
bpm_max_offset = 2
folder_offset = 20 # This is a byte offset into the raw field
# Single/double difficulty array offsets
single_difficulties = 4
double_difficulties = 9
# Groove gauge offsets
groove_single_beginner = 22
groove_single_basic = 14
groove_single_difficult = 15
groove_single_expert = 16
groove_single_challenge = 17
groove_double_basic = 18
groove_double_difficult = 19
groove_double_expert = 20
groove_double_challenge = 21
# Relative offsets for each groove gauge value
voltage = 0
stream = 9
air = 18
chaos = 27
freeze = 36
# Folder start version
folder_start = 14
elif self.version == VersionConstants.DDR_2014:
# Based on MDX:A:A:A:2015122100
offset = 0x2B72B0
size = 0x1D0
length = 1466
# Basic stuff like ID, bpm, chart difficulties
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
unpackfmt = (
"<xxxxxxxxHHxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxHHBBBBBBBBBB"
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Groove radar
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
unpackfmt += "HHHHHHHHH" * 5
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if len(unpackfmt) < size:
# Skew is because I'm too lazy to count the Hs above
skew = 3 + (9 * 5)
# Just pad it for ease of construction
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
unpackfmt = unpackfmt + ("x" * (size - len(unpackfmt) - skew))
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Basic offsets
id_offset = 1
edit_offset = 0
bpm_min_offset = 3
bpm_max_offset = 2
folder_offset = 20 # This is a byte offset into the raw field
# Single/double difficulty array offsets
single_difficulties = 4
double_difficulties = 9
# Groove gauge offsets
groove_single_beginner = 22
groove_single_basic = 14
groove_single_difficult = 15
groove_single_expert = 16
groove_single_challenge = 17
groove_double_basic = 18
groove_double_difficult = 19
groove_double_expert = 20
groove_double_challenge = 21
# Relative offsets for each groove gauge value
voltage = 0
stream = 9
air = 18
chaos = 27
freeze = 36
# Folder start version
folder_start = 15
else:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
raise Exception("Unknown game version!")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
songs = []
for i in range(length):
start = offset + (i * size)
end = offset + ((i + 1) * size)
chunk = data[start:end]
# First, figure out if it is actually a song
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
ssqcode = chunk[0:6].decode("ascii").replace("\0", "").strip()
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if len(ssqcode) == 0:
continue
unpacked = struct.unpack(unpackfmt, chunk)
songinfo = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"id": unpacked[id_offset],
"edit_id": unpacked[edit_offset],
"ssqcode": ssqcode,
"difficulty": {
"single": {
"beginner": unpacked[single_difficulties + 0],
"basic": unpacked[single_difficulties + 1],
"difficult": unpacked[single_difficulties + 2],
"expert": unpacked[single_difficulties + 3],
"challenge": unpacked[single_difficulties + 4],
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"double": {
"beginner": unpacked[double_difficulties + 0],
"basic": unpacked[double_difficulties + 1],
"difficult": unpacked[double_difficulties + 2],
"expert": unpacked[double_difficulties + 3],
"challenge": unpacked[double_difficulties + 4],
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
},
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"groove_gauge": {
"single": {
"beginner": {
"voltage": unpacked[groove_single_beginner + voltage],
"stream": unpacked[groove_single_beginner + stream],
"air": unpacked[groove_single_beginner + air],
"chaos": unpacked[groove_single_beginner + chaos],
"freeze": unpacked[groove_single_beginner + freeze],
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"basic": {
"voltage": unpacked[groove_single_basic + voltage],
"stream": unpacked[groove_single_basic + stream],
"air": unpacked[groove_single_basic + air],
"chaos": unpacked[groove_single_basic + chaos],
"freeze": unpacked[groove_single_basic + freeze],
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"difficult": {
"voltage": unpacked[groove_single_difficult + voltage],
"stream": unpacked[groove_single_difficult + stream],
"air": unpacked[groove_single_difficult + air],
"chaos": unpacked[groove_single_difficult + chaos],
"freeze": unpacked[groove_single_difficult + freeze],
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"expert": {
"voltage": unpacked[groove_single_expert + voltage],
"stream": unpacked[groove_single_expert + stream],
"air": unpacked[groove_single_expert + air],
"chaos": unpacked[groove_single_expert + chaos],
"freeze": unpacked[groove_single_expert + freeze],
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"challenge": {
"voltage": unpacked[groove_single_challenge + voltage],
"stream": unpacked[groove_single_challenge + stream],
"air": unpacked[groove_single_challenge + air],
"chaos": unpacked[groove_single_challenge + chaos],
"freeze": unpacked[groove_single_challenge + freeze],
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
},
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"double": {
"beginner": {
"voltage": 0,
"stream": 0,
"air": 0,
"chaos": 0,
"freeze": 0,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"basic": {
"voltage": unpacked[groove_double_basic + voltage],
"stream": unpacked[groove_double_basic + stream],
"air": unpacked[groove_double_basic + air],
"chaos": unpacked[groove_double_basic + chaos],
"freeze": unpacked[groove_double_basic + freeze],
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"difficult": {
"voltage": unpacked[groove_double_difficult + voltage],
"stream": unpacked[groove_double_difficult + stream],
"air": unpacked[groove_double_difficult + air],
"chaos": unpacked[groove_double_difficult + chaos],
"freeze": unpacked[groove_double_difficult + freeze],
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"expert": {
"voltage": unpacked[groove_double_expert + voltage],
"stream": unpacked[groove_double_expert + stream],
"air": unpacked[groove_double_expert + air],
"chaos": unpacked[groove_double_expert + chaos],
"freeze": unpacked[groove_double_expert + freeze],
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"challenge": {
"voltage": unpacked[groove_double_challenge + voltage],
"stream": unpacked[groove_double_challenge + stream],
"air": unpacked[groove_double_challenge + air],
"chaos": unpacked[groove_double_challenge + chaos],
"freeze": unpacked[groove_double_challenge + freeze],
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
},
},
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"bpm_min": unpacked[bpm_min_offset],
"bpm_max": unpacked[bpm_max_offset],
"folder": folder_start - chunk[folder_offset],
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
}
songs.append(songinfo)
return songs
def hydrate(self, songs: List[Dict[str, Any]], infile: str) -> List[Dict[str, Any]]:
tree = ET.parse(infile)
root = tree.getroot()
data = {}
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
for music_entry in root.find("mdblist"):
musicid = int(music_entry.attrib["reclink"])
title = ""
artist = ""
title_en = ""
artist_en = ""
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
for child in music_entry:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
if child.tag == "name":
if child.attrib["lang"] == "ja":
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
title = child.text
else:
title_en = child.text
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
if child.tag == "artist":
if child.attrib["lang"] == "ja":
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
artist = child.text
else:
artist_en = child.text
data[musicid] = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"title": title,
"artist": artist,
"title_en": title_en,
"artist_en": artist_en,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
}
songs = copy.deepcopy(songs)
for song in songs:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
newdata = data.get(song["id"])
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if newdata is not None:
song.update(newdata)
return songs
def parse_xml(self, arcfile: str) -> List[Dict[str, Any]]:
with open(arcfile, mode="rb") as myfile:
data = myfile.read()
myfile.close()
arc = ARC(data)
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
if "data/gamedata/musicdb.xml" in arc.filenames:
xmldata = arc.read_file("data/gamedata/musicdb.xml")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
else:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
raise Exception("Invalid .arc file provided!")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
xml = xmldata.decode("utf-8")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
root = ET.fromstring(xml)
songs = []
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
if root.tag != "mdb":
raise Exception("Invalid musicdb.xml file in .arc!")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
for music_entry in root:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
songid = int(music_entry.find("mcode").text)
title = music_entry.find("title").text
artist = music_entry.find("artist").text
ssqcode = music_entry.find("basename").text
bpm = int(music_entry.find("bpmmax").text)
folder = int(music_entry.find("series").text)
difficulties = [int(x) for x in music_entry.find("diffLv").text.split(" ")]
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# For some reason Ace thinks of itself as 17, and DDR 2013/2014 as 14, 15 and 16
# somewhat spread out. Fix that here.
folder = {
1: 1,
2: 2,
3: 3,
4: 4,
5: 5,
6: 6,
7: 7,
8: 8,
9: 9,
10: 10,
11: 11,
12: 12,
13: 13,
14: 14,
15: 15,
16: 15,
17: 16,
18: 17,
}[folder]
songinfo = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"id": songid,
"edit_id": songid,
"ssqcode": ssqcode,
"title": title,
"artist": artist,
"difficulty": {
"single": {
"beginner": difficulties[0],
"basic": difficulties[1],
"difficult": difficulties[2],
"expert": difficulties[3],
"challenge": difficulties[4],
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"double": {
"beginner": difficulties[5],
"basic": difficulties[6],
"difficult": difficulties[7],
"expert": difficulties[8],
"challenge": difficulties[9],
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
},
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"bpm_min": bpm,
"bpm_max": bpm,
"folder": folder,
"groove_gauge": {},
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
}
# Groove information is calculated on the fly, maybe some day we will
# duplicate that here, but for now, zero it out.
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
for playmode in ["single", "double"]:
songinfo["groove_gauge"][playmode] = {} # type: ignore
for charttype in [
"beginner",
"basic",
"difficult",
"expert",
"challenge",
]:
songinfo["groove_gauge"][playmode][charttype] = { # type: ignore
"voltage": 0,
"stream": 0,
"air": 0,
"chaos": 0,
"freeze": 0,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
}
songs.append(songinfo)
return songs
def lookup(self, server: str, token: str) -> List[Dict[str, Any]]:
# Grab music info from remote server
music = self.remote_music(server, token)
songs = music.get_all_songs(self.game, self.version)
lut: Dict[int, Dict[str, Any]] = {}
chart_map = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
0: ("single", "beginner"),
1: ("single", "basic"),
2: ("single", "difficult"),
3: ("single", "expert"),
4: ("single", "challenge"),
5: ("double", "beginner"),
6: ("double", "basic"),
7: ("double", "difficult"),
8: ("double", "expert"),
9: ("double", "challenge"),
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
}
# Format it the way we expect
for song in songs:
if song.chart not in chart_map:
# Ignore charts on songs we don't support/care about.
continue
if song.id not in lut:
lut[song.id] = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"id": song.id,
"edit_id": song.data.get_int("edit_id"),
"ssqcode": "",
"title": song.name,
"artist": song.artist,
"difficulty": {
"single": {
"beginner": 0,
"basic": 0,
"difficult": 0,
"expert": 0,
"challenge": 0,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"double": {
"beginner": 0,
"basic": 0,
"difficult": 0,
"expert": 0,
"challenge": 0,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
},
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"groove_gauge": {
"single": {
"beginner": {
"voltage": 0,
"stream": 0,
"air": 0,
"chaos": 0,
"freeze": 0,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"basic": {
"voltage": 0,
"stream": 0,
"air": 0,
"chaos": 0,
"freeze": 0,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"difficult": {
"voltage": 0,
"stream": 0,
"air": 0,
"chaos": 0,
"freeze": 0,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"expert": {
"voltage": 0,
"stream": 0,
"air": 0,
"chaos": 0,
"freeze": 0,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"challenge": {
"voltage": 0,
"stream": 0,
"air": 0,
"chaos": 0,
"freeze": 0,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
},
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"double": {
"beginner": {
"voltage": 0,
"stream": 0,
"air": 0,
"chaos": 0,
"freeze": 0,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"basic": {
"voltage": 0,
"stream": 0,
"air": 0,
"chaos": 0,
"freeze": 0,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"difficult": {
"voltage": 0,
"stream": 0,
"air": 0,
"chaos": 0,
"freeze": 0,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"expert": {
"voltage": 0,
"stream": 0,
"air": 0,
"chaos": 0,
"freeze": 0,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"challenge": {
"voltage": 0,
"stream": 0,
"air": 0,
"chaos": 0,
"freeze": 0,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
},
},
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"bpm_min": song.data.get_int("bpm_min"),
"bpm_max": song.data.get_int("bpm_max"),
"folder": song.data.get_int("category"),
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
}
style, chart = chart_map[song.chart]
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
lut[song.id]["difficulty"][style][chart] = song.data.get_int("difficulty")
lut[song.id]["groove_gauge"][style][chart]["air"] = song.data.get_dict(
"groove"
).get_int("air")
lut[song.id]["groove_gauge"][style][chart]["chaos"] = song.data.get_dict(
"groove"
).get_int("chaos")
lut[song.id]["groove_gauge"][style][chart]["freeze"] = song.data.get_dict(
"groove"
).get_int("freeze")
lut[song.id]["groove_gauge"][style][chart]["stream"] = song.data.get_dict(
"groove"
).get_int("stream")
lut[song.id]["groove_gauge"][style][chart]["voltage"] = song.data.get_dict(
"groove"
).get_int("voltage")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Return the reassembled data
return [val for _, val in lut.items()]
def import_music_db(self, songs: List[Dict[str, Any]]) -> None:
for song in songs:
self.start_batch()
for chart in self.charts:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
key = ["beginner", "basic", "difficult", "expert", "challenge"]
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if chart in [0, 1, 2, 3, 4]:
# For singles only!
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
difficulty = song["difficulty"]["single"][key[chart]]
groovestats = song["groove_gauge"]["single"][key[chart]]
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
elif chart in [5, 6, 7, 8, 9]:
# Doubles!
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
difficulty = song["difficulty"]["double"][key[chart - 5]]
groovestats = song["groove_gauge"]["double"][key[chart - 5]]
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
else:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
raise Exception("Unrecognized chart type!")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if difficulty == 0:
# No chart for this difficulty
continue
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
if song["edit_id"] == 0:
raise Exception("Expected non-zero edit id!")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# DDR is stupid and changes in-game IDs around willy-nilly, but the edit ID is stable.
# So, create a virtual edit ID entry and link everything to that. We can't just store
# the edit ID as the real ID because in-game the protocol uses the changing ID.
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
old_id = self.get_music_id_for_song(song["edit_id"], chart, version=0)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if self.no_combine or old_id is None:
# Insert original
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
print(
f"New entry for {song['title']} {song['artist']} ({song['id']} chart {chart})"
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
next_id = self.get_next_music_id()
else:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
print(
f"Reused entry for {song['title']} {song['artist']} ({song['id']} chart {chart})"
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
next_id = old_id
# Add the virtual entry we talked about above, so we can link this song in the future.
self.insert_music_id_for_song(
next_id,
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
song["edit_id"],
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
chart,
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
song["title"],
song["artist"],
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
None, # No genres in DDR
{
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"category": song["folder"],
"bpm_min": song["bpm_min"],
"bpm_max": song["bpm_max"],
"difficulty": difficulty,
"groove": groovestats,
"edit_id": song["edit_id"],
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
version=0,
)
# Add the normal entry so the game finds the song.
self.insert_music_id_for_song(
next_id,
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
song["id"],
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
chart,
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
song["title"],
song["artist"],
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
None, # No genres in DDR
{
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"category": song["folder"],
"bpm_min": song["bpm_min"],
"bpm_max": song["bpm_max"],
"difficulty": difficulty,
"groove": groovestats,
"edit_id": song["edit_id"],
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
)
self.finish_batch()
class ImportSDVX(ImportBase):
def __init__(
self,
Make Config a real class instead of a Dict, for safer access and defaults.
2021-08-20 06:43:59 +02:00
config: Config,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
version: str,
no_combine: bool,
update: bool,
) -> None:
actual_version = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"1": VersionConstants.SDVX_BOOTH,
"2": VersionConstants.SDVX_INFINITE_INFECTION,
"3": VersionConstants.SDVX_GRAVITY_WARS,
"4": VersionConstants.SDVX_HEAVENLY_HAVEN,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
}.get(version, -1)
if actual_version == VersionConstants.SDVX_BOOTH:
self.charts = [0, 1, 2]
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
elif actual_version in [
VersionConstants.SDVX_INFINITE_INFECTION,
VersionConstants.SDVX_GRAVITY_WARS,
]:
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
self.charts = [0, 1, 2, 3]
elif actual_version == VersionConstants.SDVX_HEAVENLY_HAVEN:
self.charts = [0, 1, 2, 3, 4]
else:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
raise Exception(
"Unsupported SDVX version, expected one of the following: 1, 2, 3, 4!"
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
super().__init__(config, GameConstants.SDVX, actual_version, no_combine, update)
def scrape(self, infile: str) -> List[Dict[str, Any]]:
with open(infile, mode="rb") as myfile:
data = myfile.read()
myfile.close()
Move PE operations into common library, fix 64-bit struct pointer dereferencing.
2021-08-15 02:41:37 +02:00
pe = PEFile(data)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if self.version == VersionConstants.SDVX_BOOTH:
offset = 0xFFF28
size = 163
stride = 40
else:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
raise Exception("Unsupported version for catalog scrape!")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
def read_string(spot: int) -> str:
# First, translate load offset in memory to disk offset
Move PE operations into common library, fix 64-bit struct pointer dereferencing.
2021-08-15 02:41:37 +02:00
spot = pe.virtual_to_physical(spot)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Now, grab bytes until we're null-terminated
bytestring = []
while data[spot] != 0:
bytestring.append(data[spot])
spot = spot + 1
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
return bytes(bytestring).decode("shift_jis")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
entries = []
for i in range(size):
start = offset + i * stride
end = offset + (i + 1) * stride
chunk = data[start:end]
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
values = struct.unpack("<IIIIIIIIII", chunk)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Price looks to be fixed here, assert it so we catch problems
if values[3] != values[4]:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
raise Exception("Expected price values to match!")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
entry = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"catalogid": values[0],
"musicid": values[1],
"chart": values[2],
"price": values[3],
"condition_jp": read_string(values[8]),
"condition_en": read_string(values[9]),
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
}
entries.append(entry)
return entries
def import_catalog(self, dllfile: str) -> None:
entries = self.scrape(dllfile)
for entry in entries:
self.start_batch()
Convert most of the format() string calls to f-strings using libcst. Exact commands run were: python3 -m libcst.tool codemod convert_format_to_fstring.ConvertFormatStringCommand . --no-format python3 setup.py build_ext --inplace
2020-01-07 22:29:07 +01:00
print(f"New catalog entry for {entry['musicid']} chart {entry['chart']}")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
self.insert_catalog_entry(
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"song_unlock",
entry["catalogid"],
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
{
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"blocks": entry["price"],
"musicid": entry["musicid"],
"chart": entry["chart"],
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
)
self.finish_batch()
def import_appeal_cards(self, csvfile: str) -> None:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
with open(csvfile, "rb") as csvhandle:
csvdata = csvhandle.read().decode("shift_jisx0213")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
csvstr = io.StringIO(csvdata)
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
appealreader = csv.reader(csvstr, delimiter=",", quotechar='"')
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
for row in appealreader:
appealids = []
if self.version == VersionConstants.SDVX_INFINITE_INFECTION:
try:
appealids.append(int(row[-5]))
except (TypeError, ValueError):
pass
elif self.version == VersionConstants.SDVX_GRAVITY_WARS:
try:
appealids.append(int(row[-9]))
except (TypeError, ValueError):
pass
else:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
raise Exception(
f"Cannot import appeal cards for SDVX version {self.version}"
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
self.start_batch()
for appealid in appealids:
Convert most of the format() string calls to f-strings using libcst. Exact commands run were: python3 -m libcst.tool codemod convert_format_to_fstring.ConvertFormatStringCommand . --no-format python3 setup.py build_ext --inplace
2020-01-07 22:29:07 +01:00
print(f"New catalog entry for appeal card {appealid}")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
self.insert_catalog_entry(
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"appealcard",
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
appealid,
{},
)
self.finish_batch()
def import_music_db_or_appeal_cards(self, xmlfile: str) -> None:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
with open(xmlfile, "rb") as fp:
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# This is gross, but elemtree won't do it for us so whatever
bytedata = fp.read()
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
strdata = bytedata.decode("shift_jisx0213", errors="replace")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
root = ET.fromstring(strdata)
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
for music_entry in root.findall("music"):
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Grab the ID
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
songid = int(music_entry.attrib["id"])
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
title = None
artist = None
bpm_min = None
bpm_max = None
limited = [0, 0, 0, 0, 0]
difficulties = [0, 0, 0, 0, 0]
if self.version == VersionConstants.SDVX_BOOTH:
# Find normal info about the song
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
for info in music_entry.findall("info"):
if info.attrib["attr"] == "title_yomigana":
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
title = jaconv.h2z(info.text)
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
if info.attrib["attr"] == "artist_yomigana":
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
artist = jaconv.h2z(info.text)
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
if info.attrib["attr"] == "bpm_min":
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
bpm_min = float(info.text)
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
if info.attrib["attr"] == "bpm_max":
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
bpm_max = float(info.text)
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
if info.attrib["attr"] == "limited":
limited = [
int(info.text),
int(info.text),
int(info.text),
int(info.text),
]
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Make sure we got everything
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
if (
title is None
or artist is None
or bpm_min is None
or bpm_max is None
):
raise Exception(f"Couldn't parse info for song {songid}")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Grab valid difficulties
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
for difficulty in music_entry.findall("difficulty"):
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Figure out the actual difficulty
offset = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"novice": 0,
"advanced": 1,
"exhaust": 2,
}.get(difficulty.attrib["attr"])
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if offset is None:
continue
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
difficulties[offset] = int(difficulty.find("difnum").text)
elif self.version in [
VersionConstants.SDVX_INFINITE_INFECTION,
VersionConstants.SDVX_GRAVITY_WARS,
]:
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Find normal info about the song
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
info = music_entry.find("info")
title = info.find("title_name").text
artist = info.find("artist_name").text
bpm_min = float(info.find("bpm_min").text) / 100.0
bpm_max = float(info.find("bpm_max").text) / 100.0
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Grab valid difficulties
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
for difficulty in music_entry.find("difficulty"):
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Figure out the actual difficulty
offset = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"novice": 0,
"advanced": 1,
"exhaust": 2,
"infinite": 3,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
}.get(difficulty.tag)
if offset is None:
continue
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
difficulties[offset] = int(difficulty.find("difnum").text)
limited[offset] = int(difficulty.find("limited").text)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
elif self.version == VersionConstants.SDVX_HEAVENLY_HAVEN:
# Find normal info about the song
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
info = music_entry.find("info")
title = info.find("title_name").text
artist = info.find("artist_name").text
bpm_min = float(info.find("bpm_min").text) / 100.0
bpm_max = float(info.find("bpm_max").text) / 100.0
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Grab valid difficulties
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
for difficulty in music_entry.find("difficulty"):
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Figure out the actual difficulty
offset = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"novice": 0,
"advanced": 1,
"exhaust": 2,
"infinite": 3,
"maximum": 4,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
}.get(difficulty.tag)
if offset is None:
continue
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
difficulties[offset] = int(difficulty.find("difnum").text)
limited[offset] = int(difficulty.find("limited").text)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Fix accent issues with title/artist
accent_lut: Dict[str, str] = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"驩": "Ø",
"齲": "♥",
"齶": "♡",
"趁": "Ǣ",
"騫": "á",
"曦": "à",
"驫": "ā",
"齷": "é",
"曩": "è",
"䧺": "ê",
"骭": "ü",
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
}
for orig, rep in accent_lut.items():
title = title.replace(orig, rep)
artist = artist.replace(orig, rep)
# Import it
self.start_batch()
for chart in self.charts:
# First, try to find in the DB from another version
old_id = self.get_music_id_for_song(songid, chart)
if self.no_combine or old_id is None:
# Insert original
Convert most of the format() string calls to f-strings using libcst. Exact commands run were: python3 -m libcst.tool codemod convert_format_to_fstring.ConvertFormatStringCommand . --no-format python3 setup.py build_ext --inplace
2020-01-07 22:29:07 +01:00
print(f"New entry for {songid} chart {chart}")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
next_id = self.get_next_music_id()
else:
# Insert pointing at same ID so scores transfer
Convert most of the format() string calls to f-strings using libcst. Exact commands run were: python3 -m libcst.tool codemod convert_format_to_fstring.ConvertFormatStringCommand . --no-format python3 setup.py build_ext --inplace
2020-01-07 22:29:07 +01:00
print(f"Reused entry for {songid} chart {chart}")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
next_id = old_id
data = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"limited": limited[chart],
"difficulty": difficulties[chart],
"bpm_min": bpm_min,
"bpm_max": bpm_max,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
}
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
self.insert_music_id_for_song(
next_id, songid, chart, title, artist, None, data
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
self.finish_batch()
appealids: List[int] = []
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
for appeal_entry in root.findall("card"):
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Grab the ID
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
appealids.append(int(appeal_entry.attrib["id"]))
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if appealids:
self.start_batch()
for appealid in appealids:
Convert most of the format() string calls to f-strings using libcst. Exact commands run were: python3 -m libcst.tool codemod convert_format_to_fstring.ConvertFormatStringCommand . --no-format python3 setup.py build_ext --inplace
2020-01-07 22:29:07 +01:00
print(f"New catalog entry for appeal card {appealid}")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
self.insert_catalog_entry(
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"appealcard",
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
appealid,
{},
)
self.finish_batch()
def import_from_server(self, server: str, token: str) -> None:
# First things first, lets try to import the music DB. We want to make
# sure that even if the server doesn't respond right, we have a chart
# entry for every chart for each song we're importing.
music = self.remote_music(server, token)
music_lut: Dict[int, Dict[int, Song]] = {}
for entry in music.get_all_songs(self.game, self.version):
if entry.id not in music_lut:
music_lut[entry.id] = {
chart: Song(
entry.game,
entry.version,
entry.id,
chart,
entry.name,
entry.artist,
entry.genre,
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
{},
)
for chart in self.charts
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
}
music_lut[entry.id][entry.chart] = entry
# Import it
for _, songs in music_lut.items():
self.start_batch()
for _, song in songs.items():
old_id = self.get_music_id_for_song(song.id, song.chart)
if self.no_combine or old_id is None:
# Insert original
Convert most of the format() string calls to f-strings using libcst. Exact commands run were: python3 -m libcst.tool codemod convert_format_to_fstring.ConvertFormatStringCommand . --no-format python3 setup.py build_ext --inplace
2020-01-07 22:29:07 +01:00
print(f"New entry for {song.id} chart {song.chart}")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
next_id = self.get_next_music_id()
else:
# Insert pointing at same ID so scores transfer
Convert most of the format() string calls to f-strings using libcst. Exact commands run were: python3 -m libcst.tool codemod convert_format_to_fstring.ConvertFormatStringCommand . --no-format python3 setup.py build_ext --inplace
2020-01-07 22:29:07 +01:00
print(f"Reused entry for {song.id} chart {song.chart}")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
next_id = old_id
data = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"limited": song.data.get_int("limited"),
"difficulty": song.data.get_int("difficulty"),
"bpm_min": song.data.get_int("bpm_min"),
"bpm_max": song.data.get_int("bpm_max"),
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
}
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
self.insert_music_id_for_song(
next_id, song.id, song.chart, song.name, song.artist, None, data
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
self.finish_batch()
# Now, attempt to insert any catalog items we got for this version.
game = self.remote_game(server, token)
self.start_batch()
for item in game.get_items(self.game, self.version):
if item.type == "appealcard":
Convert most of the format() string calls to f-strings using libcst. Exact commands run were: python3 -m libcst.tool codemod convert_format_to_fstring.ConvertFormatStringCommand . --no-format python3 setup.py build_ext --inplace
2020-01-07 22:29:07 +01:00
print(f"New catalog entry for appeal card {item.id}")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
self.insert_catalog_entry(
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"appealcard",
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
item.id,
{},
)
elif item.type == "song_unlock":
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
print(
f"New catalog entry for {item.data.get_int('musicid')} chart {item.data.get_int('chart')}"
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
self.insert_catalog_entry(
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"song_unlock",
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
item.id,
{
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"blocks": item.data.get_int("blocks"),
"musicid": item.data.get_int("musicid"),
"chart": item.data.get_int("chart"),
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
)
self.finish_batch()
class ImportMuseca(ImportBase):
def __init__(
self,
Make Config a real class instead of a Dict, for safer access and defaults.
2021-08-20 06:43:59 +02:00
config: Config,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
version: str,
no_combine: bool,
update: bool,
) -> None:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
if version in ["1", "1+1/2", "plus"]:
MUSECA: Support Museca Plus modpack
2020-12-06 03:11:55 +01:00
actual_version = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"1": VersionConstants.MUSECA,
"1+1/2": VersionConstants.MUSECA_1_PLUS,
"plus": VersionConstants.MUSECA_1_PLUS
+ DBConstants.OMNIMIX_VERSION_BUMP,
MUSECA: Support Museca Plus modpack
2020-12-06 03:11:55 +01:00
}.get(version, -1)
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
if actual_version in [
VersionConstants.MUSECA,
VersionConstants.MUSECA_1_PLUS,
VersionConstants.MUSECA_1_PLUS + DBConstants.OMNIMIX_VERSION_BUMP,
]:
MUSECA: Support Museca Plus modpack
2020-12-06 03:11:55 +01:00
self.charts = [0, 1, 2, 3]
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
else:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
raise Exception(
"Unsupported Museca version, expected one of the following: 1, 1+1/2, plus!"
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
super().__init__(
config, GameConstants.MUSECA, actual_version, no_combine, update
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
def import_music_db(self, xmlfile: str) -> None:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
with open(xmlfile, "rb") as fp:
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# This is gross, but elemtree won't do it for us so whatever
bytedata = fp.read()
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
strdata = bytedata.decode("shift_jisx0213")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
root = ET.fromstring(strdata)
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
for music_entry in root.findall("music"):
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Grab the ID
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
songid = int(music_entry.attrib["id"])
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
title = None
artist = None
bpm_min = None
bpm_max = None
limited = [0, 0, 0, 0]
difficulties = [0, 0, 0, 0]
# Find normal info about the song
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
info = music_entry.find("info")
title = info.find("title_name").text
artist = info.find("artist_name").text
bpm_min = float(info.find("bpm_min").text) / 100.0
bpm_max = float(info.find("bpm_max").text) / 100.0
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Grab valid difficulties
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
for difficulty in music_entry.find("difficulty"):
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Figure out the actual difficulty
offset = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"novice": 0,
"advanced": 1,
"exhaust": 2,
"infinite": 3,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
}.get(difficulty.tag)
if offset is None:
continue
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
difficulties[offset] = int(difficulty.find("difnum").text)
limited[offset] = int(difficulty.find("limited").text)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Import it
self.start_batch()
for chart in self.charts:
# First, try to find in the DB from another version
old_id = self.get_music_id_for_song(songid, chart)
if self.no_combine or old_id is None:
# Insert original
Convert most of the format() string calls to f-strings using libcst. Exact commands run were: python3 -m libcst.tool codemod convert_format_to_fstring.ConvertFormatStringCommand . --no-format python3 setup.py build_ext --inplace
2020-01-07 22:29:07 +01:00
print(f"New entry for {songid} chart {chart}")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
next_id = self.get_next_music_id()
else:
# Insert pointing at same ID so scores transfer
Convert most of the format() string calls to f-strings using libcst. Exact commands run were: python3 -m libcst.tool codemod convert_format_to_fstring.ConvertFormatStringCommand . --no-format python3 setup.py build_ext --inplace
2020-01-07 22:29:07 +01:00
print(f"Reused entry for {songid} chart {chart}")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
next_id = old_id
data = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"limited": limited[chart],
"difficulty": difficulties[chart],
"bpm_min": bpm_min,
"bpm_max": bpm_max,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
}
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
self.insert_music_id_for_song(
next_id, songid, chart, title, artist, None, data
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
self.finish_batch()
def import_from_server(self, server: str, token: str) -> None:
# First things first, lets try to import the music DB. We want to make
# sure that even if the server doesn't respond right, we have a chart
# entry for every chart for each song we're importing.
music = self.remote_music(server, token)
music_lut: Dict[int, Dict[int, Song]] = {}
for entry in music.get_all_songs(self.game, self.version):
if entry.id not in music_lut:
music_lut[entry.id] = {
chart: Song(
entry.game,
entry.version,
entry.id,
chart,
entry.name,
entry.artist,
entry.genre,
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
{},
)
for chart in self.charts
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
}
music_lut[entry.id][entry.chart] = entry
# Import it
for _, songs in music_lut.items():
self.start_batch()
for _, song in songs.items():
# First, try to find in the DB from another version
old_id = self.get_music_id_for_song(song.id, song.chart)
if self.no_combine or old_id is None:
# Insert original
Convert most of the format() string calls to f-strings using libcst. Exact commands run were: python3 -m libcst.tool codemod convert_format_to_fstring.ConvertFormatStringCommand . --no-format python3 setup.py build_ext --inplace
2020-01-07 22:29:07 +01:00
print(f"New entry for {song.id} chart {song.chart}")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
next_id = self.get_next_music_id()
else:
# Insert pointing at same ID so scores transfer
Convert most of the format() string calls to f-strings using libcst. Exact commands run were: python3 -m libcst.tool codemod convert_format_to_fstring.ConvertFormatStringCommand . --no-format python3 setup.py build_ext --inplace
2020-01-07 22:29:07 +01:00
print(f"Reused entry for {song.id} chart {song.chart}")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
next_id = old_id
data = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"limited": song.data.get_int("limited"),
"difficulty": song.data.get_int("difficulty"),
"bpm_min": song.data.get_int("bpm_min"),
"bpm_max": song.data.get_int("bpm_max"),
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
}
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
self.insert_music_id_for_song(
next_id, song.id, song.chart, song.name, song.artist, None, data
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
self.finish_batch()
class ImportReflecBeat(ImportBase):
def __init__(
self,
Make Config a real class instead of a Dict, for safer access and defaults.
2021-08-20 06:43:59 +02:00
config: Config,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
version: str,
no_combine: bool,
update: bool,
) -> None:
# We always have 4 charts, even if we're importing from Colette and below,
# so that we guarantee a stable song ID. We'll be in trouble if Reflec
# ever adds a fifth chart.
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
if version in ["1", "2", "3", "4", "5", "6"]:
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
actual_version = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"1": VersionConstants.REFLEC_BEAT,
"2": VersionConstants.REFLEC_BEAT_LIMELIGHT,
"3": VersionConstants.REFLEC_BEAT_COLETTE,
"4": VersionConstants.REFLEC_BEAT_GROOVIN,
"5": VersionConstants.REFLEC_BEAT_VOLZZA,
"6": VersionConstants.REFLEC_BEAT_VOLZZA_2,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
}[version]
self.charts = [0, 1, 2, 3]
else:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
raise Exception(
"Unsupported ReflecBeat version, expected one of the following: 1, 2, 3, 4, 5, 6"
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
super().__init__(
config, GameConstants.REFLEC_BEAT, actual_version, no_combine, update
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
def scrape(self, infile: str) -> List[Dict[str, Any]]:
with open(infile, mode="rb") as myfile:
data = myfile.read()
myfile.close()
if self.version == VersionConstants.REFLEC_BEAT:
# Based on KBR:A:A:A:2011112300
offset = 0xBFBD0
stride = 280
max_songs = 93
max_difficulties = 3
song_offset = 0x4C
song_length = 0x40
# Artists aren't included in this mix.
artist_offset = None
artist_length = None
chart_offset = 0xD5
chart_length = 0x20
difficulties_offset = 0xD2
elif self.version == VersionConstants.REFLEC_BEAT_LIMELIGHT:
# Based on LBR:A:A:A:2012082900
offset = 0x132C48
stride = 220
max_songs = 191
max_difficulties = 3
song_offset = 0x4C
song_length = 0x40
# Artists aren't included in this mix.
artist_offset = None
artist_length = None
chart_offset = 0x9B
chart_length = 0x20
difficulties_offset = 0x98
elif self.version == VersionConstants.REFLEC_BEAT_COLETTE:
# Based on MBR:J:A:A:2014011600
offset = 0x1E6880
stride = 468
max_songs = 443
max_difficulties = 3
song_offset = 0x34
song_length = 0x80
artist_offset = 0xB4
artist_length = 0x80
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
chart_offset = 0x1B4
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
chart_length = 0x20
difficulties_offset = 0x1A8
elif self.version == VersionConstants.REFLEC_BEAT_GROOVIN:
# Based on MBR:J:A:A:2015102100
offset = 0x212EC0
stride = 524
max_songs = 698
max_difficulties = 4
song_offset = 0x3C
song_length = 0x80
artist_offset = 0xBC
artist_length = 0x80
chart_offset = 0x1E8
chart_length = 0x20
difficulties_offset = 0x1D0
elif self.version == VersionConstants.REFLEC_BEAT_VOLZZA:
# Based on MBR:J:A:A:2016030200
offset = 0x1A0EC8
stride = 552
max_songs = 805
max_difficulties = 4
song_offset = 0x38
song_length = 0x80
artist_offset = 0xB8
artist_length = 0x80
chart_offset = 0x1E4
chart_length = 0x20
difficulties_offset = 0x1CC
elif self.version == VersionConstants.REFLEC_BEAT_VOLZZA_2:
# Based on MBR:J:A:A:2016100400
offset = 0x1CBC68
stride = 552
max_songs = 850
max_difficulties = 4
song_offset = 0x38
song_length = 0x80
artist_offset = 0xB8
artist_length = 0x80
chart_offset = 0x1E4
chart_length = 0x20
difficulties_offset = 0x1CC
else:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
raise Exception(f"Unsupported ReflecBeat version {self.version}")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
def convert_string(inb: bytes) -> str:
end = None
for i in range(len(inb)):
if inb[i] == 0:
end = i
break
if end is None:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
raise Exception("Invalid string!")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if end == 0:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
return ""
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
return inb[:end].decode("shift_jisx0213")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
def convert_version(songid: int, folder: int) -> int:
if self.version == VersionConstants.REFLEC_BEAT_VOLZZA_2:
# Reflec Volzza 2 appears from network and DLL perspective to be identical
# to Volzza 1, including what version the game thinks it is for songs. So,
# hard code the new song IDs so we can show the difference on the frontend.
if folder == 5:
if songid in [733, 760, 772, 773, 774, 782, 785, 786]:
return 6
if songid >= 788:
return 6
return folder
songs = []
for i in range(max_songs):
start = offset + (stride * i)
end = start + stride
songdata = data[start:end]
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
title = convert_string(songdata[song_offset : (song_offset + song_length)])
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if artist_offset is None:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
artist = ""
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
else:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
artist = convert_string(
songdata[artist_offset : (artist_offset + artist_length)]
)
if title == "" and artist == "":
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
continue
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
songid = struct.unpack("<I", songdata[0:4])[0]
chart = convert_string(
songdata[chart_offset : (chart_offset + chart_length)]
)
difficulties = [
d
for d in songdata[
difficulties_offset : (difficulties_offset + max_difficulties)
]
]
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
difficulties = [0 if d == 255 else d for d in difficulties]
folder = convert_version(songid, int(chart[0]))
while len(difficulties) < 4:
difficulties.append(0)
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
songs.append(
{
"id": songid,
"title": title,
"artist": artist,
"chartid": chart[:4],
"difficulties": difficulties,
"folder": folder,
}
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
return songs
def lookup(self, server: str, token: str) -> List[Dict[str, Any]]:
# Grab music info from remote server
music = self.remote_music(server, token)
songs = music.get_all_songs(self.game, self.version)
lut: Dict[int, Dict[str, Any]] = {}
for song in songs:
if song.id not in lut:
lut[song.id] = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"id": song.id,
"title": song.name,
"artist": song.artist,
"chartid": song.data.get_str("chart_id"),
"difficulties": [0] * len(self.charts),
"folder": song.data.get_int("folder"),
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
}
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
lut[song.id]["difficulties"][song.chart] = song.data.get_int("difficulty")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Return the reassembled data
return [val for _, val in lut.items()]
def import_music_db(self, songs: List[Dict[str, Any]]) -> None:
for song in songs:
self.start_batch()
for chart in self.charts:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
songid = song["id"]
chartid = song["chartid"]
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# ReflecBeat re-numbers some of their songs and overlaps with IDs from older
# versions, so we need to keep a virtual mapping similar to DDR. Its not good
# enough to just do title/artist because Reflec also has revival charts that
# are named the same. Luckily we have internal chart ID to map on!
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
old_id = self.get_music_id_for_song_data(
None, None, chartid, chart, version=0
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if self.no_combine or old_id is None:
# Insert original
Convert most of the format() string calls to f-strings using libcst. Exact commands run were: python3 -m libcst.tool codemod convert_format_to_fstring.ConvertFormatStringCommand . --no-format python3 setup.py build_ext --inplace
2020-01-07 22:29:07 +01:00
print(f"New entry for {songid} chart {chart}")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
next_id = self.get_next_music_id()
else:
# Insert pointing at same ID so scores transfer
Convert most of the format() string calls to f-strings using libcst. Exact commands run were: python3 -m libcst.tool codemod convert_format_to_fstring.ConvertFormatStringCommand . --no-format python3 setup.py build_ext --inplace
2020-01-07 22:29:07 +01:00
print(f"Reused entry for {songid} chart {chart}")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
next_id = old_id
if old_id is None:
# Add the virtual music entry we talked about above. Use the song ID when we discover
# the song, plus the folder as a way to make them unique.
self.insert_music_id_for_song(
next_id,
self.version * 10000 + songid,
chart,
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
song["title"],
song["artist"],
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
chartid, # Chart goes into genre for reflec, so we can handle revival charts
{
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"difficulty": song["difficulties"][chart],
"folder": song["folder"],
"chart_id": chartid,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
version=0,
)
else:
if self.update:
# Force a folder/difficulty update for this song.
self.update_metadata_for_music_id(
old_id,
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
song["title"],
song["artist"],
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
chartid, # Chart goes into genre for reflec, so we can handle revival charts
{
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"difficulty": song["difficulties"][chart],
"folder": song["folder"],
"chart_id": chartid,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
version=0,
)
# Add the normal entry so the game finds the song.
self.insert_music_id_for_song(
next_id,
songid,
chart,
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
song["title"],
song["artist"],
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
None, # Reflec Beat has no genres for real songs.
{
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"difficulty": song["difficulties"][chart],
"folder": song["folder"],
"chart_id": chartid,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
},
)
self.finish_batch()
class ImportDanceEvolution(ImportBase):
def __init__(
self,
Make Config a real class instead of a Dict, for safer access and defaults.
2021-08-20 06:43:59 +02:00
config: Config,
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
version: str,
no_combine: bool,
update: bool,
) -> None:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
if version in ["1"]:
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
actual_version = 1
else:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
raise Exception(
"Unsupported Dance Evolution version, expected one of the following: 1"
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
super().__init__(
config, GameConstants.DANCE_EVOLUTION, actual_version, no_combine, update
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
def scrape(self, infile: str) -> List[Dict[str, Any]]:
with open(infile, mode="rb") as myfile:
data = myfile.read()
myfile.close()
arc = ARC(data)
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
data = arc.read_file("data/song/song_params.plist")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# First, do a header check like the game does
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
if data[0:4] != b"MS02":
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
raise Exception("Invalid song params file!")
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
if data[4:6] not in [b"BE", b"LE"]:
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
raise Exception("Invalid song params file!")
def get_string(offset: int, default: Optional[str] = None) -> str:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
lut_offset = struct.unpack(">I", data[(offset) : (offset + 4)])[0]
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if lut_offset == 0:
if default is None:
raise Exception("Expecting a string, got empty!")
return default
length = 0
while data[lut_offset + length] != 0:
length += 1
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
return (
data[lut_offset : (lut_offset + length)]
.decode("utf-8")
.replace("\n", " ")
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
def get_int(offset: int) -> int:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
return struct.unpack(">I", data[(offset) : (offset + 4)])[0]
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Now, make sure we know how long the file is
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
numsongs = struct.unpack(">I", data[8:12])[0]
filelen = struct.unpack(">I", data[12:16])[0]
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if filelen != len(data):
raise Exception("Invalid song params file!")
# Now, extract the meaningful data for each song
retval = []
for i in range(numsongs):
offset = (i * 128) + 16
songcode = get_string(offset + 0) # noqa: F841
songres1 = get_string(offset + 4) # noqa: F841
songres2 = get_string(offset + 8) # noqa: F841
bpm_min = get_int(offset + 12)
bpm_max = get_int(offset + 16)
copyright = get_string(offset + 24, "")
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
title = get_string(offset + 52, "Unknown song")
artist = get_string(offset + 56, "Unknown artist")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
level = get_int(offset + 64)
charares1 = get_string(offset + 72) # noqa: F841
charares2 = get_string(offset + 76) # noqa: F841
kana_sort = get_string(offset + 108)
flag1 = data[offset + 33] != 0x00 # noqa: F841
flag2 = data[offset + 34] == 0x01 # noqa: F841
flag3 = data[offset + 34] == 0x02 # noqa: F841
flag4 = data[offset + 116] != 0x00 # noqa: F841
# TODO: Get the real music ID from the data, once we have in-game traffic.
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
retval.append(
{
"id": i,
"title": title,
"artist": artist,
"copyright": copyright or None,
"sort_key": kana_sort,
"bpm_min": bpm_min,
"bpm_max": bpm_max,
"level": level,
}
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
return retval
def lookup(self, server: str, token: str) -> List[Dict[str, Any]]:
# TODO: We never got far enough to support DanEvo in the server, or
# specify it in BEMAPI. So this is a dead function for now, but maybe
# some year in the future I'll be able to support this.
return []
def import_music_db(self, songs: List[Dict[str, Any]]) -> None:
for song in songs:
# Import it
self.start_batch()
# First, try to find in the DB from another version
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
old_id = self.get_music_id_for_song(song["id"], 0)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if self.no_combine or old_id is None:
# Insert original
Convert most of the format() string calls to f-strings using libcst. Exact commands run were: python3 -m libcst.tool codemod convert_format_to_fstring.ConvertFormatStringCommand . --no-format python3 setup.py build_ext --inplace
2020-01-07 22:29:07 +01:00
print(f"New entry for {song['id']} chart {0}")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
next_id = self.get_next_music_id()
else:
# Insert pointing at same ID so scores transfer
Convert most of the format() string calls to f-strings using libcst. Exact commands run were: python3 -m libcst.tool codemod convert_format_to_fstring.ConvertFormatStringCommand . --no-format python3 setup.py build_ext --inplace
2020-01-07 22:29:07 +01:00
print(f"Reused entry for {song['id']} chart {0}")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
next_id = old_id
data = {
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"level": song["level"],
"bpm_min": song["bpm_min"],
"bpm_max": song["bpm_max"],
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
}
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
self.insert_music_id_for_song(
next_id, song["id"], 0, song["title"], song["artist"], None, data
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
self.finish_batch()
if __name__ == "__main__":
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
parser = argparse.ArgumentParser(description="Import Game Music DB")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
parser.add_argument(
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"--series",
action="store",
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
type=str,
required=True,
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
help="The game series we are importing.",
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
)
parser.add_argument(
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"--version",
dest="version",
action="store",
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
type=str,
required=True,
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
help="The game version we are importing.",
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
)
parser.add_argument(
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"--csv",
dest="csv",
action="store",
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
type=str,
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
help="The CSV file to read, for applicable games.",
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
)
parser.add_argument(
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"--tsv",
dest="tsv",
action="store",
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
type=str,
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
help="The TSV file to read, for applicable games.",
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
)
parser.add_argument(
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"--xml",
dest="xml",
action="store",
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
type=str,
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
help="The game XML file to read, for applicable games.",
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
)
parser.add_argument(
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"--bin",
dest="bin",
action="store",
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
type=str,
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
help="The game binary file to read, for applicable games.",
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
)
parser.add_argument(
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"--assets",
dest="assets",
action="store",
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
type=str,
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
help="The game sound assets directory, for applicable games.",
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
)
parser.add_argument(
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"--no-combine",
dest="no_combine",
action="store_true",
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
default=False,
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
help="Don't look for the same music ID in other versions.",
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
)
parser.add_argument(
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"--update",
dest="update",
action="store_true",
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
default=False,
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
help="Overwrite data with updated values when it already exists.",
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
)
parser.add_argument(
"--config",
type=str,
default="config.yaml",
help="Core configuration for importing to DB. Defaults to 'config.yaml'.",
)
parser.add_argument(
"--server",
type=str,
help="The remote BEMAPI server to read data from.",
)
parser.add_argument(
"--token",
type=str,
help="The access token to use with the remote BEMAPI server.",
)
# Parse args, validate invariants.
args = parser.parse_args()
if (args.token and not args.server) or (args.server and not args.token):
raise Exception("Must specify both --server and --token together!")
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
if (args.csv or args.tsv or args.xml or args.bin or args.assets) and (
args.server or args.token
):
raise Exception(
"Cannot specify both a remote server and a local file to read from!"
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# Load the config so we can talk to the server
Make Config a real class instead of a Dict, for safer access and defaults.
2021-08-20 06:43:59 +02:00
config = Config()
Unify config loading for remaining utilities that read the same server.yaml.
2021-08-20 06:43:36 +02:00
load_config(args.config, config)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
Fix a few places using enums directly instead of their value, formalize 'support' entry in config.
2021-08-19 21:25:10 +02:00
series = None
try:
series = GameConstants(args.series)
except ValueError:
pass
if series == GameConstants.POPN_MUSIC:
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
popn = ImportPopn(config, args.version, args.no_combine, args.update)
if args.bin:
songs = popn.scrape(args.bin)
elif args.server and args.token:
songs = popn.lookup(args.server, args.token)
else:
raise Exception(
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"No game DLL provided and no remote server specified! Please "
+ "provide either a --bin or a --server and --token option!"
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
)
popn.import_music_db(songs)
popn.close()
Fix a few places using enums directly instead of their value, formalize 'support' entry in config.
2021-08-19 21:25:10 +02:00
elif series == GameConstants.JUBEAT:
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
jubeat = ImportJubeat(config, args.version, args.no_combine, args.update)
if args.tsv is not None:
# Special case for Jubeat, grab the title/artist metadata that was
# hand-populated since its not in the music DB.
jubeat.import_metadata(args.tsv)
else:
Teach network how to import/export Jubeat emblem catalog.
2020-04-24 21:20:27 +02:00
# Normal case, doing a music DB or emblem import.
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if args.xml is not None:
Teach network how to import/export Jubeat emblem catalog.
2020-04-24 21:20:27 +02:00
songs, emblems = jubeat.scrape(args.xml)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
elif args.server and args.token:
Teach network how to import/export Jubeat emblem catalog.
2020-04-24 21:20:27 +02:00
songs, emblems = jubeat.lookup(args.server, args.token)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
else:
raise Exception(
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"No music_info.xml or TSV provided and no remote server specified! Please "
+ "provide either a --xml, --tsv or a --server and --token option!"
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
)
jubeat.import_music_db(songs)
Teach network how to import/export Jubeat emblem catalog.
2020-04-24 21:20:27 +02:00
jubeat.import_emblems(emblems)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
jubeat.close()
Fix a few places using enums directly instead of their value, formalize 'support' entry in config.
2021-08-19 21:25:10 +02:00
elif series == GameConstants.IIDX:
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
iidx = ImportIIDX(config, args.version, args.no_combine, args.update)
if args.tsv is not None:
# Special case for IIDX, grab the title/artist metadata that was
# wrong in the music DB, and correct it.
iidx.import_metadata(args.tsv)
else:
# Normal case, doing a music DB import.
if args.bin is not None:
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
songs, qpros = iidx.scrape(args.bin, args.assets)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
elif args.server and args.token:
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
songs, qpros = iidx.lookup(args.server, args.token)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
else:
raise Exception(
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"No music_data.bin or TSV provided and no remote server specified! Please "
+ "provide either a --bin, --tsv or a --server and --token option!"
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
)
iidx.import_music_db(songs)
iidx: Support cannonballers and rootage. Add stubs for heroic verse and bistrover
2021-05-17 23:51:26 +02:00
iidx.import_qpros(qpros)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
iidx.close()
Fix a few places using enums directly instead of their value, formalize 'support' entry in config.
2021-08-19 21:25:10 +02:00
elif series == GameConstants.DDR:
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
ddr = ImportDDR(config, args.version, args.no_combine, args.update)
if args.server and args.token:
songs = ddr.lookup(args.server, args.token)
else:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
if args.version == "16":
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if args.bin is None:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
raise Exception("No startup.arc provided!")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# DDR Ace has a different format altogether
songs = ddr.parse_xml(args.bin)
else:
if args.bin is None:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
raise Exception("No game DLL provided!")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if args.xml is None:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
raise Exception("No game music XML provided!")
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
# DDR splits the music DB between the DLL and external XML
# (Why??), so we must first scrape then hydrate with extra
# data to get the full DB.
songs = ddr.scrape(args.bin)
songs = ddr.hydrate(songs, args.xml)
ddr.import_music_db(songs)
ddr.close()
Fix a few places using enums directly instead of their value, formalize 'support' entry in config.
2021-08-19 21:25:10 +02:00
elif series == GameConstants.SDVX:
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
sdvx = ImportSDVX(config, args.version, args.no_combine, args.update)
if args.server and args.token:
sdvx.import_from_server(args.server, args.token)
else:
if args.xml is None and args.bin is None and args.csv is None:
raise Exception(
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"No XML file or game DLL or appeal card CSV provided and "
+ "no remote server specified! Please provide either a --xml, "
+ "--bin, --csv or a --server and --token option!"
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
)
if args.xml is not None:
sdvx.import_music_db_or_appeal_cards(args.xml)
if args.bin is not None:
sdvx.import_catalog(args.bin)
if args.csv is not None:
sdvx.import_appeal_cards(args.csv)
sdvx.close()
Fix a few places using enums directly instead of their value, formalize 'support' entry in config.
2021-08-19 21:25:10 +02:00
elif series == GameConstants.MUSECA:
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
museca = ImportMuseca(config, args.version, args.no_combine, args.update)
if args.server and args.token:
museca.import_from_server(args.server, args.token)
elif args.xml is not None:
museca.import_music_db(args.xml)
else:
raise Exception(
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"No music-info.xml provided and no remote server specified! "
+ "Please provide either a --xml or a --server and --token option!"
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
)
museca.close()
Fix a few places using enums directly instead of their value, formalize 'support' entry in config.
2021-08-19 21:25:10 +02:00
elif series == GameConstants.REFLEC_BEAT:
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
reflec = ImportReflecBeat(config, args.version, args.no_combine, args.update)
if args.bin is not None:
songs = reflec.scrape(args.bin)
elif args.server and args.token:
songs = reflec.lookup(args.server, args.token)
else:
raise Exception(
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"No game DLL provided and no remote server specified! "
+ "Please provide either a --bin or a --server and --token option!"
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
)
reflec.import_music_db(songs)
reflec.close()
Fix a few places using enums directly instead of their value, formalize 'support' entry in config.
2021-08-19 21:25:10 +02:00
elif series == GameConstants.DANCE_EVOLUTION:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
danevo = ImportDanceEvolution(
config, args.version, args.no_combine, args.update
)
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
if args.server and args.token:
songs = danevo.lookup(args.server, args.token)
elif args.bin is not None:
songs = danevo.scrape(args.bin)
else:
raise Exception(
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
"No resource_lists.arc provided and no remote server "
+ "specified! Please provide either a --bin or a "
+ "--server and --token option!",
Initial commit of BEMANI Utilities to GitHub.
2019-12-08 22:43:49 +01:00
)
danevo.import_music_db(songs)
danevo.close()
else:
Format code with black, include wrapper script to handle that, update linter checks to accomodate, fix a few miscelaneous type errors.
2022-10-15 20:56:30 +02:00
raise Exception("Unsupported game series!")
Copy Permalink