From 1205e4dcbac18b8dce87e5a60dde228b664f4125 Mon Sep 17 00:00:00 2001 From: Jennifer Taylor Date: Sat, 29 Jul 2023 21:51:03 +0000 Subject: [PATCH] Update read.py to support possibly scraping from multiple different Qpro databases in different DLLs given a single version. --- README.md | 2 +- bemani/utils/read.py | 457 ++++++++++++++++++++++++++++--------------- 2 files changed, 298 insertions(+), 161 deletions(-) diff --git a/README.md b/README.md index 5020578..4398385 100644 --- a/README.md +++ b/README.md @@ -527,7 +527,7 @@ For Qpro editing to work properly, you will also need to import the Qpro databas the mix you wish to support. This does not need to be run separately for omnimix versions, the base version Qpros will be used for both that version and the omnimix of that version. This network supports editing Qpros for versions 20-26 so you will want to run this command -once for every version, giving the correcct DLL file: +once for every version, giving the correct DLL file: ``` ./read --config config/server.yaml --series iidx --version 22 --bin bm2dx.dll diff --git a/bemani/utils/read.py b/bemani/utils/read.py index 0fd2925..ca26a82 100644 --- a/bemani/utils/read.py +++ b/bemani/utils/read.py @@ -1940,6 +1940,43 @@ class ImportJubeat(ImportBase): self.finish_batch() +class IIDXScrapeConfiguration: + def __init__( + self, + *, + version: str, + stride: int, + qp_head_offset: int, + qp_head_length: int, + qp_hair_offset: int, + qp_hair_length: int, + qp_face_offset: int, + qp_face_length: int, + qp_hand_offset: int, + qp_hand_length: int, + qp_body_offset: int, + qp_body_length: int, + filename_offset: int, + qpro_id_offset: Optional[int], + packedfmt: str, + ) -> None: + self.version = version + self.stride = stride + self.qp_head_offset = qp_head_offset + self.qp_head_length = qp_head_length + self.qp_hair_offset = qp_hair_offset + self.qp_hair_length = qp_hair_length + self.qp_face_offset = qp_face_offset + self.qp_face_length = qp_face_length + self.qp_hand_offset = qp_hand_offset + self.qp_hand_length = qp_hand_length + self.qp_body_offset = qp_body_offset + self.qp_body_length = qp_body_length + self.filename_offset = filename_offset + self.qpro_id_offset = qpro_id_offset + self.packedfmt = packedfmt + + class ImportIIDX(ImportBase): # Tutorial charts that shouldn't be on the UI BANNED_CHARTS = [ @@ -2158,8 +2195,10 @@ class ImportIIDX(ImportBase): except BaseException: import_qpros = False # if it failed then we're reading a music db file, not the executable - songs: List[Dict[str, Any]] = [] if not import_qpros: + # We should assume this is a standard music_data.bin, and read the soungs out of it. + songs: List[Dict[str, Any]] = [] + musicdb = IIDXMusicDB(binarydata) for song in musicdb.songs: bpm = (0, 0) @@ -2229,168 +2268,266 @@ class ImportIIDX(ImportBase): } ) - qpros: List[Dict[str, Any]] = [] - if self.version == VersionConstants.IIDX_TRICORO: - # Based on LDJ:J:A:A:2013090900 - stride = 4 - qp_head_offset = 0x1CCB18 # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in - qp_head_length = 79 # the qpro id numbers, it's best to store them as separate types in the catalog as well. - qp_hair_offset = 0x1CCC58 - qp_hair_length = 103 - qp_face_offset = 0x1CCDF8 - qp_face_length = 50 - qp_hand_offset = 0x1CCEC0 - qp_hand_length = 103 - qp_body_offset = 0x1CD060 - qp_body_length = 106 - filename_offset = 0 - packedfmt = "I" # filename - if self.version == VersionConstants.IIDX_SPADA: - # Based on LDJ:J:A:A:2014071600 - stride = 4 - qp_head_offset = 0x213B50 # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in - qp_head_length = 125 # the qpro id numbers, it's best to store them as separate types in the catalog as well. - qp_hair_offset = 0x213D48 - qp_hair_length = 126 - qp_face_offset = 0x213F40 - qp_face_length = 72 - qp_hand_offset = 0x214060 - qp_hand_length = 135 - qp_body_offset = 0x214280 - qp_body_length = 135 - filename_offset = 0 - packedfmt = "I" # filename - if self.version == VersionConstants.IIDX_PENDUAL: - # Based on LDJ:J:A:A:2015080500 - stride = 4 - qp_head_offset = 0x1D5228 # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in - qp_head_length = 163 # the qpro id numbers, it's best to store them as separate types in the catalog as well. - qp_hair_offset = 0x1D54B8 - qp_hair_length = 182 - qp_face_offset = 0x1D5790 - qp_face_length = 106 - qp_hand_offset = 0x1D5938 - qp_hand_length = 184 - qp_body_offset = 0x1D5C18 - qp_body_length = 191 - filename_offset = 0 - packedfmt = "I" # filename - if self.version == VersionConstants.IIDX_COPULA: - # Based on LDJ:J:A:A:2016083100 - stride = 8 - qp_head_offset = 0x12F9D8 # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in - qp_head_length = 186 # the qpro id numbers, it's best to store them as separate types in the catalog as well. - qp_hair_offset = 0x12FFA8 - qp_hair_length = 202 - qp_face_offset = 0x1305F8 - qp_face_length = 126 - qp_hand_offset = 0x1309E8 - qp_hand_length = 206 - qp_body_offset = 0x131058 - qp_body_length = 211 - filename_offset = 0 - qpro_id_offset = 1 - packedfmt = "I" "I" # filename # string containing id and name of the part - if self.version == VersionConstants.IIDX_SINOBUZ: - # Based on LDJ:J:A:A:2017082800 - stride = 8 - qp_head_offset = 0x149F88 # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in - qp_head_length = 211 # the qpro id numbers, it's best to store them as separate types in the catalog as well. - qp_hair_offset = 0x14A620 - qp_hair_length = 245 - qp_face_offset = 0x14ADC8 - qp_face_length = 152 - qp_hand_offset = 0x14B288 - qp_hand_length = 236 - qp_body_offset = 0x14B9E8 - qp_body_length = 256 - filename_offset = 0 - qpro_id_offset = 1 - packedfmt = "I" "I" # filename # string containing id and name of the part - if self.version == VersionConstants.IIDX_CANNON_BALLERS: - # Based on LDJ:J:A:A:2018091900 - stride = 16 - qp_head_offset = 0x2339E0 # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in - qp_head_length = 231 # the qpro id numbers, it's best to store them as separate types in the catalog as well. - qp_hair_offset = 0x234850 - qp_hair_length = 267 - qp_face_offset = 0x235900 - qp_face_length = 173 - qp_hand_offset = 0x2363D0 - qp_hand_length = 261 - qp_body_offset = 0x237420 - qp_body_length = 282 - filename_offset = 0 - qpro_id_offset = 1 - packedfmt = "Q" "Q" # filename # string containing id and name of the part - if self.version == VersionConstants.IIDX_ROOTAGE: - # Based on LDJ:J:A:A:2019090200 - stride = 16 - qp_head_offset = 0x5065F0 # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in - qp_head_length = 259 # the qpro id numbers, it's best to store them as separate types in the catalog as well. - qp_hair_offset = 0x507620 - qp_hair_length = 288 - qp_face_offset = 0x508820 - qp_face_length = 193 - qp_hand_offset = 0x509430 - qp_hand_length = 287 - qp_body_offset = 0x50A620 - qp_body_length = 304 - filename_offset = 0 - qpro_id_offset = 1 - packedfmt = "Q" "Q" # filename # string containing id and name of the part - - def read_string(offset: int) -> str: - # First, translate load offset in memory to disk offset - offset = pe.virtual_to_physical(offset) - - # Now, grab bytes until we're null-terminated - bytestring = [] - while binarydata[offset] != 0: - bytestring.append(binarydata[offset]) - offset = offset + 1 - - # Its shift-jis encoded, so decode it now - return bytes(bytestring).decode("shift_jisx0213") - - def read_qpro_db(offset: int, length: int, qp_type: str) -> None: - for qpro_id in range(length): - chunkoffset = offset + (stride * qpro_id) - chunkdata = binarydata[chunkoffset : (chunkoffset + stride)] - unpacked = struct.unpack(packedfmt, chunkdata) - filename = read_string(unpacked[filename_offset]).replace("qp_", "") - remove = f"_{qp_type}.ifs" - filename = ( - filename.replace(remove, "") - .replace("_head1.ifs", "") - .replace("_head2.ifs", "") + # We only import one or the other here, I know its a weird function. + return songs, [] + else: + # We should assume that we want to extract QPro data from this DLL. + configurations: List[IIDXScrapeConfiguration] = [] + if self.version == VersionConstants.IIDX_TRICORO: + # Based on LDJ:J:A:A:2013090900 + configurations.append( + IIDXScrapeConfiguration( + version="LDJ:J:A:A:2013090900", + stride=4, + qp_head_offset=0x1CCB18, # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in + qp_head_length=79, # the qpro id numbers, it's best to store them as separate types in the catalog as well. + qp_hair_offset=0x1CCC58, + qp_hair_length=103, + qp_face_offset=0x1CCDF8, + qp_face_length=50, + qp_hand_offset=0x1CCEC0, + qp_hand_length=103, + qp_body_offset=0x1CD060, + qp_body_length=106, + filename_offset=0, + qpro_id_offset=None, + packedfmt="I", # filename + ) + ) + if self.version == VersionConstants.IIDX_SPADA: + # Based on LDJ:J:A:A:2014071600 + configurations.append( + IIDXScrapeConfiguration( + version="LDJ:J:A:A:2014071600", + stride=4, + qp_head_offset=0x213B50, # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in + qp_head_length=125, # the qpro id numbers, it's best to store them as separate types in the catalog as well. + qp_hair_offset=0x213D48, + qp_hair_length=126, + qp_face_offset=0x213F40, + qp_face_length=72, + qp_hand_offset=0x214060, + qp_hand_length=135, + qp_body_offset=0x214280, + qp_body_length=135, + filename_offset=0, + qpro_id_offset=None, + packedfmt="I", # filename + ) + ) + if self.version == VersionConstants.IIDX_PENDUAL: + # Based on LDJ:J:A:A:2015080500 + configurations.append( + IIDXScrapeConfiguration( + version="LDJ:J:A:A:2015080500", + stride=4, + qp_head_offset=0x1D5228, # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in + qp_head_length=163, # the qpro id numbers, it's best to store them as separate types in the catalog as well. + qp_hair_offset=0x1D54B8, + qp_hair_length=182, + qp_face_offset=0x1D5790, + qp_face_length=106, + qp_hand_offset=0x1D5938, + qp_hand_length=184, + qp_body_offset=0x1D5C18, + qp_body_length=191, + filename_offset=0, + qpro_id_offset=None, + packedfmt="I", # filename + ) + ) + if self.version == VersionConstants.IIDX_COPULA: + # Based on LDJ:J:A:A:2016083100 + configurations.append( + IIDXScrapeConfiguration( + version="LDJ:J:A:A:2016083100", + stride=8, + qp_head_offset=0x12F9D8, # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in + qp_head_length=186, # the qpro id numbers, it's best to store them as separate types in the catalog as well. + qp_hair_offset=0x12FFA8, + qp_hair_length=202, + qp_face_offset=0x1305F8, + qp_face_length=126, + qp_hand_offset=0x1309E8, + qp_hand_length=206, + qp_body_offset=0x131058, + qp_body_length=211, + filename_offset=0, + qpro_id_offset=1, + packedfmt="I" + "I", # filename # string containing id and name of the part + ) + ) + if self.version == VersionConstants.IIDX_SINOBUZ: + # Based on LDJ:J:A:A:2017082800 + configurations.append( + IIDXScrapeConfiguration( + version="LDJ:J:A:A:2017082800", + stride=8, + qp_head_offset=0x149F88, # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in + qp_head_length=211, # the qpro id numbers, it's best to store them as separate types in the catalog as well. + qp_hair_offset=0x14A620, + qp_hair_length=245, + qp_face_offset=0x14ADC8, + qp_face_length=152, + qp_hand_offset=0x14B288, + qp_hand_length=236, + qp_body_offset=0x14B9E8, + qp_body_length=256, + filename_offset=0, + qpro_id_offset=1, + packedfmt="I" + "I", # filename # string containing id and name of the part + ) + ) + if self.version == VersionConstants.IIDX_CANNON_BALLERS: + # Based on LDJ:J:A:A:2018091900 + configurations.append( + IIDXScrapeConfiguration( + version="LDJ:J:A:A:2018091900", + stride=16, + qp_head_offset=0x2339E0, # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in + qp_head_length=231, # the qpro id numbers, it's best to store them as separate types in the catalog as well. + qp_hair_offset=0x234850, + qp_hair_length=267, + qp_face_offset=0x235900, + qp_face_length=173, + qp_hand_offset=0x2363D0, + qp_hand_length=261, + qp_body_offset=0x237420, + qp_body_length=282, + filename_offset=0, + qpro_id_offset=1, + packedfmt="Q" + "Q", # filename # string containing id and name of the part + ) + ) + if self.version == VersionConstants.IIDX_ROOTAGE: + # Based on LDJ:J:A:A:2019090200 + configurations.append( + IIDXScrapeConfiguration( + version="LDJ:J:A:A:2019090200", + stride=16, + qp_head_offset=0x5065F0, # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in + qp_head_length=259, # the qpro id numbers, it's best to store them as separate types in the catalog as well. + qp_hair_offset=0x507620, + qp_hair_length=288, + qp_face_offset=0x508820, + qp_face_length=193, + qp_hand_offset=0x509430, + qp_hand_length=287, + qp_body_offset=0x50A620, + qp_body_length=304, + filename_offset=0, + qpro_id_offset=1, + packedfmt="Q" + "Q", # filename # string containing id and name of the part + ) ) - if self.version in [ - VersionConstants.IIDX_TRICORO, - VersionConstants.IIDX_SPADA, - VersionConstants.IIDX_PENDUAL, - ]: - name = filename # qpro names are not stored in these 3 games so use the identifier instead - else: - name = read_string(unpacked[qpro_id_offset])[ - 4: - ] # qpro name is stored in second string of form "000:name" - qproinfo = { - "identifier": filename, - "id": qpro_id, - "name": name, - "type": qp_type, - } - qpros.append(qproinfo) - if import_qpros: - read_qpro_db(qp_head_offset, qp_head_length, "head") - read_qpro_db(qp_hair_offset, qp_hair_length, "hair") - read_qpro_db(qp_face_offset, qp_face_length, "face") - read_qpro_db(qp_hand_offset, qp_hand_length, "hand") - read_qpro_db(qp_body_offset, qp_body_length, "body") + def read_string(offset: int) -> str: + # First, translate load offset in memory to disk offset + offset = pe.virtual_to_physical(offset) - return songs, qpros + # Now, grab bytes until we're null-terminated + bytestring = [] + while binarydata[offset] != 0: + bytestring.append(binarydata[offset]) + offset = offset + 1 + + # Its shift-jis encoded, so decode it now + return bytes(bytestring).decode("shift_jisx0213") + + def read_qpro_db( + qpros: List[Dict[str, Any]], + config: IIDXScrapeConfiguration, + offset: int, + length: int, + qp_type: str, + ) -> None: + for qpro_id in range(length): + chunkoffset = offset + (config.stride * qpro_id) + chunkdata = binarydata[chunkoffset : (chunkoffset + config.stride)] + unpacked = struct.unpack(config.packedfmt, chunkdata) + filename = read_string(unpacked[config.filename_offset]).replace( + "qp_", "" + ) + remove = f"_{qp_type}.ifs" + filename = ( + filename.replace(remove, "") + .replace("_head1.ifs", "") + .replace("_head2.ifs", "") + ) + if config.qpro_id_offset is None: + name = filename # qpro names are not stored in these games so use the identifier instead + else: + name = read_string(unpacked[config.qpro_id_offset])[ + 4: + ] # qpro name is stored in second string of form "000:name" + qproinfo = { + "identifier": filename, + "id": qpro_id, + "name": name, + "type": qp_type, + } + qpros.append(qproinfo) + + for config in configurations: + try: + print(f"Trying configuration for game version {config.version}...") + + qpros: List[Dict[str, Any]] = [] + + # For each category, attempt to append all the types. + read_qpro_db( + qpros, + config, + config.qp_head_offset, + config.qp_head_length, + "head", + ) + read_qpro_db( + qpros, + config, + config.qp_hair_offset, + config.qp_hair_length, + "hair", + ) + read_qpro_db( + qpros, + config, + config.qp_face_offset, + config.qp_face_length, + "face", + ) + read_qpro_db( + qpros, + config, + config.qp_hand_offset, + config.qp_hand_length, + "hand", + ) + read_qpro_db( + qpros, + config, + config.qp_body_offset, + config.qp_body_length, + "body", + ) + + # If we got here, that means we ran into no issues and didn't have to attempt another offset. + print("Successfully parsed game DB!") + + # We only import one or the other here, I know its a weird function. + return [], qpros + except UnicodeError: + # These offsets are possibly not correct, so try the next configuration. + print("Failed to parse game DB!") + pass + + raise Exception( + "Could not determine correct binary parser configuration for IIDX version {self.version}" + ) def lookup( self, server: str, token: str