Cool_Tools/bemaniutils
1
0
Fork 0
mirror of synced 2024-11-27 15:40:48 +01:00
Code Releases Activity

Update read.py to support possibly scraping from multiple different Qpro databases in different DLLs given a single version.

Browse Source
This commit is contained in:
Jennifer Taylor 2023-07-29 21:51:03 +00:00
parent 8477841101
commit 1205e4dcba
2 changed files with 298 additions and 161 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -527,7 +527,7 @@ For Qpro editing to work properly, you will also need to import the Qpro databas
the mix you wish to support. This does not need to be run separately for omnimix versions,
the base version Qpros will be used for both that version and the omnimix of that version.
This network supports editing Qpros for versions 20-26 so you will want to run this command
once for every version, giving the correcct DLL file:
once for every version, giving the correct DLL file:
```
./read --config config/server.yaml --series iidx --version 22 --bin bm2dx.dll

457
bemani/utils/read.py
View File

@ -1940,6 +1940,43 @@ class ImportJubeat(ImportBase):
self.finish_batch()
class IIDXScrapeConfiguration:
def __init__(
self,
*,
version: str,
stride: int,
qp_head_offset: int,
qp_head_length: int,
qp_hair_offset: int,
qp_hair_length: int,
qp_face_offset: int,
qp_face_length: int,
qp_hand_offset: int,
qp_hand_length: int,
qp_body_offset: int,
qp_body_length: int,
filename_offset: int,
qpro_id_offset: Optional[int],
packedfmt: str,
) -> None:
self.version = version
self.stride = stride
self.qp_head_offset = qp_head_offset
self.qp_head_length = qp_head_length
self.qp_hair_offset = qp_hair_offset
self.qp_hair_length = qp_hair_length
self.qp_face_offset = qp_face_offset
self.qp_face_length = qp_face_length
self.qp_hand_offset = qp_hand_offset
self.qp_hand_length = qp_hand_length
self.qp_body_offset = qp_body_offset
self.qp_body_length = qp_body_length
self.filename_offset = filename_offset
self.qpro_id_offset = qpro_id_offset
self.packedfmt = packedfmt
class ImportIIDX(ImportBase):
# Tutorial charts that shouldn't be on the UI
BANNED_CHARTS = [
@ -2158,8 +2195,10 @@ class ImportIIDX(ImportBase):
except BaseException:
import_qpros = False # if it failed then we're reading a music db file, not the executable
songs: List[Dict[str, Any]] = []
if not import_qpros:
# We should assume this is a standard music_data.bin, and read the soungs out of it.
songs: List[Dict[str, Any]] = []
musicdb = IIDXMusicDB(binarydata)
for song in musicdb.songs:
bpm = (0, 0)
@ -2229,168 +2268,266 @@ class ImportIIDX(ImportBase):
}
)
qpros: List[Dict[str, Any]] = []
if self.version == VersionConstants.IIDX_TRICORO:
# Based on LDJ:J:A:A:2013090900
stride = 4
qp_head_offset = 0x1CCB18 # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in
qp_head_length = 79 # the qpro id numbers, it's best to store them as separate types in the catalog as well.
qp_hair_offset = 0x1CCC58
qp_hair_length = 103
qp_face_offset = 0x1CCDF8
qp_face_length = 50
qp_hand_offset = 0x1CCEC0
qp_hand_length = 103
qp_body_offset = 0x1CD060
qp_body_length = 106
filename_offset = 0
packedfmt = "I" # filename
if self.version == VersionConstants.IIDX_SPADA:
# Based on LDJ:J:A:A:2014071600
stride = 4
qp_head_offset = 0x213B50 # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in
qp_head_length = 125 # the qpro id numbers, it's best to store them as separate types in the catalog as well.
qp_hair_offset = 0x213D48
qp_hair_length = 126
qp_face_offset = 0x213F40
qp_face_length = 72
qp_hand_offset = 0x214060
qp_hand_length = 135
qp_body_offset = 0x214280
qp_body_length = 135
filename_offset = 0
packedfmt = "I" # filename
if self.version == VersionConstants.IIDX_PENDUAL:
# Based on LDJ:J:A:A:2015080500
stride = 4
qp_head_offset = 0x1D5228 # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in
qp_head_length = 163 # the qpro id numbers, it's best to store them as separate types in the catalog as well.
qp_hair_offset = 0x1D54B8
qp_hair_length = 182
qp_face_offset = 0x1D5790
qp_face_length = 106
qp_hand_offset = 0x1D5938
qp_hand_length = 184
qp_body_offset = 0x1D5C18
qp_body_length = 191
filename_offset = 0
packedfmt = "I" # filename
if self.version == VersionConstants.IIDX_COPULA:
# Based on LDJ:J:A:A:2016083100
stride = 8
qp_head_offset = 0x12F9D8 # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in
qp_head_length = 186 # the qpro id numbers, it's best to store them as separate types in the catalog as well.
qp_hair_offset = 0x12FFA8
qp_hair_length = 202
qp_face_offset = 0x1305F8
qp_face_length = 126
qp_hand_offset = 0x1309E8
qp_hand_length = 206
qp_body_offset = 0x131058
qp_body_length = 211
filename_offset = 0
qpro_id_offset = 1
packedfmt = "I" "I" # filename # string containing id and name of the part
if self.version == VersionConstants.IIDX_SINOBUZ:
# Based on LDJ:J:A:A:2017082800
stride = 8
qp_head_offset = 0x149F88 # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in
qp_head_length = 211 # the qpro id numbers, it's best to store them as separate types in the catalog as well.
qp_hair_offset = 0x14A620
qp_hair_length = 245
qp_face_offset = 0x14ADC8
qp_face_length = 152
qp_hand_offset = 0x14B288
qp_hand_length = 236
qp_body_offset = 0x14B9E8
qp_body_length = 256
filename_offset = 0
qpro_id_offset = 1
packedfmt = "I" "I" # filename # string containing id and name of the part
if self.version == VersionConstants.IIDX_CANNON_BALLERS:
# Based on LDJ:J:A:A:2018091900
stride = 16
qp_head_offset = 0x2339E0 # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in
qp_head_length = 231 # the qpro id numbers, it's best to store them as separate types in the catalog as well.
qp_hair_offset = 0x234850
qp_hair_length = 267
qp_face_offset = 0x235900
qp_face_length = 173
qp_hand_offset = 0x2363D0
qp_hand_length = 261
qp_body_offset = 0x237420
qp_body_length = 282
filename_offset = 0
qpro_id_offset = 1
packedfmt = "Q" "Q" # filename # string containing id and name of the part
if self.version == VersionConstants.IIDX_ROOTAGE:
# Based on LDJ:J:A:A:2019090200
stride = 16
qp_head_offset = 0x5065F0 # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in
qp_head_length = 259 # the qpro id numbers, it's best to store them as separate types in the catalog as well.
qp_hair_offset = 0x507620
qp_hair_length = 288
qp_face_offset = 0x508820
qp_face_length = 193
qp_hand_offset = 0x509430
qp_hand_length = 287
qp_body_offset = 0x50A620
qp_body_length = 304
filename_offset = 0
qpro_id_offset = 1
packedfmt = "Q" "Q" # filename # string containing id and name of the part
def read_string(offset: int) -> str:
# First, translate load offset in memory to disk offset
offset = pe.virtual_to_physical(offset)
# Now, grab bytes until we're null-terminated
bytestring = []
while binarydata[offset] != 0:
bytestring.append(binarydata[offset])
offset = offset + 1
# Its shift-jis encoded, so decode it now
return bytes(bytestring).decode("shift_jisx0213")
def read_qpro_db(offset: int, length: int, qp_type: str) -> None:
for qpro_id in range(length):
chunkoffset = offset + (stride * qpro_id)
chunkdata = binarydata[chunkoffset : (chunkoffset + stride)]
unpacked = struct.unpack(packedfmt, chunkdata)
filename = read_string(unpacked[filename_offset]).replace("qp_", "")
remove = f"_{qp_type}.ifs"
filename = (
filename.replace(remove, "")
.replace("_head1.ifs", "")
.replace("_head2.ifs", "")
# We only import one or the other here, I know its a weird function.
return songs, []
else:
# We should assume that we want to extract QPro data from this DLL.
configurations: List[IIDXScrapeConfiguration] = []
if self.version == VersionConstants.IIDX_TRICORO:
# Based on LDJ:J:A:A:2013090900
configurations.append(
IIDXScrapeConfiguration(
version="LDJ:J:A:A:2013090900",
stride=4,
qp_head_offset=0x1CCB18, # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in
qp_head_length=79, # the qpro id numbers, it's best to store them as separate types in the catalog as well.
qp_hair_offset=0x1CCC58,
qp_hair_length=103,
qp_face_offset=0x1CCDF8,
qp_face_length=50,
qp_hand_offset=0x1CCEC0,
qp_hand_length=103,
qp_body_offset=0x1CD060,
qp_body_length=106,
filename_offset=0,
qpro_id_offset=None,
packedfmt="I", # filename
)
)
if self.version == VersionConstants.IIDX_SPADA:
# Based on LDJ:J:A:A:2014071600
configurations.append(
IIDXScrapeConfiguration(
version="LDJ:J:A:A:2014071600",
stride=4,
qp_head_offset=0x213B50, # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in
qp_head_length=125, # the qpro id numbers, it's best to store them as separate types in the catalog as well.
qp_hair_offset=0x213D48,
qp_hair_length=126,
qp_face_offset=0x213F40,
qp_face_length=72,
qp_hand_offset=0x214060,
qp_hand_length=135,
qp_body_offset=0x214280,
qp_body_length=135,
filename_offset=0,
qpro_id_offset=None,
packedfmt="I", # filename
)
)
if self.version == VersionConstants.IIDX_PENDUAL:
# Based on LDJ:J:A:A:2015080500
configurations.append(
IIDXScrapeConfiguration(
version="LDJ:J:A:A:2015080500",
stride=4,
qp_head_offset=0x1D5228, # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in
qp_head_length=163, # the qpro id numbers, it's best to store them as separate types in the catalog as well.
qp_hair_offset=0x1D54B8,
qp_hair_length=182,
qp_face_offset=0x1D5790,
qp_face_length=106,
qp_hand_offset=0x1D5938,
qp_hand_length=184,
qp_body_offset=0x1D5C18,
qp_body_length=191,
filename_offset=0,
qpro_id_offset=None,
packedfmt="I", # filename
)
)
if self.version == VersionConstants.IIDX_COPULA:
# Based on LDJ:J:A:A:2016083100
configurations.append(
IIDXScrapeConfiguration(
version="LDJ:J:A:A:2016083100",
stride=8,
qp_head_offset=0x12F9D8, # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in
qp_head_length=186, # the qpro id numbers, it's best to store them as separate types in the catalog as well.
qp_hair_offset=0x12FFA8,
qp_hair_length=202,
qp_face_offset=0x1305F8,
qp_face_length=126,
qp_hand_offset=0x1309E8,
qp_hand_length=206,
qp_body_offset=0x131058,
qp_body_length=211,
filename_offset=0,
qpro_id_offset=1,
packedfmt="I"
"I", # filename # string containing id and name of the part
)
)
if self.version == VersionConstants.IIDX_SINOBUZ:
# Based on LDJ:J:A:A:2017082800
configurations.append(
IIDXScrapeConfiguration(
version="LDJ:J:A:A:2017082800",
stride=8,
qp_head_offset=0x149F88, # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in
qp_head_length=211, # the qpro id numbers, it's best to store them as separate types in the catalog as well.
qp_hair_offset=0x14A620,
qp_hair_length=245,
qp_face_offset=0x14ADC8,
qp_face_length=152,
qp_hand_offset=0x14B288,
qp_hand_length=236,
qp_body_offset=0x14B9E8,
qp_body_length=256,
filename_offset=0,
qpro_id_offset=1,
packedfmt="I"
"I", # filename # string containing id and name of the part
)
)
if self.version == VersionConstants.IIDX_CANNON_BALLERS:
# Based on LDJ:J:A:A:2018091900
configurations.append(
IIDXScrapeConfiguration(
version="LDJ:J:A:A:2018091900",
stride=16,
qp_head_offset=0x2339E0, # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in
qp_head_length=231, # the qpro id numbers, it's best to store them as separate types in the catalog as well.
qp_hair_offset=0x234850,
qp_hair_length=267,
qp_face_offset=0x235900,
qp_face_length=173,
qp_hand_offset=0x2363D0,
qp_hand_length=261,
qp_body_offset=0x237420,
qp_body_length=282,
filename_offset=0,
qpro_id_offset=1,
packedfmt="Q"
"Q", # filename # string containing id and name of the part
)
)
if self.version == VersionConstants.IIDX_ROOTAGE:
# Based on LDJ:J:A:A:2019090200
configurations.append(
IIDXScrapeConfiguration(
version="LDJ:J:A:A:2019090200",
stride=16,
qp_head_offset=0x5065F0, # qpro body parts are stored in 5 separate arrays in the game data, since there can be collision in
qp_head_length=259, # the qpro id numbers, it's best to store them as separate types in the catalog as well.
qp_hair_offset=0x507620,
qp_hair_length=288,
qp_face_offset=0x508820,
qp_face_length=193,
qp_hand_offset=0x509430,
qp_hand_length=287,
qp_body_offset=0x50A620,
qp_body_length=304,
filename_offset=0,
qpro_id_offset=1,
packedfmt="Q"
"Q", # filename # string containing id and name of the part
)
)
if self.version in [
VersionConstants.IIDX_TRICORO,
VersionConstants.IIDX_SPADA,
VersionConstants.IIDX_PENDUAL,
]:
name = filename # qpro names are not stored in these 3 games so use the identifier instead
else:
name = read_string(unpacked[qpro_id_offset])[
4:
] # qpro name is stored in second string of form "000:name"
qproinfo = {
"identifier": filename,
"id": qpro_id,
"name": name,
"type": qp_type,
}
qpros.append(qproinfo)
if import_qpros:
read_qpro_db(qp_head_offset, qp_head_length, "head")
read_qpro_db(qp_hair_offset, qp_hair_length, "hair")
read_qpro_db(qp_face_offset, qp_face_length, "face")
read_qpro_db(qp_hand_offset, qp_hand_length, "hand")
read_qpro_db(qp_body_offset, qp_body_length, "body")
def read_string(offset: int) -> str:
# First, translate load offset in memory to disk offset
offset = pe.virtual_to_physical(offset)
return songs, qpros
# Now, grab bytes until we're null-terminated
bytestring = []
while binarydata[offset] != 0:
bytestring.append(binarydata[offset])
offset = offset + 1
# Its shift-jis encoded, so decode it now
return bytes(bytestring).decode("shift_jisx0213")
def read_qpro_db(
qpros: List[Dict[str, Any]],
config: IIDXScrapeConfiguration,
offset: int,
length: int,
qp_type: str,
) -> None:
for qpro_id in range(length):
chunkoffset = offset + (config.stride * qpro_id)
chunkdata = binarydata[chunkoffset : (chunkoffset + config.stride)]
unpacked = struct.unpack(config.packedfmt, chunkdata)
filename = read_string(unpacked[config.filename_offset]).replace(
"qp_", ""
)
remove = f"_{qp_type}.ifs"
filename = (
filename.replace(remove, "")
.replace("_head1.ifs", "")
.replace("_head2.ifs", "")
)
if config.qpro_id_offset is None:
name = filename # qpro names are not stored in these games so use the identifier instead
else:
name = read_string(unpacked[config.qpro_id_offset])[
4:
] # qpro name is stored in second string of form "000:name"
qproinfo = {
"identifier": filename,
"id": qpro_id,
"name": name,
"type": qp_type,
}
qpros.append(qproinfo)
for config in configurations:
try:
print(f"Trying configuration for game version {config.version}...")
qpros: List[Dict[str, Any]] = []
# For each category, attempt to append all the types.
read_qpro_db(
qpros,
config,
config.qp_head_offset,
config.qp_head_length,
"head",
)
read_qpro_db(
qpros,
config,
config.qp_hair_offset,
config.qp_hair_length,
"hair",
)
read_qpro_db(
qpros,
config,
config.qp_face_offset,
config.qp_face_length,
"face",
)
read_qpro_db(
qpros,
config,
config.qp_hand_offset,
config.qp_hand_length,
"hand",
)
read_qpro_db(
qpros,
config,
config.qp_body_offset,
config.qp_body_length,
"body",
)
# If we got here, that means we ran into no issues and didn't have to attempt another offset.
print("Successfully parsed game DB!")
# We only import one or the other here, I know its a weird function.
return [], qpros
except UnicodeError:
# These offsets are possibly not correct, so try the next configuration.
print("Failed to parse game DB!")
pass
raise Exception(
"Could not determine correct binary parser configuration for IIDX version {self.version}"
)
def lookup(
self, server: str, token: str