1
0
mirror of synced 2025-02-27 23:20:40 +01:00

Update read.py to support possibly scraping from multiple different DDR data releases given a single version.

This commit is contained in:
Jennifer Taylor 2023-07-29 21:22:33 +00:00
parent 3bfe3b2c1d
commit 8477841101

@ -416,7 +416,7 @@ class ImportPopn(ImportBase):
self.charts = [0, 1, 2, 3] self.charts = [0, 1, 2, 3]
else: else:
raise Exception( raise Exception(
"Unsupported Pop'n Music version, expected one of the following: 19, 20, 21, 22, 23, 24, 25, 26!" "Unsupported Pop'n Music version, expected one of the following: 19, 20, 21, 22, 23, 24, omni-24, 25, omni-25, 26, omni-26!"
) )
super().__init__( super().__init__(
@ -2566,6 +2566,67 @@ class ImportIIDX(ImportBase):
self.finish_batch() self.finish_batch()
class DDRScrapeConfiguration:
def __init__(
self,
*,
version: str,
offset: int,
size: int,
length: int,
unpackfmt: str,
id_offset: int,
edit_offset: int,
bpm_min_offset: int,
bpm_max_offset: int,
folder_offset: int,
single_difficulties: int,
double_difficulties: int,
groove_single_beginner: int,
groove_single_basic: int,
groove_single_difficult: int,
groove_single_expert: int,
groove_single_challenge: int,
groove_double_basic: int,
groove_double_difficult: int,
groove_double_expert: int,
groove_double_challenge: int,
voltage: int,
stream: int,
air: int,
chaos: int,
freeze: int,
folder_start: int,
) -> None:
self.version = version
self.offset = offset
self.size = size
self.length = length
self.unpackfmt = unpackfmt
self.id_offset = id_offset
self.edit_offset = edit_offset
self.bpm_min_offset = bpm_min_offset
self.bpm_max_offset = bpm_max_offset
self.folder_offset = folder_offset
self.single_difficulties = single_difficulties
self.double_difficulties = double_difficulties
self.groove_single_beginner = groove_single_beginner
self.groove_single_basic = groove_single_basic
self.groove_single_difficult = groove_single_difficult
self.groove_single_expert = groove_single_expert
self.groove_single_challenge = groove_single_challenge
self.groove_double_basic = groove_double_basic
self.groove_double_difficult = groove_double_difficult
self.groove_double_expert = groove_double_expert
self.groove_double_challenge = groove_double_challenge
self.voltage = voltage
self.stream = stream
self.air = air
self.chaos = chaos
self.freeze = freeze
self.folder_start = folder_start
class ImportDDR(ImportBase): class ImportDDR(ImportBase):
def __init__( def __init__(
self, self,
@ -2595,312 +2656,423 @@ class ImportDDR(ImportBase):
data = myfile.read() data = myfile.read()
myfile.close() myfile.close()
def add_skew(unpackfmt: str, size: int, skew: int) -> str:
# Skew is because I'm too lazy to count the Hs in the format, so just
# pad it for ease of construction here.
return unpackfmt + ("x" * (size - len(unpackfmt) - skew))
configurations: List[DDRScrapeConfiguration] = []
if self.version == VersionConstants.DDR_X2: if self.version == VersionConstants.DDR_X2:
# Based on JDX:J:A:A:2010111000 # Based on JDX:J:A:A:2010111000
offset = 0x254FC0 configurations.append(
size = 0x14C DDRScrapeConfiguration(
length = 894 version="JDX:J:A:A:2010111000",
# Basic stuff like ID, bpm, chart difficulties offset=0x254FC0,
unpackfmt = "<xxxxxxxxHHxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxHHBBBBBBBBBB" size=0x14C,
# Groove radar length=894,
unpackfmt += "HHHHHHHHH" * 5 unpackfmt=add_skew(
if len(unpackfmt) < size: (
# Skew is because I'm too lazy to count the Hs above # Basic stuff like ID, bpm, chart difficulties
skew = 3 + (9 * 5) "<xxxxxxxxHHxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxHHBBBBBBBBBB"
# Just pad it for ease of construction # Groove radar
unpackfmt = unpackfmt + ("x" * (size - len(unpackfmt) - skew)) + ("HHHHHHHHH" * 5)
# Basic offsets ),
id_offset = 1 0x14C,
edit_offset = 0 3 + (9 * 5),
bpm_min_offset = 3 ),
bpm_max_offset = 2 # Basic offsets
folder_offset = 24 # This is a byte offset into the raw field id_offset=1,
edit_offset=0,
# Single/double difficulty array offsets bpm_min_offset=3,
single_difficulties = 4 bpm_max_offset=2,
double_difficulties = 9 folder_offset=24, # This is a byte offset into the raw field
# Single/double difficulty array offsets
# Groove gauge offsets single_difficulties=4,
groove_single_beginner = 22 double_difficulties=9,
groove_single_basic = 14 # Groove gauge offsets
groove_single_difficult = 15 groove_single_beginner=22,
groove_single_expert = 16 groove_single_basic=14,
groove_single_challenge = 17 groove_single_difficult=15,
groove_single_expert=16,
groove_double_basic = 18 groove_single_challenge=17,
groove_double_difficult = 19 groove_double_basic=18,
groove_double_expert = 20 groove_double_difficult=19,
groove_double_challenge = 21 groove_double_expert=20,
groove_double_challenge=21,
# Relative offsets for each groove gauge value # Relative offsets for each groove gauge value
voltage = 0 voltage=0,
stream = 9 stream=9,
air = 18 air=18,
chaos = 27 chaos=27,
freeze = 36 freeze=36,
# Folder start version
# Folder start version folder_start=12,
folder_start = 12 )
)
elif self.version == VersionConstants.DDR_X3_VS_2NDMIX: elif self.version == VersionConstants.DDR_X3_VS_2NDMIX:
# Based on KDX:J:A:A:2012112600 # Based on KDX:J:A:A:2012112600
offset = 0x27A4C8 configurations.append(
size = 0x150 DDRScrapeConfiguration(
length = 1062 version="KDX:J:A:A:2012112600",
# Basic stuff like ID, bpm, chart difficulties offset=0x27A4C8,
unpackfmt = ( size=0x150,
"<xxxxxxxxHHxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxHHBBBBBBBBBB" length=1062,
unpackfmt=add_skew(
(
# Basic stuff like ID, bpm, chart difficulties
"<xxxxxxxxHHxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxHHBBBBBBBBBB"
# Groove radar
+ ("HHHHHHHHH" * 5)
),
0x150,
3 + (9 * 5),
),
# Basic offsets
id_offset=1,
edit_offset=0,
bpm_min_offset=3,
bpm_max_offset=2,
folder_offset=24, # This is a byte offset into the raw field
# Single/double difficulty array offsets
single_difficulties=4,
double_difficulties=9,
# Groove gauge offsets
groove_single_beginner=22,
groove_single_basic=14,
groove_single_difficult=15,
groove_single_expert=16,
groove_single_challenge=17,
groove_double_basic=18,
groove_double_difficult=19,
groove_double_expert=20,
groove_double_challenge=21,
# Relative offsets for each groove gauge value
voltage=0,
stream=9,
air=18,
chaos=27,
freeze=36,
# Folder start version
folder_start=13,
)
) )
# Groove radar
unpackfmt += "HHHHHHHHH" * 5
if len(unpackfmt) < size:
# Skew is because I'm too lazy to count the Hs above
skew = 3 + (9 * 5)
# Just pad it for ease of construction
unpackfmt = unpackfmt + ("x" * (size - len(unpackfmt) - skew))
# Basic offsets
id_offset = 1
edit_offset = 0
bpm_min_offset = 3
bpm_max_offset = 2
folder_offset = 24 # This is a byte offset into the raw field
# Single/double difficulty array offsets
single_difficulties = 4
double_difficulties = 9
# Groove gauge offsets
groove_single_beginner = 22
groove_single_basic = 14
groove_single_difficult = 15
groove_single_expert = 16
groove_single_challenge = 17
groove_double_basic = 18
groove_double_difficult = 19
groove_double_expert = 20
groove_double_challenge = 21
# Relative offsets for each groove gauge value
voltage = 0
stream = 9
air = 18
chaos = 27
freeze = 36
# Folder start version
folder_start = 13
elif self.version == VersionConstants.DDR_2013: elif self.version == VersionConstants.DDR_2013:
# Based on MDX:J:A:A:2014032700 # Based on MDX:J:A:A:2014032700
offset = 0x2663D8 configurations.append(
size = 0x1D0 DDRScrapeConfiguration(
length = 1238 version="MDX:J:A:A:2014032700",
# Basic stuff like ID, bpm, chart difficulties offset=0x2663D8,
unpackfmt = ( size=0x1D0,
"<xxxxxxxxHHxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxHHBBBBBBBBBB" length=1238,
unpackfmt=add_skew(
(
# Basic stuff like ID, bpm, chart difficulties
"<xxxxxxxxHHxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxHHBBBBBBBBBB"
# Groove radar
+ ("HHHHHHHHH" * 5)
),
0x1D0,
3 + (9 * 5),
),
# Basic offsets
id_offset=1,
edit_offset=0,
bpm_min_offset=3,
bpm_max_offset=2,
folder_offset=20, # This is a byte offset into the raw field
# Single/double difficulty array offsets
single_difficulties=4,
double_difficulties=9,
# Groove gauge offsets
groove_single_beginner=22,
groove_single_basic=14,
groove_single_difficult=15,
groove_single_expert=16,
groove_single_challenge=17,
groove_double_basic=18,
groove_double_difficult=19,
groove_double_expert=20,
groove_double_challenge=21,
# Relative offsets for each groove gauge value
voltage=0,
stream=9,
air=18,
chaos=27,
freeze=36,
# Folder start version
folder_start=14,
)
) )
# Groove radar
unpackfmt += "HHHHHHHHH" * 5
if len(unpackfmt) < size:
# Skew is because I'm too lazy to count the Hs above
skew = 3 + (9 * 5)
# Just pad it for ease of construction
unpackfmt = unpackfmt + ("x" * (size - len(unpackfmt) - skew))
# Basic offsets
id_offset = 1
edit_offset = 0
bpm_min_offset = 3
bpm_max_offset = 2
folder_offset = 20 # This is a byte offset into the raw field
# Single/double difficulty array offsets
single_difficulties = 4
double_difficulties = 9
# Groove gauge offsets
groove_single_beginner = 22
groove_single_basic = 14
groove_single_difficult = 15
groove_single_expert = 16
groove_single_challenge = 17
groove_double_basic = 18
groove_double_difficult = 19
groove_double_expert = 20
groove_double_challenge = 21
# Relative offsets for each groove gauge value
voltage = 0
stream = 9
air = 18
chaos = 27
freeze = 36
# Folder start version
folder_start = 14
elif self.version == VersionConstants.DDR_2014: elif self.version == VersionConstants.DDR_2014:
# Based on MDX:A:A:A:2015122100 # Based on MDX:A:A:A:2015122100
offset = 0x2B72B0 configurations.append(
size = 0x1D0 DDRScrapeConfiguration(
length = 1466 version="MDX:A:A:A:2015122100",
# Basic stuff like ID, bpm, chart difficulties offset=0x2B72B0,
unpackfmt = ( size=0x1D0,
"<xxxxxxxxHHxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxHHBBBBBBBBBB" length=1466,
unpackfmt=add_skew(
(
# Basic stuff like ID, bpm, chart difficulties
"<xxxxxxxxHHxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxHHBBBBBBBBBB"
# Groove radar
+ ("HHHHHHHHH" * 5)
),
0x1D0,
3 + (9 * 5),
),
# Basic offsets
id_offset=1,
edit_offset=0,
bpm_min_offset=3,
bpm_max_offset=2,
folder_offset=20, # This is a byte offset into the raw field
# Single/double difficulty array offsets
single_difficulties=4,
double_difficulties=9,
# Groove gauge offsets
groove_single_beginner=22,
groove_single_basic=14,
groove_single_difficult=15,
groove_single_expert=16,
groove_single_challenge=17,
groove_double_basic=18,
groove_double_difficult=19,
groove_double_expert=20,
groove_double_challenge=21,
# Relative offsets for each groove gauge value
voltage=0,
stream=9,
air=18,
chaos=27,
freeze=36,
# Folder start version
folder_start=15,
)
) )
# Groove radar
unpackfmt += "HHHHHHHHH" * 5
if len(unpackfmt) < size:
# Skew is because I'm too lazy to count the Hs above
skew = 3 + (9 * 5)
# Just pad it for ease of construction
unpackfmt = unpackfmt + ("x" * (size - len(unpackfmt) - skew))
# Basic offsets
id_offset = 1
edit_offset = 0
bpm_min_offset = 3
bpm_max_offset = 2
folder_offset = 20 # This is a byte offset into the raw field
# Single/double difficulty array offsets
single_difficulties = 4
double_difficulties = 9
# Groove gauge offsets
groove_single_beginner = 22
groove_single_basic = 14
groove_single_difficult = 15
groove_single_expert = 16
groove_single_challenge = 17
groove_double_basic = 18
groove_double_difficult = 19
groove_double_expert = 20
groove_double_challenge = 21
# Relative offsets for each groove gauge value
voltage = 0
stream = 9
air = 18
chaos = 27
freeze = 36
# Folder start version
folder_start = 15
else: else:
raise Exception("Unknown game version!") raise Exception("Unknown game version!")
songs = []
for i in range(length): for config in configurations:
start = offset + (i * size) try:
end = offset + ((i + 1) * size) print(f"Trying configuration for game version {config.version}...")
chunk = data[start:end]
# First, figure out if it is actually a song songs = []
ssqcode = chunk[0:6].decode("ascii").replace("\0", "").strip()
if len(ssqcode) == 0: for i in range(config.length):
continue start = config.offset + (i * config.size)
unpacked = struct.unpack(unpackfmt, chunk) end = config.offset + ((i + 1) * config.size)
songinfo = { chunk = data[start:end]
"id": unpacked[id_offset],
"edit_id": unpacked[edit_offset], # First, figure out if it is actually a song
"ssqcode": ssqcode, ssqcode = chunk[0:6].decode("ascii").replace("\0", "").strip()
"difficulty": { if len(ssqcode) == 0:
"single": { continue
"beginner": unpacked[single_difficulties + 0], unpacked = struct.unpack(config.unpackfmt, chunk)
"basic": unpacked[single_difficulties + 1], songinfo = {
"difficult": unpacked[single_difficulties + 2], "id": unpacked[config.id_offset],
"expert": unpacked[single_difficulties + 3], "edit_id": unpacked[config.edit_offset],
"challenge": unpacked[single_difficulties + 4], "ssqcode": ssqcode,
}, "difficulty": {
"double": { "single": {
"beginner": unpacked[double_difficulties + 0], "beginner": unpacked[config.single_difficulties + 0],
"basic": unpacked[double_difficulties + 1], "basic": unpacked[config.single_difficulties + 1],
"difficult": unpacked[double_difficulties + 2], "difficult": unpacked[config.single_difficulties + 2],
"expert": unpacked[double_difficulties + 3], "expert": unpacked[config.single_difficulties + 3],
"challenge": unpacked[double_difficulties + 4], "challenge": unpacked[config.single_difficulties + 4],
}, },
}, "double": {
"groove_gauge": { "beginner": unpacked[config.double_difficulties + 0],
"single": { "basic": unpacked[config.double_difficulties + 1],
"beginner": { "difficult": unpacked[config.double_difficulties + 2],
"voltage": unpacked[groove_single_beginner + voltage], "expert": unpacked[config.double_difficulties + 3],
"stream": unpacked[groove_single_beginner + stream], "challenge": unpacked[config.double_difficulties + 4],
"air": unpacked[groove_single_beginner + air], },
"chaos": unpacked[groove_single_beginner + chaos],
"freeze": unpacked[groove_single_beginner + freeze],
}, },
"basic": { "groove_gauge": {
"voltage": unpacked[groove_single_basic + voltage], "single": {
"stream": unpacked[groove_single_basic + stream], "beginner": {
"air": unpacked[groove_single_basic + air], "voltage": unpacked[
"chaos": unpacked[groove_single_basic + chaos], config.groove_single_beginner + config.voltage
"freeze": unpacked[groove_single_basic + freeze], ],
"stream": unpacked[
config.groove_single_beginner + config.stream
],
"air": unpacked[
config.groove_single_beginner + config.air
],
"chaos": unpacked[
config.groove_single_beginner + config.chaos
],
"freeze": unpacked[
config.groove_single_beginner + config.freeze
],
},
"basic": {
"voltage": unpacked[
config.groove_single_basic + config.voltage
],
"stream": unpacked[
config.groove_single_basic + config.stream
],
"air": unpacked[
config.groove_single_basic + config.air
],
"chaos": unpacked[
config.groove_single_basic + config.chaos
],
"freeze": unpacked[
config.groove_single_basic + config.freeze
],
},
"difficult": {
"voltage": unpacked[
config.groove_single_difficult + config.voltage
],
"stream": unpacked[
config.groove_single_difficult + config.stream
],
"air": unpacked[
config.groove_single_difficult + config.air
],
"chaos": unpacked[
config.groove_single_difficult + config.chaos
],
"freeze": unpacked[
config.groove_single_difficult + config.freeze
],
},
"expert": {
"voltage": unpacked[
config.groove_single_expert + config.voltage
],
"stream": unpacked[
config.groove_single_expert + config.stream
],
"air": unpacked[
config.groove_single_expert + config.air
],
"chaos": unpacked[
config.groove_single_expert + config.chaos
],
"freeze": unpacked[
config.groove_single_expert + config.freeze
],
},
"challenge": {
"voltage": unpacked[
config.groove_single_challenge + config.voltage
],
"stream": unpacked[
config.groove_single_challenge + config.stream
],
"air": unpacked[
config.groove_single_challenge + config.air
],
"chaos": unpacked[
config.groove_single_challenge + config.chaos
],
"freeze": unpacked[
config.groove_single_challenge + config.freeze
],
},
},
"double": {
"beginner": {
"voltage": 0,
"stream": 0,
"air": 0,
"chaos": 0,
"freeze": 0,
},
"basic": {
"voltage": unpacked[
config.groove_double_basic + config.voltage
],
"stream": unpacked[
config.groove_double_basic + config.stream
],
"air": unpacked[
config.groove_double_basic + config.air
],
"chaos": unpacked[
config.groove_double_basic + config.chaos
],
"freeze": unpacked[
config.groove_double_basic + config.freeze
],
},
"difficult": {
"voltage": unpacked[
config.groove_double_difficult + config.voltage
],
"stream": unpacked[
config.groove_double_difficult + config.stream
],
"air": unpacked[
config.groove_double_difficult + config.air
],
"chaos": unpacked[
config.groove_double_difficult + config.chaos
],
"freeze": unpacked[
config.groove_double_difficult + config.freeze
],
},
"expert": {
"voltage": unpacked[
config.groove_double_expert + config.voltage
],
"stream": unpacked[
config.groove_double_expert + config.stream
],
"air": unpacked[
config.groove_double_expert + config.air
],
"chaos": unpacked[
config.groove_double_expert + config.chaos
],
"freeze": unpacked[
config.groove_double_expert + config.freeze
],
},
"challenge": {
"voltage": unpacked[
config.groove_double_challenge + config.voltage
],
"stream": unpacked[
config.groove_double_challenge + config.stream
],
"air": unpacked[
config.groove_double_challenge + config.air
],
"chaos": unpacked[
config.groove_double_challenge + config.chaos
],
"freeze": unpacked[
config.groove_double_challenge + config.freeze
],
},
},
}, },
"difficult": { "bpm_min": unpacked[config.bpm_min_offset],
"voltage": unpacked[groove_single_difficult + voltage], "bpm_max": unpacked[config.bpm_max_offset],
"stream": unpacked[groove_single_difficult + stream], "folder": config.folder_start - chunk[config.folder_offset],
"air": unpacked[groove_single_difficult + air], }
"chaos": unpacked[groove_single_difficult + chaos], songs.append(songinfo)
"freeze": unpacked[groove_single_difficult + freeze],
}, # If we got here, that means we ran into no issues and didn't have to attempt another offset.
"expert": { print("Successfully parsed game DB!")
"voltage": unpacked[groove_single_expert + voltage],
"stream": unpacked[groove_single_expert + stream], return songs
"air": unpacked[groove_single_expert + air], except UnicodeError:
"chaos": unpacked[groove_single_expert + chaos], # These offsets are possibly not correct, so try the next configuration.
"freeze": unpacked[groove_single_expert + freeze], print("Failed to parse game DB!")
}, pass
"challenge": {
"voltage": unpacked[groove_single_challenge + voltage], raise Exception(
"stream": unpacked[groove_single_challenge + stream], "Could not determine correct binary parser configuration for DDR version {self.version}"
"air": unpacked[groove_single_challenge + air], )
"chaos": unpacked[groove_single_challenge + chaos],
"freeze": unpacked[groove_single_challenge + freeze],
},
},
"double": {
"beginner": {
"voltage": 0,
"stream": 0,
"air": 0,
"chaos": 0,
"freeze": 0,
},
"basic": {
"voltage": unpacked[groove_double_basic + voltage],
"stream": unpacked[groove_double_basic + stream],
"air": unpacked[groove_double_basic + air],
"chaos": unpacked[groove_double_basic + chaos],
"freeze": unpacked[groove_double_basic + freeze],
},
"difficult": {
"voltage": unpacked[groove_double_difficult + voltage],
"stream": unpacked[groove_double_difficult + stream],
"air": unpacked[groove_double_difficult + air],
"chaos": unpacked[groove_double_difficult + chaos],
"freeze": unpacked[groove_double_difficult + freeze],
},
"expert": {
"voltage": unpacked[groove_double_expert + voltage],
"stream": unpacked[groove_double_expert + stream],
"air": unpacked[groove_double_expert + air],
"chaos": unpacked[groove_double_expert + chaos],
"freeze": unpacked[groove_double_expert + freeze],
},
"challenge": {
"voltage": unpacked[groove_double_challenge + voltage],
"stream": unpacked[groove_double_challenge + stream],
"air": unpacked[groove_double_challenge + air],
"chaos": unpacked[groove_double_challenge + chaos],
"freeze": unpacked[groove_double_challenge + freeze],
},
},
},
"bpm_min": unpacked[bpm_min_offset],
"bpm_max": unpacked[bpm_max_offset],
"folder": folder_start - chunk[folder_offset],
}
songs.append(songinfo)
return songs
def hydrate(self, songs: List[Dict[str, Any]], infile: str) -> List[Dict[str, Any]]: def hydrate(self, songs: List[Dict[str, Any]], infile: str) -> List[Dict[str, Any]]:
tree = ET.parse(infile) tree = ET.parse(infile)