Update read.py to support possibly scraping from multiple different data releases given a single version.

2dxutils

@@ -5,6 +5,7 @@ if __name__ == "__main__":
 
     import sys
     sys.path.append(path)
+    os.environ["SQLALCHEMY_SILENCE_UBER_WARNING"] = "1"
 
     import runpy
     runpy.run_module("bemani.utils.twodxutils", run_name="__main__")

README.md

@@ -607,7 +607,15 @@ Then, run the following command, modifying the version parameter as required.
 
 For Reflec Beat, get the game DLL from the version of the game you want to import and
 run a command like so. This network supports Reflec Beat up through Volzza 2, so you
-will want to run this with versions 1-6 to completely initialize:
+will want to run this with versions 1-6 to completely initialize. Use the version
+corresponding to version in the following table:
+
+* Reflec Beat: 1
+* Limelight: 2
+* Colette: 3
+* Groovin'!!: 4
+* VOLZZA: 5
+* VOLZZA 2: 6
 
 ```
 ./read --config config/server.yaml --series reflec --version 1 --bin reflecbeat.dll

afputils

@@ -6,6 +6,7 @@ if __name__ == "__main__":
 
         import sys
         sys.path.append(path)
+        os.environ["SQLALCHEMY_SILENCE_UBER_WARNING"] = "1"
 
         import runpy
         runpy.run_module(f"bemani.utils.{name}", run_name="__main__")

api

@@ -6,6 +6,7 @@ if __name__ == "__main__":
 
         import sys
         sys.path.append(path)
+        os.environ["SQLALCHEMY_SILENCE_UBER_WARNING"] = "1"
 
         import runpy
         runpy.run_module(f"bemani.utils.{name}", run_name="__main__")

arcutils

@@ -6,6 +6,7 @@ if __name__ == "__main__":
 
         import sys
         sys.path.append(path)
+        os.environ["SQLALCHEMY_SILENCE_UBER_WARNING"] = "1"
 
         import runpy
         runpy.run_module(f"bemani.utils.{name}", run_name="__main__")

assetparse

@@ -6,6 +6,7 @@ if __name__ == "__main__":
 
         import sys
         sys.path.append(path)
+        os.environ["SQLALCHEMY_SILENCE_UBER_WARNING"] = "1"
 
         import runpy
         runpy.run_module(f"bemani.utils.{name}", run_name="__main__")

bemani/common/pe.py

@@ -71,6 +71,18 @@ class RetException(Exception):
     pass
 
 
+class InvalidOffsetException(Exception):
+    pass
+
+
+class InvalidVirtualOffsetException(InvalidOffsetException):
+    pass
+
+
+class InvalidPhysicalOffsetException(InvalidOffsetException):
+    pass
+
+
 class PEFile:
     def __init__(self, data: bytes) -> None:
         self.data = data
@@ -89,7 +101,7 @@ class PEFile:
             if offset == virtual:
                 return physical
 
-        raise Exception(
+        raise InvalidVirtualOffsetException(
             f"Couldn't find physical offset for virtual offset 0x{offset:08x}"
         )
 
@@ -109,7 +121,7 @@ class PEFile:
             if offset == physical:
                 return virtual
 
-        raise Exception(
+        raise InvalidPhysicalOffsetException(
             f"Couldn't find virtual offset for physical offset 0x{offset:08x}"
         )
 

bemani/utils/read.py

@@ -1713,7 +1713,6 @@ class ImportJubeat(ImportBase):
             VersionConstants.JUBEAT_FESTO,
         }:
             for emblem_entry in root.find("emblem_list") or []:
-                print(emblem_entry)
                 index = int(emblem_entry.find("index").text)
                 layer = int(emblem_entry.find("layer").text)
                 music_id = int(emblem_entry.find("music_id").text)
@@ -3775,6 +3774,37 @@ class ImportMuseca(ImportBase):
             self.finish_batch()
 
 
+class ReflecBeatScrapeConfiguration:
+    def __init__(
+        self,
+        *,
+        version: str,
+        offset: int,
+        stride: int,
+        max_songs: int,
+        max_difficulties: int,
+        song_offset: int,
+        song_length: int,
+        artist_offset: Optional[int],
+        artist_length: Optional[int],
+        chart_offset: int,
+        chart_length: int,
+        difficulties_offset: int,
+    ) -> None:
+        self.version = version
+        self.offset = offset
+        self.stride = stride
+        self.max_songs = max_songs
+        self.max_difficulties = max_difficulties
+        self.song_offset = song_offset
+        self.song_length = song_length
+        self.artist_offset = artist_offset
+        self.artist_length = artist_length
+        self.chart_offset = chart_offset
+        self.chart_length = chart_length
+        self.difficulties_offset = difficulties_offset
+
+
 class ImportReflecBeat(ImportBase):
     def __init__(
         self,
@@ -3810,92 +3840,117 @@ class ImportReflecBeat(ImportBase):
             data = myfile.read()
             myfile.close()
 
+        configurations: List[ReflecBeatScrapeConfiguration] = []
         if self.version == VersionConstants.REFLEC_BEAT:
             # Based on KBR:A:A:A:2011112300
-            offset = 0xBFBD0
-            stride = 280
-            max_songs = 93
-            max_difficulties = 3
-
-            song_offset = 0x4C
-            song_length = 0x40
+            configurations.append(
+                ReflecBeatScrapeConfiguration(
+                    version="KBR:A:A:A:2011112300",
+                    offset=0xBFBD0,
+                    stride=280,
+                    max_songs=93,
+                    max_difficulties=3,
+                    song_offset=0x4C,
+                    song_length=0x40,
                     # Artists aren't included in this mix.
-            artist_offset = None
-            artist_length = None
-            chart_offset = 0xD5
-            chart_length = 0x20
-            difficulties_offset = 0xD2
+                    artist_offset=None,
+                    artist_length=None,
+                    chart_offset=0xD5,
+                    chart_length=0x20,
+                    difficulties_offset=0xD2,
+                )
+            )
         elif self.version == VersionConstants.REFLEC_BEAT_LIMELIGHT:
             # Based on LBR:A:A:A:2012082900
-            offset = 0x132C48
-            stride = 220
-            max_songs = 191
-            max_difficulties = 3
-
-            song_offset = 0x4C
-            song_length = 0x40
+            configurations.append(
+                ReflecBeatScrapeConfiguration(
+                    version="LBR:A:A:A:2012082900",
+                    offset=0x132C48,
+                    stride=220,
+                    max_songs=191,
+                    max_difficulties=3,
+                    song_offset=0x4C,
+                    song_length=0x40,
                     # Artists aren't included in this mix.
-            artist_offset = None
-            artist_length = None
-            chart_offset = 0x9B
-            chart_length = 0x20
-            difficulties_offset = 0x98
+                    artist_offset=None,
+                    artist_length=None,
+                    chart_offset=0x9B,
+                    chart_length=0x20,
+                    difficulties_offset=0x98,
+                )
+            )
         elif self.version == VersionConstants.REFLEC_BEAT_COLETTE:
             # Based on MBR:J:A:A:2014011600
-            offset = 0x1E6880
-            stride = 468
-            max_songs = 443
-            max_difficulties = 3
-
-            song_offset = 0x34
-            song_length = 0x80
-            artist_offset = 0xB4
-            artist_length = 0x80
-            chart_offset = 0x1B4
-            chart_length = 0x20
-            difficulties_offset = 0x1A8
+            configurations.append(
+                ReflecBeatScrapeConfiguration(
+                    version="MBR:J:A:A:2014011600",
+                    offset=0x1E6880,
+                    stride=468,
+                    max_songs=443,
+                    max_difficulties=3,
+                    song_offset=0x34,
+                    song_length=0x80,
+                    artist_offset=0xB4,
+                    artist_length=0x80,
+                    chart_offset=0x1B4,
+                    chart_length=0x20,
+                    difficulties_offset=0x1A8,
+                )
+            )
         elif self.version == VersionConstants.REFLEC_BEAT_GROOVIN:
             # Based on MBR:J:A:A:2015102100
-            offset = 0x212EC0
-            stride = 524
-            max_songs = 698
-            max_difficulties = 4
-
-            song_offset = 0x3C
-            song_length = 0x80
-            artist_offset = 0xBC
-            artist_length = 0x80
-            chart_offset = 0x1E8
-            chart_length = 0x20
-            difficulties_offset = 0x1D0
+            configurations.append(
+                ReflecBeatScrapeConfiguration(
+                    version="MBR:J:A:A:2015102100",
+                    offset=0x212EC0,
+                    stride=524,
+                    max_songs=698,
+                    max_difficulties=4,
+                    song_offset=0x3C,
+                    song_length=0x80,
+                    artist_offset=0xBC,
+                    artist_length=0x80,
+                    chart_offset=0x1E8,
+                    chart_length=0x20,
+                    difficulties_offset=0x1D0,
+                )
+            )
         elif self.version == VersionConstants.REFLEC_BEAT_VOLZZA:
             # Based on MBR:J:A:A:2016030200
-            offset = 0x1A0EC8
-            stride = 552
-            max_songs = 805
-            max_difficulties = 4
-
-            song_offset = 0x38
-            song_length = 0x80
-            artist_offset = 0xB8
-            artist_length = 0x80
-            chart_offset = 0x1E4
-            chart_length = 0x20
-            difficulties_offset = 0x1CC
+            configurations.append(
+                ReflecBeatScrapeConfiguration(
+                    version="MBR:J:A:A:2016030200",
+                    offset=0x1A0EC8,
+                    stride=552,
+                    max_songs=805,
+                    max_difficulties=4,
+                    song_offset=0x38,
+                    song_length=0x80,
+                    artist_offset=0xB8,
+                    artist_length=0x80,
+                    chart_offset=0x1E4,
+                    chart_length=0x20,
+                    difficulties_offset=0x1CC,
+                )
+            )
         elif self.version == VersionConstants.REFLEC_BEAT_VOLZZA_2:
             # Based on MBR:J:A:A:2016100400
-            offset = 0x1CBC68
-            stride = 552
-            max_songs = 850
-            max_difficulties = 4
-
-            song_offset = 0x38
-            song_length = 0x80
-            artist_offset = 0xB8
-            artist_length = 0x80
-            chart_offset = 0x1E4
-            chart_length = 0x20
-            difficulties_offset = 0x1CC
+            configurations.append(
+                ReflecBeatScrapeConfiguration(
+                    version="MBR:J:A:A:2016100400",
+                    offset=0x1CBC68,
+                    stride=552,
+                    max_songs=850,
+                    max_difficulties=4,
+                    song_offset=0x38,
+                    song_length=0x80,
+                    artist_offset=0xB8,
+                    artist_length=0x80,
+                    chart_offset=0x1E4,
+                    chart_length=0x20,
+                    difficulties_offset=0x1CC,
+                )
+            )
         else:
             raise Exception(f"Unsupported ReflecBeat version {self.version}")
 
@@ -3906,7 +3961,7 @@ class ImportReflecBeat(ImportBase):
                     end = i
                     break
             if end is None:
-                raise Exception("Invalid string!")
+                raise UnicodeError("Invalid string!")
             if end == 0:
                 return ""
 
@@ -3925,29 +3980,49 @@ class ImportReflecBeat(ImportBase):
 
             return folder
 
+        for config in configurations:
+            try:
+                print(f"Trying configuration for game version {config.version}...")
+
                 songs = []
-        for i in range(max_songs):
-            start = offset + (stride * i)
-            end = start + stride
+                for i in range(config.max_songs):
+                    start = config.offset + (config.stride * i)
+                    end = start + config.stride
                     songdata = data[start:end]
 
-            title = convert_string(songdata[song_offset : (song_offset + song_length)])
-            if artist_offset is None:
+                    title = convert_string(
+                        songdata[
+                            config.song_offset : (
+                                config.song_offset + config.song_length
+                            )
+                        ]
+                    )
+                    if config.artist_offset is None:
                         artist = ""
                     else:
                         artist = convert_string(
-                    songdata[artist_offset : (artist_offset + artist_length)]
+                            songdata[
+                                config.artist_offset : (
+                                    config.artist_offset + config.artist_length
+                                )
+                            ]
                         )
                     if title == "" and artist == "":
                         continue
                     songid = struct.unpack("<I", songdata[0:4])[0]
                     chart = convert_string(
-                songdata[chart_offset : (chart_offset + chart_length)]
+                        songdata[
+                            config.chart_offset : (
+                                config.chart_offset + config.chart_length
+                            )
+                        ]
                     )
                     difficulties = [
                         d
                         for d in songdata[
-                    difficulties_offset : (difficulties_offset + max_difficulties)
+                            config.difficulties_offset : (
+                                config.difficulties_offset + config.max_difficulties
+                            )
                         ]
                     ]
                     difficulties = [0 if d == 255 else d for d in difficulties]
@@ -3966,7 +4041,19 @@ class ImportReflecBeat(ImportBase):
                             "folder": folder,
                         }
                     )
+
+                # If we got here, that means we ran into no issues and didn't have to attempt another offset.
+                print("Successfully parsed game DB!")
+
                 return songs
+            except UnicodeError:
+                # These offsets are possibly not correct, so try the next configuration.
+                print("Failed to parse game DB!")
+                pass
+
+        raise Exception(
+            "Could not determine correct binary parser configuration for ReflecBeat version {self.version}"
+        )
 
     def lookup(self, server: str, token: str) -> List[Dict[str, Any]]:
         # Grab music info from remote server

bemanishark

@@ -6,6 +6,7 @@ if __name__ == "__main__":
 
         import sys
         sys.path.append(path)
+        os.environ["SQLALCHEMY_SILENCE_UBER_WARNING"] = "1"
 
         import runpy
         runpy.run_module(f"bemani.utils.{name}", run_name="__main__")

binutils

@@ -6,6 +6,7 @@ if __name__ == "__main__":
 
         import sys
         sys.path.append(path)
+        os.environ["SQLALCHEMY_SILENCE_UBER_WARNING"] = "1"
 
         import runpy
         runpy.run_module(f"bemani.utils.{name}", run_name="__main__")

cardconvert

@@ -6,6 +6,7 @@ if __name__ == "__main__":
 
         import sys
         sys.path.append(path)
+        os.environ["SQLALCHEMY_SILENCE_UBER_WARNING"] = "1"
 
         import runpy
         runpy.run_module(f"bemani.utils.{name}", run_name="__main__")

dbutils

@@ -6,6 +6,7 @@ if __name__ == "__main__":
 
         import sys
         sys.path.append(path)
+        os.environ["SQLALCHEMY_SILENCE_UBER_WARNING"] = "1"
 
         import runpy
         runpy.run_module(f"bemani.utils.{name}", run_name="__main__")

frontend

@@ -6,6 +6,7 @@ if __name__ == "__main__":
 
         import sys
         sys.path.append(path)
+        os.environ["SQLALCHEMY_SILENCE_UBER_WARNING"] = "1"
 
         import runpy
         runpy.run_module(f"bemani.utils.{name}", run_name="__main__")

ifsutils

@@ -6,6 +6,7 @@ if __name__ == "__main__":
 
         import sys
         sys.path.append(path)
+        os.environ["SQLALCHEMY_SILENCE_UBER_WARNING"] = "1"
 
         import runpy
         runpy.run_module(f"bemani.utils.{name}", run_name="__main__")

iidxutils

@@ -6,6 +6,7 @@ if __name__ == "__main__":
 
         import sys
         sys.path.append(path)
+        os.environ["SQLALCHEMY_SILENCE_UBER_WARNING"] = "1"
 
         import runpy
         runpy.run_module(f"bemani.utils.{name}", run_name="__main__")

jsx

@@ -6,6 +6,7 @@ if __name__ == "__main__":
 
         import sys
         sys.path.append(path)
+        os.environ["SQLALCHEMY_SILENCE_UBER_WARNING"] = "1"
 
         import runpy
         runpy.run_module(f"bemani.utils.{name}", run_name="__main__")

proxy

@@ -6,6 +6,7 @@ if __name__ == "__main__":
 
         import sys
         sys.path.append(path)
+        os.environ["SQLALCHEMY_SILENCE_UBER_WARNING"] = "1"
 
         import runpy
         runpy.run_module(f"bemani.utils.{name}", run_name="__main__")

psmap

@@ -6,6 +6,7 @@ if __name__ == "__main__":
 
         import sys
         sys.path.append(path)
+        os.environ["SQLALCHEMY_SILENCE_UBER_WARNING"] = "1"
 
         import runpy
         runpy.run_module(f"bemani.utils.{name}", run_name="__main__")

read

@@ -6,6 +6,7 @@ if __name__ == "__main__":
 
         import sys
         sys.path.append(path)
+        os.environ["SQLALCHEMY_SILENCE_UBER_WARNING"] = "1"
 
         import runpy
         runpy.run_module(f"bemani.utils.{name}", run_name="__main__")

replay

@@ -6,6 +6,7 @@ if __name__ == "__main__":
 
         import sys
         sys.path.append(path)
+        os.environ["SQLALCHEMY_SILENCE_UBER_WARNING"] = "1"
 
         import runpy
         runpy.run_module(f"bemani.utils.{name}", run_name="__main__")

responsegen

@@ -6,6 +6,7 @@ if __name__ == "__main__":
 
         import sys
         sys.path.append(path)
+        os.environ["SQLALCHEMY_SILENCE_UBER_WARNING"] = "1"
 
         import runpy
         runpy.run_module(f"bemani.utils.{name}", run_name="__main__")

sampleclient

@@ -6,6 +6,7 @@ if __name__ == "__main__":
 
         import sys
         sys.path.append(path)
+        os.environ["SQLALCHEMY_SILENCE_UBER_WARNING"] = "1"
 
         import runpy
         runpy.run_module(f"bemani.utils.{name}", run_name="__main__")

scheduler

@@ -6,6 +6,7 @@ if __name__ == "__main__":
 
         import sys
         sys.path.append(path)
+        os.environ["SQLALCHEMY_SILENCE_UBER_WARNING"] = "1"
 
         import runpy
         runpy.run_module(f"bemani.utils.{name}", run_name="__main__")

services

@@ -6,6 +6,7 @@ if __name__ == "__main__":
 
         import sys
         sys.path.append(path)
+        os.environ["SQLALCHEMY_SILENCE_UBER_WARNING"] = "1"
 
         import runpy
         runpy.run_module(f"bemani.utils.{name}", run_name="__main__")

struct

@@ -6,6 +6,7 @@ if __name__ == "__main__":
 
         import sys
         sys.path.append(path)
+        os.environ["SQLALCHEMY_SILENCE_UBER_WARNING"] = "1"
 
         import runpy
         runpy.run_module(f"bemani.utils.{name}", run_name="__main__")

trafficgen

@@ -6,6 +6,7 @@ if __name__ == "__main__":
 
         import sys
         sys.path.append(path)
+        os.environ["SQLALCHEMY_SILENCE_UBER_WARNING"] = "1"
 
         import runpy
         runpy.run_module(f"bemani.utils.{name}", run_name="__main__")