Teach network how to import/export Jubeat emblem catalog.

README.md

@@ -402,6 +402,14 @@ after importing all mixes:
 ./read --config config/server.yaml --series jubeat --version all --tsv data/jubeat.tsv
 ```
 
+For Jubeat Prop and later versions, you will also need to import the emblem DB, or emblems
+will not work properly. An example is as follows:
+
+```
+./read --config config/server.yaml --series jubeat --version prop \
+      --xml data/emblem-info/emblem-info.xml
+```
+
 ### IIDX
 
 For IIDX, you will need the data directory of the mix you wish to support. The import

bemani/api/objects/catalog.py

@@ -128,9 +128,38 @@ class CatalogObject(BaseObject):
                 ],
             }
 
+    def __format_jubeat_extras(self) -> Dict[str, List[Dict[str, Any]]]:
+        # Gotta look up the unlock catalog
+        items = self.data.local.game.get_items(self.game, self.version)
+
+        # Format it depending on the version
+        if self.version in {
+            VersionConstants.JUBEAT_PROP,
+            VersionConstants.JUBEAT_QUBELL,
+            VersionConstants.JUBEAT_CLAN,
+        }:
+            return {
+                "emblems": [
+                    {
+                        "index": str(item.id),
+                        "song": item.data.get_int("music_id"),
+                        "layer": item.data.get_int("layer"),
+                        "evolved": item.data.get_int("evolved"),
+                        "rarity": item.data.get_int("rarity"),
+                        "name": item.data.get_str("name"),
+                    }
+                    for item in items
+                    if item.type == "emblem"
+                ],
+            }
+        else:
+            return {"emblems": []}
+
     def __format_extras(self) -> Dict[str, List[Dict[str, Any]]]:
         if self.game == GameConstants.SDVX:
             return self.__format_sdvx_extras()
+        elif self.game == GameConstants.JUBEAT:
+            return self.__format_jubeat_extras()
         else:
             return {}
 

bemani/data/api/game.py

@@ -25,6 +25,19 @@ class GlobalGameData(BaseGlobalData):
             {},
         )
 
+    def __translate_jubeat_emblems(self, entry: Dict[str, Any]) -> Item:
+        return Item(
+            "emblem",
+            int(entry["index"]),
+            {
+                "music_id": int(entry["song"]),
+                "layer": int(entry["layer"]),
+                "evolved": int(entry["evolved"]),
+                "rarity": int(entry["rarity"]),
+                "name": entry["name"],
+            },
+        )
+
     def get_items(self, game: str, version: int) -> List[Item]:
         """
         Given a game/userid, find all items in the catalog.
@@ -51,6 +64,10 @@ class GlobalGameData(BaseGlobalData):
                         "purchases": self.__translate_sdvx_song_unlock,
                         "appealcards": self.__translate_sdvx_appealcard,
                     }.get(catalogtype, None)
+                elif game == GameConstants.JUBEAT:
+                    translation = {
+                        "emblems": self.__translate_jubeat_emblems,
+                    }.get(catalogtype, None)
                 else:
                     translation = None
 

bemani/utils/read.py

@@ -1239,15 +1239,22 @@ class ImportJubeat(ImportBase):
 
         super().__init__(config, GameConstants.JUBEAT, actual_version, no_combine, update)
 
-    def scrape(self, xmlfile: str) -> List[Dict[str, Any]]:
+    def scrape(self, xmlfile: str) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
         if self.version is None:
             raise Exception('Can\'t scrape Jubeat database for \'all\' version!')
 
+        try:
+            # Probably UTF-8 music DB
             tree = ET.parse(xmlfile)
             root = tree.getroot()
+        except ValueError:
+            # Probably shift-jis emblems
+            with open(xmlfile, 'rb') as xmlhandle:
+                xmldata = xmlhandle.read().decode('shift_jisx0213')
+            root = ET.fromstring(xmldata)
 
         songs: List[Dict[str, Any]] = []
-        for music_entry in root.find('body'):
+        for music_entry in root.find('body') or []:
             songid = int(music_entry.find('music_id').text)
             bpm_min = float(music_entry.find('bpm_min').text)
             bpm_max = float(music_entry.find('bpm_max').text)
@@ -1278,9 +1285,34 @@ class ImportJubeat(ImportBase):
                     'extreme': difficulties[2],
                 },
             })
-        return songs
 
-    def lookup(self, server: str, token: str) -> List[Dict[str, Any]]:
+        emblems: List[Dict[str, Any]] = []
+        if self.version in {
+            VersionConstants.JUBEAT_PROP,
+            VersionConstants.JUBEAT_QUBELL,
+            VersionConstants.JUBEAT_CLAN,
+        }:
+            for emblem_entry in root.find('emblem_list') or []:
+                print(emblem_entry)
+                index = int(emblem_entry.find('index').text)
+                layer = int(emblem_entry.find('layer').text)
+                music_id = int(emblem_entry.find('music_id').text)
+                evolved = int(emblem_entry.find('evolved').text)
+                rarity = int(emblem_entry.find('rarity').text)
+                name = emblem_entry.find('name').text
+
+                emblems.append({
+                    'id': index,
+                    'layer': layer,
+                    'music_id': music_id,
+                    'evolved': evolved,
+                    'rarity': rarity,
+                    'name': name,
+                })
+
+        return songs, emblems
+
+    def lookup(self, server: str, token: str) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
         if self.version is None:
             raise Exception('Can\'t look up Jubeat database for \'all\' version!')
 
@@ -1316,8 +1348,28 @@ class ImportJubeat(ImportBase):
                 }
             lut[song.id]['difficulty'][chart_map[song.chart]] = song.data.get_int('difficulty')
 
-        # Return the reassembled data
+        # Reassemble the data
-        return [val for _, val in lut.items()]
+        reassembled_songs = [val for _, val in lut.items()]
+
+        emblems: List[Dict[str, Any]] = []
+        if self.version in {
+            VersionConstants.JUBEAT_PROP,
+            VersionConstants.JUBEAT_QUBELL,
+            VersionConstants.JUBEAT_CLAN,
+        }:
+            game = self.remote_game(server, token)
+            for item in game.get_items(self.game, self.version):
+                if item.type == "emblem":
+                    emblems.append({
+                        'id': item.id,
+                        'layer': item.data.get_int('layer'),
+                        'music_id': item.data.get_int('music_id'),
+                        'evolved': item.data.get_int('evolved'),
+                        'rarity': item.data.get_int('rarity'),
+                        'name': item.data.get_str('name'),
+                    })
+
+        return reassembled_songs, emblems
 
     def import_music_db(self, songs: List[Dict[str, Any]]) -> None:
         if self.version is None:
@@ -1356,6 +1408,32 @@ class ImportJubeat(ImportBase):
                 self.insert_music_id_for_song(next_id, songid, chart, song['title'], song['artist'], song['genre'], data)
             self.finish_batch()
 
+    def import_emblems(self, emblems: List[Dict[str, Any]]) -> None:
+        if self.version is None:
+            raise Exception('Can\'t import Jubeat database for \'all\' version!')
+
+        self.start_batch()
+        for i, emblem in enumerate(emblems):
+            # Make importing faster but still do it in chunks
+            if (i % 16) == 15:
+                self.finish_batch()
+                self.start_batch()
+
+            print(f"New catalog entry for {emblem['music_id']}")
+            self.insert_catalog_entry(
+                'emblem',
+                emblem['id'],
+                {
+                    'layer': emblem['layer'],
+                    'music_id': emblem['music_id'],
+                    'evolved': emblem['evolved'],
+                    'rarity': emblem['rarity'],
+                    'name': emblem['name'],
+                },
+            )
+
+        self.finish_batch()
+
     def import_metadata(self, tsvfile: str) -> None:
         if self.version is not None:
             raise Exception("Unsupported Jubeat version, expected one of the following: all")
@@ -3381,17 +3459,18 @@ if __name__ == "__main__":
             # hand-populated since its not in the music DB.
             jubeat.import_metadata(args.tsv)
         else:
-            # Normal case, doing a music DB import.
+            # Normal case, doing a music DB or emblem import.
             if args.xml is not None:
-                songs = jubeat.scrape(args.xml)
+                songs, emblems = jubeat.scrape(args.xml)
             elif args.server and args.token:
-                songs = jubeat.lookup(args.server, args.token)
+                songs, emblems = jubeat.lookup(args.server, args.token)
             else:
                 raise Exception(
                     'No music_info.xml or TSV provided and no remote server specified! Please ' +
                     'provide either a --xml, --tsv or a --server and --token option!'
                 )
             jubeat.import_music_db(songs)
+            jubeat.import_emblems(emblems)
         jubeat.close()
 
     elif args.series == GameConstants.IIDX: