Set up an asset concept, use it to display emblem previews on Jubeat settings. Huge thanks to Subject38 for lots of this code!

README.md

@@ -59,6 +59,14 @@ A utility for unpacking `.arc` files. This does not currently repack files. Howe
 the format is so trivial that adding such a feature would be fairly easy. Run it
 like `./arcutils --help` to see help output and determine how to use this.
 
+## assetparse
+
+A utility which takes a particular game's asset directory and converts files for use
+on the frontend. This optionally enables things such as customization previews on the
+frontend. Much like "read", this requires a properly setup config file which points
+the frontend as well as this utility at the correct location to store converted
+assets. See `config/server.yaml` for an example file that you can modify.
+
 ## bemanishark
 
 A wire sniffer that can decode eAmuse packets and print them. Run it on a computer
@@ -463,6 +471,16 @@ will not work properly. An example is as follows:
       --xml data/emblem-info/emblem-info.xml
 ```
 
+If you wish to display emblem graphics on the frontend, you will also need to convert the
+emblem assets. Failing to do so will disable the emblem graphics rendering feature for the
+frontend. Note that this only applies to versions where you have also imported the emblem
+DB. An example is as follows:
+
+```
+./assetparse --config config/server.yaml --series jubeat --version prop \
+      --xml data/emblem-info/emblem-info.xml --assets data/emblem-textures/
+```
+
 ### IIDX
 
 For IIDX, you will need the data directory of the mix you wish to support. The import
@@ -735,7 +753,9 @@ can set up a nginx directory to serve the static resources directly by pointing
 static directory inside your virtualenv.
 
 For example configurations, an example install script, and an example script to back
-up your MySQL instance, see the `examples/` directory.
+up your MySQL instance, see the `examples/` directory. Note that several files have
+sections where you are expected to substitute your own values so please read over them
+carefully.
 
 # Contributing
 

assetparse

@@ -0,0 +1,11 @@
+#! /usr/bin/env python3
+if __name__ == "__main__":
+	import os
+	path = os.path.abspath(os.path.dirname(__file__))
+	name = os.path.basename(__file__)
+
+	import sys
+	sys.path.append(path)
+
+	import runpy
+	runpy.run_module(f"bemani.utils.{name}", run_name="__main__")

bemani/data/config.py

@@ -160,6 +160,21 @@ class DiscordWebHooks:
         return str(uri) if uri else None
 
 
+class Assets:
+    def __init__(self, parent_config: "Config") -> None:
+        self.jubeat = JubeatAssets(parent_config)
+
+
+class JubeatAssets:
+    def __init__(self, parent_config: "Config") -> None:
+        self.__config = parent_config
+
+    @property
+    def emblems(self) -> Optional[str]:
+        directory = self.__config.get("assets", {}).get("jubeat", {}).get("emblems")
+        return str(directory) if directory else None
+
+
 class Config(dict):
     def __init__(self, existing_contents: Dict[str, Any] = {}) -> None:
         super().__init__(existing_contents or {})
@@ -169,6 +184,7 @@ class Config(dict):
         self.client = Client(self)
         self.paseli = PASELI(self)
         self.webhooks = WebHooks(self)
+        self.assets = Assets(self)
         self.machine = Machine(self)
 
     def clone(self) -> "Config":

bemani/frontend/app.py

@@ -1,3 +1,4 @@
+import mimetypes
 import os
 import re
 import traceback
@@ -10,6 +11,7 @@ from flask import (
     redirect,
     Response,
     url_for,
+    abort,
     render_template,
     got_request_exception,
     jsonify as flask_jsonify,
@@ -152,6 +154,10 @@ def cacheable(max_age: int) -> Callable:
     return __cache
 
 
+# Note that this should really only be used for debug builds. In production, you should use
+# the "jsx" utility to bulk convert your JSX files and put them in a directory where your
+# actual webserver (nginx, apache, etc) can find them and serve them without going through
+# this endpoint.
 @app.route("/jsx/<path:filename>")
 @cacheable(86400)
 def jsx(filename: str) -> Response:
@@ -200,6 +206,36 @@ def jsx(filename: str) -> Response:
             raise
 
 
+# Note that this should really only be used for debug builds. In production, you should use
+# the "assetparse" utility to bulk convert your game asset files and put them in directories
+# where your actual webserver (nginx, apache, etc) can find them and serve them without
+# going through this endpoint.
+@app.route("/assets/<path:filename>")
+@cacheable(86400)
+def assets(filename: str) -> Response:
+    # Map of all assets. We could walk the config using reflection, but meh.
+    assetdirs: Dict[str, str] = {
+        "jubeat/emblems/": config.assets.jubeat.emblems,
+    }
+
+    # Figure out what asset pack this is from.
+    for prefix, directory in assetdirs.items():
+        if filename.startswith(prefix):
+            filename = filename[len(prefix) :]
+            normalized_path = os.path.join(directory, filename)
+
+            # Check for path traversal exploit
+            if not normalized_path.startswith(directory):
+                raise IOError("Path traversal exploit detected!")
+
+            mimetype, _ = mimetypes.guess_type(normalized_path)
+            with open(normalized_path, "rb") as f:
+                return Response(f.read(), mimetype=mimetype)
+    else:
+        # No asset for this.
+        abort(404)
+
+
 def polyfill_fragments(jsx: str) -> str:
     jsx = jsx.replace("<>", "<React.Fragment>")
     jsx = jsx.replace("</>", "</React.Fragment>")

bemani/frontend/jubeat/endpoints.py

@@ -363,6 +363,7 @@ def viewsettings() -> Response:
                 version: name for (game, version, name) in frontend.all_games()
             },
             "emblems": all_emblems,
+            "assets_available": g.config.assets.jubeat.emblems is not None,
         },
         {
             "updatename": url_for("jubeat_pages.updatename"),

bemani/frontend/static/controllers/jubeat/settings.react.js

@@ -154,10 +154,46 @@ var settings_view = createReactClass({
         );
     },
 
+    getDefaultEmblem: function() {
+        // A hack for displaying defaults when they aren't set.
+        var items = window.emblems[this.state.version].filter(function (emblem) {
+            return emblem.rarity == 1 && emblem.layer == 2;
+        }.bind(this));
+
+        if (items.length > 0) {
+            return items[0].index;
+        }
+
+        return 0;
+    },
+
     renderEmblem: function(player) {
         return (
             <div className="section">
                 <h3>Emblem</h3>
+                {
+                    window.assets_available ?
+                        <div className="emblem">
+                        {
+                            valid_emblem_options.map(function(emblem_option) {
+                                var player = this.state.player[this.state.version];
+                                var emblem = player.emblem[emblem_option];
+                                if (emblem_option == "main" && emblem == 0) {
+                                    emblem = this.getDefaultEmblem();
+                                }
+
+                                if (emblem == 0) {
+                                    // Emblem part isn't set.
+                                    return null;
+                                }
+
+                                var player = this.state.player[this.state.version]
+                                var src = `/assets/jubeat/emblems/${this.state.version}/${emblem}.png`
+                                return <div><img src={src}/></div>;
+                            }.bind(this))
+                        }
+                        </div> : null
+                }
                 {
                     valid_emblem_options.map(function(emblem_option) {
                         var player = this.state.player[this.state.version]

bemani/frontend/static/themes/dark/section.css

@@ -62,6 +62,10 @@ div.labelledsection.iidx.qprooption select {
     width: 300px;
 }
 
+div.labelledsection.jubeat.emblemoption select {
+    width: 300px;
+}
+
 div.labelledsection.ddr.option select {
     width: 200px;
 }
@@ -197,3 +201,19 @@ span.slider-label {
     margin-top: 1px;
     margin-bottom: -1px;
 } 
+
+div.emblem {
+    position: relative;
+    width: 256px;
+    height: 256px;
+    padding-bottom: 10px;
+}
+
+div.emblem div {
+    position: absolute;
+}
+
+div.emblem div img {
+    width: 256px;
+    height: 256px;
+}

bemani/frontend/static/themes/default/section.css

@@ -62,6 +62,10 @@ div.labelledsection.iidx.qprooption select {
     width: 300px;
 }
 
+div.labelledsection.jubeat.emblemoption select {
+    width: 300px;
+}
+
 div.labelledsection.ddr.option select {
     width: 200px;
 }
@@ -197,3 +201,19 @@ span.slider-label {
     margin-top: 1px;
     margin-bottom: -1px;
 }
+
+div.emblem {
+    position: relative;
+    width: 256px;
+    height: 256px;
+    padding-bottom: 10px;
+}
+
+div.emblem div {
+    position: absolute;
+}
+
+div.emblem div img {
+    width: 256px;
+    height: 256px;
+}

bemani/utils/assetparse.py

@@ -0,0 +1,163 @@
+# vim: set fileencoding=utf-8
+
+import argparse
+import os
+import xml.etree.ElementTree as ET
+from PIL import Image  # type: ignore
+from typing import Dict, Optional
+
+from bemani.common import GameConstants, VersionConstants
+from bemani.data import Config
+from bemani.utils.config import load_config
+
+
+class ImportJubeat:
+    def __init__(self, config: Config, version: str, update: bool) -> None:
+        actual_version = {
+            "saucer": VersionConstants.JUBEAT_SAUCER,
+            "saucer-fulfill": VersionConstants.JUBEAT_SAUCER_FULFILL,
+            "prop": VersionConstants.JUBEAT_PROP,
+            "qubell": VersionConstants.JUBEAT_QUBELL,
+            "clan": VersionConstants.JUBEAT_CLAN,
+            "festo": VersionConstants.JUBEAT_FESTO,
+        }.get(version, -1)
+        if actual_version in {
+            VersionConstants.JUBEAT_PROP,
+            VersionConstants.JUBEAT_QUBELL,
+            VersionConstants.JUBEAT_CLAN,
+            VersionConstants.JUBEAT_FESTO,
+        }:
+            self.version = actual_version
+        else:
+            raise Exception(
+                "Unsupported Jubeat version, expected one of the following: prop, qubell, clan, festo!"
+            )
+
+        self.config = config
+        self.update = update
+
+    def import_assets(self, xml: str, assets: Optional[str]) -> None:
+        if assets is None:
+            raise Exception("Expect a valid asset directory when importing emblems!")
+
+        with open(args.xml, "rb") as xmlhandle:
+            xmldata = xmlhandle.read().decode("shift_jisx0213")
+        root = ET.fromstring(xmldata)
+
+        file_mapping: Dict[str, str] = {}
+        for emblem in root.find("emblem_list"):
+            emblem.find("texname").text = emblem.find("texname").text.replace(
+                ".tex", ".png"
+            )
+            file_mapping[
+                emblem.find("texname").text
+            ] = f'{emblem.find("index").text}.png'
+
+        if not file_mapping:
+            # This isn't an emblem XML!
+            raise Exception("Expect a valid emblem-info.xml file!")
+
+        if not self.config.assets.jubeat.emblems:
+            # We don't have the output set!
+            raise Exception("Expect a valid directory for emblems in config file!")
+
+        # First, make the root directory structure.
+        actual_output = os.path.join(
+            self.config.assets.jubeat.emblems, f"{self.version}"
+        )
+        os.makedirs(actual_output, exist_ok=True)
+
+        for fileroot, _, files in os.walk(assets):
+            for filename in files:
+                filepath = os.path.join(fileroot, filename)
+                outname = os.path.splitext(filename)[0]
+                renamed = file_mapping.get(f"{outname}.png")
+                if renamed is None:
+                    print(f"No mapping for {filepath}, skipping!")
+                    continue
+
+                full_renamed = os.path.join(actual_output, renamed)
+
+                if os.path.exists(full_renamed) and not self.update:
+                    print(f"Skipping existing {full_renamed}!")
+                    continue
+
+                print(f"Converting {filepath} to {full_renamed}...")
+
+                # This is the image parsing section. Basically raw pixel data starting on byte 34 of the tex file.
+                rawData = open(filepath, "rb").read()
+                imgSize = (512, 512)  # the image size
+                img = Image.frombytes("RGBA", imgSize, rawData[0x34:])
+                img.save(full_renamed)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Import game assets from various game files"
+    )
+    parser.add_argument(
+        "--series",
+        action="store",
+        type=str,
+        required=True,
+        help="The game series we are importing.",
+    )
+    parser.add_argument(
+        "--version",
+        dest="version",
+        action="store",
+        type=str,
+        required=True,
+        help="The game version we are importing.",
+    )
+    parser.add_argument(
+        "--config",
+        type=str,
+        default="config.yaml",
+        help="Core configuration for determining the output location. Defaults to 'config.yaml'.",
+    )
+    parser.add_argument(
+        "--update",
+        dest="update",
+        action="store_true",
+        default=False,
+        help="Overwrite data with updated values when it already exists.",
+    )
+    parser.add_argument(
+        "--xml",
+        dest="xml",
+        action="store",
+        type=str,
+        help="The game XML file to read, for applicable games.",
+    )
+    parser.add_argument(
+        "--assets",
+        dest="assets",
+        action="store",
+        type=str,
+        help="The game assets directory, for applicable games.",
+    )
+    args = parser.parse_args()
+
+    # Load the config so we can put assets in the right directory.
+    config = Config()
+    load_config(args.config, config)
+
+    series = None
+    try:
+        series = GameConstants(args.series)
+    except ValueError:
+        pass
+
+    if series == GameConstants.JUBEAT:
+        jubeat = ImportJubeat(config, args.version, args.update)
+
+        if args.xml:
+            jubeat.import_assets(args.xml, args.assets)
+        else:
+            raise Exception(
+                "No emblem-info.xml provided! Please provide a --xml option!"
+            )
+
+    else:
+        raise Exception("Unsupported game series!")

config/server.yaml

@@ -54,6 +54,11 @@ webhooks:
         pnm: 
         - "https://discord.com/api/webhooks/1232122131321321321/eauihfafaewfhjaveuijaewuivhjawueihoi"
 
+# Assets URLs. These allow for in-game asset rendering on the front end. Delete this to disable asset rendering.
+assets:
+    jubeat:
+        emblems: "/directory/where/you/output/emblem/assets"
+
 # Global PASESLI settings, which can be overridden on a per-arcade basis. These form the default settings.
 paseli:
     # Whether PASELI is enabled on the network.

examples/nginx/frontend.nginx

@@ -19,4 +19,14 @@ server {
         include  /etc/nginx/mime.types;
         root /path/to/your/virtualenv/lib/python3.6/site-packages/bemani/frontend/;
     }
+
+    location ^~ /jsx/ {
+        include  /etc/nginx/mime.types;
+        root /path/to/your/converted/jsx/files/;
+    }
+
+    location ^~ /assets/ {
+        include  /etc/nginx/mime.types;
+        root /path/to/your/converted/asset/files/;
+    }
 }

verifytyping

@@ -4,6 +4,7 @@ declare -a arr=(
     "api"
     "afputils"
     "arcutils"
+    "assetparse"
     "bemanishark"
     "binutils"
     "cardconvert"