From c61d08a5546117da29ae5626453b4df6771fc74a Mon Sep 17 00:00:00 2001 From: Jennifer Taylor Date: Sun, 13 Nov 2022 16:56:59 +0000 Subject: [PATCH] Set up an asset concept, use it to display emblem previews on Jubeat settings. Huge thanks to Subject38 for lots of this code! --- README.md | 22 ++- assetparse | 11 ++ bemani/data/config.py | 16 ++ bemani/frontend/app.py | 36 ++++ bemani/frontend/jubeat/endpoints.py | 1 + .../controllers/jubeat/settings.react.js | 36 ++++ .../frontend/static/themes/dark/section.css | 20 +++ .../static/themes/default/section.css | 20 +++ bemani/utils/assetparse.py | 163 ++++++++++++++++++ config/server.yaml | 5 + examples/nginx/frontend.nginx | 10 ++ verifytyping | 1 + 12 files changed, 340 insertions(+), 1 deletion(-) create mode 100755 assetparse create mode 100644 bemani/utils/assetparse.py diff --git a/README.md b/README.md index 0de0173..65681e3 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,14 @@ A utility for unpacking `.arc` files. This does not currently repack files. Howe the format is so trivial that adding such a feature would be fairly easy. Run it like `./arcutils --help` to see help output and determine how to use this. +## assetparse + +A utility which takes a particular game's asset directory and converts files for use +on the frontend. This optionally enables things such as customization previews on the +frontend. Much like "read", this requires a properly setup config file which points +the frontend as well as this utility at the correct location to store converted +assets. See `config/server.yaml` for an example file that you can modify. + ## bemanishark A wire sniffer that can decode eAmuse packets and print them. Run it on a computer @@ -463,6 +471,16 @@ will not work properly. An example is as follows: --xml data/emblem-info/emblem-info.xml ``` +If you wish to display emblem graphics on the frontend, you will also need to convert the +emblem assets. Failing to do so will disable the emblem graphics rendering feature for the +frontend. Note that this only applies to versions where you have also imported the emblem +DB. An example is as follows: + +``` +./assetparse --config config/server.yaml --series jubeat --version prop \ + --xml data/emblem-info/emblem-info.xml --assets data/emblem-textures/ +``` + ### IIDX For IIDX, you will need the data directory of the mix you wish to support. The import @@ -735,7 +753,9 @@ can set up a nginx directory to serve the static resources directly by pointing static directory inside your virtualenv. For example configurations, an example install script, and an example script to back -up your MySQL instance, see the `examples/` directory. +up your MySQL instance, see the `examples/` directory. Note that several files have +sections where you are expected to substitute your own values so please read over them +carefully. # Contributing diff --git a/assetparse b/assetparse new file mode 100755 index 0000000..ce6f396 --- /dev/null +++ b/assetparse @@ -0,0 +1,11 @@ +#! /usr/bin/env python3 +if __name__ == "__main__": + import os + path = os.path.abspath(os.path.dirname(__file__)) + name = os.path.basename(__file__) + + import sys + sys.path.append(path) + + import runpy + runpy.run_module(f"bemani.utils.{name}", run_name="__main__") \ No newline at end of file diff --git a/bemani/data/config.py b/bemani/data/config.py index 7505bf4..cabdae0 100644 --- a/bemani/data/config.py +++ b/bemani/data/config.py @@ -160,6 +160,21 @@ class DiscordWebHooks: return str(uri) if uri else None +class Assets: + def __init__(self, parent_config: "Config") -> None: + self.jubeat = JubeatAssets(parent_config) + + +class JubeatAssets: + def __init__(self, parent_config: "Config") -> None: + self.__config = parent_config + + @property + def emblems(self) -> Optional[str]: + directory = self.__config.get("assets", {}).get("jubeat", {}).get("emblems") + return str(directory) if directory else None + + class Config(dict): def __init__(self, existing_contents: Dict[str, Any] = {}) -> None: super().__init__(existing_contents or {}) @@ -169,6 +184,7 @@ class Config(dict): self.client = Client(self) self.paseli = PASELI(self) self.webhooks = WebHooks(self) + self.assets = Assets(self) self.machine = Machine(self) def clone(self) -> "Config": diff --git a/bemani/frontend/app.py b/bemani/frontend/app.py index e2e4151..97c534a 100644 --- a/bemani/frontend/app.py +++ b/bemani/frontend/app.py @@ -1,3 +1,4 @@ +import mimetypes import os import re import traceback @@ -10,6 +11,7 @@ from flask import ( redirect, Response, url_for, + abort, render_template, got_request_exception, jsonify as flask_jsonify, @@ -152,6 +154,10 @@ def cacheable(max_age: int) -> Callable: return __cache +# Note that this should really only be used for debug builds. In production, you should use +# the "jsx" utility to bulk convert your JSX files and put them in a directory where your +# actual webserver (nginx, apache, etc) can find them and serve them without going through +# this endpoint. @app.route("/jsx/") @cacheable(86400) def jsx(filename: str) -> Response: @@ -200,6 +206,36 @@ def jsx(filename: str) -> Response: raise +# Note that this should really only be used for debug builds. In production, you should use +# the "assetparse" utility to bulk convert your game asset files and put them in directories +# where your actual webserver (nginx, apache, etc) can find them and serve them without +# going through this endpoint. +@app.route("/assets/") +@cacheable(86400) +def assets(filename: str) -> Response: + # Map of all assets. We could walk the config using reflection, but meh. + assetdirs: Dict[str, str] = { + "jubeat/emblems/": config.assets.jubeat.emblems, + } + + # Figure out what asset pack this is from. + for prefix, directory in assetdirs.items(): + if filename.startswith(prefix): + filename = filename[len(prefix) :] + normalized_path = os.path.join(directory, filename) + + # Check for path traversal exploit + if not normalized_path.startswith(directory): + raise IOError("Path traversal exploit detected!") + + mimetype, _ = mimetypes.guess_type(normalized_path) + with open(normalized_path, "rb") as f: + return Response(f.read(), mimetype=mimetype) + else: + # No asset for this. + abort(404) + + def polyfill_fragments(jsx: str) -> str: jsx = jsx.replace("<>", "") jsx = jsx.replace("", "") diff --git a/bemani/frontend/jubeat/endpoints.py b/bemani/frontend/jubeat/endpoints.py index 245bd70..28de520 100644 --- a/bemani/frontend/jubeat/endpoints.py +++ b/bemani/frontend/jubeat/endpoints.py @@ -363,6 +363,7 @@ def viewsettings() -> Response: version: name for (game, version, name) in frontend.all_games() }, "emblems": all_emblems, + "assets_available": g.config.assets.jubeat.emblems is not None, }, { "updatename": url_for("jubeat_pages.updatename"), diff --git a/bemani/frontend/static/controllers/jubeat/settings.react.js b/bemani/frontend/static/controllers/jubeat/settings.react.js index 4189581..fa2b659 100644 --- a/bemani/frontend/static/controllers/jubeat/settings.react.js +++ b/bemani/frontend/static/controllers/jubeat/settings.react.js @@ -154,10 +154,46 @@ var settings_view = createReactClass({ ); }, + getDefaultEmblem: function() { + // A hack for displaying defaults when they aren't set. + var items = window.emblems[this.state.version].filter(function (emblem) { + return emblem.rarity == 1 && emblem.layer == 2; + }.bind(this)); + + if (items.length > 0) { + return items[0].index; + } + + return 0; + }, + renderEmblem: function(player) { return (

Emblem

+ { + window.assets_available ? +
+ { + valid_emblem_options.map(function(emblem_option) { + var player = this.state.player[this.state.version]; + var emblem = player.emblem[emblem_option]; + if (emblem_option == "main" && emblem == 0) { + emblem = this.getDefaultEmblem(); + } + + if (emblem == 0) { + // Emblem part isn't set. + return null; + } + + var player = this.state.player[this.state.version] + var src = `/assets/jubeat/emblems/${this.state.version}/${emblem}.png` + return
; + }.bind(this)) + } +
: null + } { valid_emblem_options.map(function(emblem_option) { var player = this.state.player[this.state.version] diff --git a/bemani/frontend/static/themes/dark/section.css b/bemani/frontend/static/themes/dark/section.css index 64f7353..e5e708c 100644 --- a/bemani/frontend/static/themes/dark/section.css +++ b/bemani/frontend/static/themes/dark/section.css @@ -62,6 +62,10 @@ div.labelledsection.iidx.qprooption select { width: 300px; } +div.labelledsection.jubeat.emblemoption select { + width: 300px; +} + div.labelledsection.ddr.option select { width: 200px; } @@ -197,3 +201,19 @@ span.slider-label { margin-top: 1px; margin-bottom: -1px; } + +div.emblem { + position: relative; + width: 256px; + height: 256px; + padding-bottom: 10px; +} + +div.emblem div { + position: absolute; +} + +div.emblem div img { + width: 256px; + height: 256px; +} diff --git a/bemani/frontend/static/themes/default/section.css b/bemani/frontend/static/themes/default/section.css index 7d51e0e..64612a1 100644 --- a/bemani/frontend/static/themes/default/section.css +++ b/bemani/frontend/static/themes/default/section.css @@ -62,6 +62,10 @@ div.labelledsection.iidx.qprooption select { width: 300px; } +div.labelledsection.jubeat.emblemoption select { + width: 300px; +} + div.labelledsection.ddr.option select { width: 200px; } @@ -197,3 +201,19 @@ span.slider-label { margin-top: 1px; margin-bottom: -1px; } + +div.emblem { + position: relative; + width: 256px; + height: 256px; + padding-bottom: 10px; +} + +div.emblem div { + position: absolute; +} + +div.emblem div img { + width: 256px; + height: 256px; +} diff --git a/bemani/utils/assetparse.py b/bemani/utils/assetparse.py new file mode 100644 index 0000000..559cb8c --- /dev/null +++ b/bemani/utils/assetparse.py @@ -0,0 +1,163 @@ +# vim: set fileencoding=utf-8 + +import argparse +import os +import xml.etree.ElementTree as ET +from PIL import Image # type: ignore +from typing import Dict, Optional + +from bemani.common import GameConstants, VersionConstants +from bemani.data import Config +from bemani.utils.config import load_config + + +class ImportJubeat: + def __init__(self, config: Config, version: str, update: bool) -> None: + actual_version = { + "saucer": VersionConstants.JUBEAT_SAUCER, + "saucer-fulfill": VersionConstants.JUBEAT_SAUCER_FULFILL, + "prop": VersionConstants.JUBEAT_PROP, + "qubell": VersionConstants.JUBEAT_QUBELL, + "clan": VersionConstants.JUBEAT_CLAN, + "festo": VersionConstants.JUBEAT_FESTO, + }.get(version, -1) + if actual_version in { + VersionConstants.JUBEAT_PROP, + VersionConstants.JUBEAT_QUBELL, + VersionConstants.JUBEAT_CLAN, + VersionConstants.JUBEAT_FESTO, + }: + self.version = actual_version + else: + raise Exception( + "Unsupported Jubeat version, expected one of the following: prop, qubell, clan, festo!" + ) + + self.config = config + self.update = update + + def import_assets(self, xml: str, assets: Optional[str]) -> None: + if assets is None: + raise Exception("Expect a valid asset directory when importing emblems!") + + with open(args.xml, "rb") as xmlhandle: + xmldata = xmlhandle.read().decode("shift_jisx0213") + root = ET.fromstring(xmldata) + + file_mapping: Dict[str, str] = {} + for emblem in root.find("emblem_list"): + emblem.find("texname").text = emblem.find("texname").text.replace( + ".tex", ".png" + ) + file_mapping[ + emblem.find("texname").text + ] = f'{emblem.find("index").text}.png' + + if not file_mapping: + # This isn't an emblem XML! + raise Exception("Expect a valid emblem-info.xml file!") + + if not self.config.assets.jubeat.emblems: + # We don't have the output set! + raise Exception("Expect a valid directory for emblems in config file!") + + # First, make the root directory structure. + actual_output = os.path.join( + self.config.assets.jubeat.emblems, f"{self.version}" + ) + os.makedirs(actual_output, exist_ok=True) + + for fileroot, _, files in os.walk(assets): + for filename in files: + filepath = os.path.join(fileroot, filename) + outname = os.path.splitext(filename)[0] + renamed = file_mapping.get(f"{outname}.png") + if renamed is None: + print(f"No mapping for {filepath}, skipping!") + continue + + full_renamed = os.path.join(actual_output, renamed) + + if os.path.exists(full_renamed) and not self.update: + print(f"Skipping existing {full_renamed}!") + continue + + print(f"Converting {filepath} to {full_renamed}...") + + # This is the image parsing section. Basically raw pixel data starting on byte 34 of the tex file. + rawData = open(filepath, "rb").read() + imgSize = (512, 512) # the image size + img = Image.frombytes("RGBA", imgSize, rawData[0x34:]) + img.save(full_renamed) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Import game assets from various game files" + ) + parser.add_argument( + "--series", + action="store", + type=str, + required=True, + help="The game series we are importing.", + ) + parser.add_argument( + "--version", + dest="version", + action="store", + type=str, + required=True, + help="The game version we are importing.", + ) + parser.add_argument( + "--config", + type=str, + default="config.yaml", + help="Core configuration for determining the output location. Defaults to 'config.yaml'.", + ) + parser.add_argument( + "--update", + dest="update", + action="store_true", + default=False, + help="Overwrite data with updated values when it already exists.", + ) + parser.add_argument( + "--xml", + dest="xml", + action="store", + type=str, + help="The game XML file to read, for applicable games.", + ) + parser.add_argument( + "--assets", + dest="assets", + action="store", + type=str, + help="The game assets directory, for applicable games.", + ) + args = parser.parse_args() + + # Load the config so we can put assets in the right directory. + config = Config() + load_config(args.config, config) + + series = None + try: + series = GameConstants(args.series) + except ValueError: + pass + + if series == GameConstants.JUBEAT: + jubeat = ImportJubeat(config, args.version, args.update) + + if args.xml: + jubeat.import_assets(args.xml, args.assets) + else: + raise Exception( + "No emblem-info.xml provided! Please provide a --xml option!" + ) + + else: + raise Exception("Unsupported game series!") diff --git a/config/server.yaml b/config/server.yaml index 64f9c63..ad03ff8 100644 --- a/config/server.yaml +++ b/config/server.yaml @@ -54,6 +54,11 @@ webhooks: pnm: - "https://discord.com/api/webhooks/1232122131321321321/eauihfafaewfhjaveuijaewuivhjawueihoi" +# Assets URLs. These allow for in-game asset rendering on the front end. Delete this to disable asset rendering. +assets: + jubeat: + emblems: "/directory/where/you/output/emblem/assets" + # Global PASESLI settings, which can be overridden on a per-arcade basis. These form the default settings. paseli: # Whether PASELI is enabled on the network. diff --git a/examples/nginx/frontend.nginx b/examples/nginx/frontend.nginx index 4e16693..8d50ded 100644 --- a/examples/nginx/frontend.nginx +++ b/examples/nginx/frontend.nginx @@ -19,4 +19,14 @@ server { include /etc/nginx/mime.types; root /path/to/your/virtualenv/lib/python3.6/site-packages/bemani/frontend/; } + + location ^~ /jsx/ { + include /etc/nginx/mime.types; + root /path/to/your/converted/jsx/files/; + } + + location ^~ /assets/ { + include /etc/nginx/mime.types; + root /path/to/your/converted/asset/files/; + } } diff --git a/verifytyping b/verifytyping index ed66b36..6ba1f8a 100755 --- a/verifytyping +++ b/verifytyping @@ -4,6 +4,7 @@ declare -a arr=( "api" "afputils" "arcutils" + "assetparse" "bemanishark" "binutils" "cardconvert"