From 2491eb0767649ccd216b4831c4d8f7df54a35ebe Mon Sep 17 00:00:00 2001 From: Jennifer Taylor Date: Sun, 13 Aug 2023 18:56:43 +0000 Subject: [PATCH] Promote frontend-only cache to a system-wide context, tweak application entrypoints to work with new bemani.common.cache --- bemani/common/__init__.py | 2 ++ bemani/common/cache.py | 17 ++++++++++ bemani/data/config.py | 14 +++++++-- bemani/frontend/app.py | 15 +++------ bemani/frontend/ddr/cache.py | 11 +------ bemani/frontend/iidx/cache.py | 11 +------ bemani/frontend/jubeat/cache.py | 11 +------ bemani/frontend/museca/cache.py | 11 +------ bemani/frontend/popn/cache.py | 11 +------ bemani/frontend/reflec/cache.py | 11 +------ bemani/frontend/sdvx/cache.py | 11 +------ bemani/frontend/templates/admin/settings.html | 4 +++ bemani/utils/api.py | 12 ++++++- bemani/utils/config.py | 31 ++++++++++++++++++- bemani/utils/frontend.py | 8 +++++ bemani/utils/scheduler.py | 6 +++- bemani/utils/services.py | 9 ++++++ bemani/wsgi/api.wsgi | 3 +- bemani/wsgi/frontend.wsgi | 3 +- bemani/wsgi/proxy.wsgi | 3 +- bemani/wsgi/services.wsgi | 3 +- config/server.yaml | 6 +++- 22 files changed, 123 insertions(+), 90 deletions(-) create mode 100644 bemani/common/cache.py diff --git a/bemani/common/__init__.py b/bemani/common/__init__.py index 13db380..9179196 100644 --- a/bemani/common/__init__.py +++ b/bemani/common/__init__.py @@ -15,6 +15,7 @@ from bemani.common.aes import AESCipher from bemani.common.time import Time from bemani.common.parallel import Parallel from bemani.common.pe import PEFile, InvalidOffsetException +from bemani.common.cache import cache __all__ = [ @@ -38,4 +39,5 @@ __all__ = [ "intish", "PEFile", "InvalidOffsetException", + "cache", ] diff --git a/bemani/common/cache.py b/bemani/common/cache.py new file mode 100644 index 0000000..1f61509 --- /dev/null +++ b/bemani/common/cache.py @@ -0,0 +1,17 @@ +from flask import Flask +from flask_caching import Cache + + +# This somewhat breaks convention of trying to keep flask stuff in only the application +# routing layer, but flask-caching itself is a useful wrapper that supports a ton of +# backends like in-memory, filesystem, redis, memcached, etc. So, we centralize the +# object's ownership here and call into this to initialize it during application setup +# so that anywhere in the codebase can assume an initialized cache object for decorators. +app = Flask(__name__) + +cache = Cache( + app, + # We should overwrite this in any reasonable entrypoint to the system, but for simple + # utilities that don't want to set up the entire infrastructure, provide a sane default. + config={"CACHE_TYPE": "SimpleCache"}, +) diff --git a/bemani/data/config.py b/bemani/data/config.py index cabdae0..88cf45f 100644 --- a/bemani/data/config.py +++ b/bemani/data/config.py @@ -237,8 +237,18 @@ class Config(dict): return str(self.get("email", "nobody@nowhere.com")) @property - def cache_dir(self) -> str: - return os.path.abspath(str(self.get("cache_dir", "/tmp"))) + def cache_dir(self) -> Optional[str]: + cache_dir = self.get("cache_dir") + if cache_dir is None: + return None + return os.path.abspath(str(cache_dir)) + + @property + def memcached_server(self) -> Optional[str]: + server = self.get("memcached_server") + if server is None: + return None + return str(server) @property def theme(self) -> str: diff --git a/bemani/frontend/app.py b/bemani/frontend/app.py index 8b85702..0c07acf 100644 --- a/bemani/frontend/app.py +++ b/bemani/frontend/app.py @@ -16,10 +16,9 @@ from flask import ( got_request_exception, jsonify as flask_jsonify, ) -from flask_caching import Cache from functools import wraps -from bemani.common import AESCipher, GameConstants +from bemani.common import AESCipher, GameConstants, cache from bemani.data import Config, Data from bemani.frontend.types import g from bemani.frontend.templates import templates_location @@ -40,18 +39,14 @@ FRONTEND_CACHE_BUST: str = "site.1.3.react.16.14" @app.before_request def before_request() -> None: global config - g.cache = Cache( - app, - config={ - "CACHE_TYPE": "filesystem", - "CACHE_DIR": config.cache_dir, - }, - ) + + g.cache = cache + g.config = config + if request.endpoint in ["jsx", "static"]: # This is just serving cached compiled frontends, skip loading from DB return - g.config = config g.data = Data(config) g.sessionID = None g.userID = None diff --git a/bemani/frontend/ddr/cache.py b/bemani/frontend/ddr/cache.py index b339a7e..bbd57cb 100644 --- a/bemani/frontend/ddr/cache.py +++ b/bemani/frontend/ddr/cache.py @@ -1,19 +1,10 @@ -from flask_caching import Cache - +from bemani.common import cache from bemani.data import Config, Data -from bemani.frontend.app import app from bemani.frontend.ddr.ddr import DDRFrontend class DDRCache: @classmethod def preload(cls, data: Data, config: Config) -> None: - cache = Cache( - app, - config={ - "CACHE_TYPE": "filesystem", - "CACHE_DIR": config.cache_dir, - }, - ) frontend = DDRFrontend(data, config, cache) frontend.get_all_songs(force_db_load=True) diff --git a/bemani/frontend/iidx/cache.py b/bemani/frontend/iidx/cache.py index 0d5099a..7e1754c 100644 --- a/bemani/frontend/iidx/cache.py +++ b/bemani/frontend/iidx/cache.py @@ -1,19 +1,10 @@ -from flask_caching import Cache - +from bemani.common import cache from bemani.data import Config, Data -from bemani.frontend.app import app from bemani.frontend.iidx.iidx import IIDXFrontend class IIDXCache: @classmethod def preload(cls, data: Data, config: Config) -> None: - cache = Cache( - app, - config={ - "CACHE_TYPE": "filesystem", - "CACHE_DIR": config.cache_dir, - }, - ) frontend = IIDXFrontend(data, config, cache) frontend.get_all_songs(force_db_load=True) diff --git a/bemani/frontend/jubeat/cache.py b/bemani/frontend/jubeat/cache.py index 7ebe1aa..ae9880b 100644 --- a/bemani/frontend/jubeat/cache.py +++ b/bemani/frontend/jubeat/cache.py @@ -1,19 +1,10 @@ -from flask_caching import Cache - +from bemani.common import cache from bemani.data import Config, Data -from bemani.frontend.app import app from bemani.frontend.jubeat.jubeat import JubeatFrontend class JubeatCache: @classmethod def preload(cls, data: Data, config: Config) -> None: - cache = Cache( - app, - config={ - "CACHE_TYPE": "filesystem", - "CACHE_DIR": config.cache_dir, - }, - ) frontend = JubeatFrontend(data, config, cache) frontend.get_all_songs(force_db_load=True) diff --git a/bemani/frontend/museca/cache.py b/bemani/frontend/museca/cache.py index c6e6518..a5b6307 100644 --- a/bemani/frontend/museca/cache.py +++ b/bemani/frontend/museca/cache.py @@ -1,19 +1,10 @@ -from flask_caching import Cache - +from bemani.common import cache from bemani.data import Config, Data -from bemani.frontend.app import app from bemani.frontend.museca.museca import MusecaFrontend class MusecaCache: @classmethod def preload(cls, data: Data, config: Config) -> None: - cache = Cache( - app, - config={ - "CACHE_TYPE": "filesystem", - "CACHE_DIR": config.cache_dir, - }, - ) frontend = MusecaFrontend(data, config, cache) frontend.get_all_songs(force_db_load=True) diff --git a/bemani/frontend/popn/cache.py b/bemani/frontend/popn/cache.py index ab5d924..78f6f22 100644 --- a/bemani/frontend/popn/cache.py +++ b/bemani/frontend/popn/cache.py @@ -1,19 +1,10 @@ -from flask_caching import Cache - +from bemani.common import cache from bemani.data import Config, Data -from bemani.frontend.app import app from bemani.frontend.popn.popn import PopnMusicFrontend class PopnMusicCache: @classmethod def preload(cls, data: Data, config: Config) -> None: - cache = Cache( - app, - config={ - "CACHE_TYPE": "filesystem", - "CACHE_DIR": config.cache_dir, - }, - ) frontend = PopnMusicFrontend(data, config, cache) frontend.get_all_songs(force_db_load=True) diff --git a/bemani/frontend/reflec/cache.py b/bemani/frontend/reflec/cache.py index 55f56c7..cffb461 100644 --- a/bemani/frontend/reflec/cache.py +++ b/bemani/frontend/reflec/cache.py @@ -1,19 +1,10 @@ -from flask_caching import Cache - +from bemani.common import cache from bemani.data import Config, Data -from bemani.frontend.app import app from bemani.frontend.reflec.reflec import ReflecBeatFrontend class ReflecBeatCache: @classmethod def preload(cls, data: Data, config: Config) -> None: - cache = Cache( - app, - config={ - "CACHE_TYPE": "filesystem", - "CACHE_DIR": config.cache_dir, - }, - ) frontend = ReflecBeatFrontend(data, config, cache) frontend.get_all_songs(force_db_load=True) diff --git a/bemani/frontend/sdvx/cache.py b/bemani/frontend/sdvx/cache.py index 656e786..8cb21e0 100644 --- a/bemani/frontend/sdvx/cache.py +++ b/bemani/frontend/sdvx/cache.py @@ -1,19 +1,10 @@ -from flask_caching import Cache - +from bemani.common import cache from bemani.data import Config, Data -from bemani.frontend.app import app from bemani.frontend.sdvx.sdvx import SoundVoltexFrontend class SoundVoltexCache: @classmethod def preload(cls, data: Data, config: Config) -> None: - cache = Cache( - app, - config={ - "CACHE_TYPE": "filesystem", - "CACHE_DIR": config.cache_dir, - }, - ) frontend = SoundVoltexFrontend(data, config, cache) frontend.get_all_songs(force_db_load=True) diff --git a/bemani/frontend/templates/admin/settings.html b/bemani/frontend/templates/admin/settings.html index 9a66b0f..85ef5b9 100644 --- a/bemani/frontend/templates/admin/settings.html +++ b/bemani/frontend/templates/admin/settings.html @@ -37,6 +37,10 @@
Cache Directory
{{ config.cache_dir }}
+
+
memcached Server
+
{{ config.memcached_server }}
+

Frontend Settings

diff --git a/bemani/utils/api.py b/bemani/utils/api.py index dfaaf81..287720b 100644 --- a/bemani/utils/api.py +++ b/bemani/utils/api.py @@ -1,7 +1,11 @@ import argparse +from typing import Any from bemani.api import app, config # noqa: F401 -from bemani.utils.config import load_config as base_load_config +from bemani.utils.config import ( + load_config as base_load_config, + instantiate_cache as base_instantiate_cache, +) def load_config(filename: str) -> None: @@ -9,6 +13,11 @@ def load_config(filename: str) -> None: base_load_config(filename, config) +def instantiate_cache(app: Any) -> None: + global config + base_instantiate_cache(app, config) + + def main() -> None: parser = argparse.ArgumentParser( description="An API services provider for eAmusement games, conforming to BEMAPI specs." @@ -48,6 +57,7 @@ def main() -> None: app.wsgi_app = ProfilerMiddleware(app.wsgi_app, profile_dir=".") # type: ignore # Run the app + instantiate_cache(app) app.run(host="0.0.0.0", port=args.port, debug=True) diff --git a/bemani/utils/config.py b/bemani/utils/config.py index f092168..81b9bae 100644 --- a/bemani/utils/config.py +++ b/bemani/utils/config.py @@ -1,4 +1,5 @@ import yaml +from flask import Flask from typing import Set from bemani.backend.iidx import IIDXFactory @@ -10,7 +11,7 @@ from bemani.backend.sdvx import SoundVoltexFactory from bemani.backend.reflec import ReflecBeatFactory from bemani.backend.museca import MusecaFactory from bemani.backend.mga import MetalGearArcadeFactory -from bemani.common import GameConstants +from bemani.common import GameConstants, cache from bemani.data import Config, Data @@ -26,6 +27,34 @@ def load_config(filename: str, config: Config) -> None: config["support"] = supported_series +def instantiate_cache(app: Flask, config: Config) -> None: + # This could easily be extended to add support for any other backend that flask-caching + # supports but right now the only demand is for in-memory, filesystem and memcached. + if config.memcached_server is not None: + cache.init_app( + app, + config={ + "CACHE_TYPE": "MemcachedCache", + "CACHE_MEMCACHED_SERVERS": [config.memcached_server], + }, + ) + elif config.cache_dir is not None: + cache.init_app( + app, + config={ + "CACHE_TYPE": "FileSystemCache", + "CACHE_DIR": config.cache_dir, + }, + ) + else: + cache.init_app( + app, + config={ + "CACHE_TYPE": "SimpleCache", + }, + ) + + def register_games(config: Config) -> None: if GameConstants.POPN_MUSIC in config.support: PopnMusicFactory.register_all() diff --git a/bemani/utils/frontend.py b/bemani/utils/frontend.py index 429cbea..e54d64a 100644 --- a/bemani/utils/frontend.py +++ b/bemani/utils/frontend.py @@ -1,4 +1,5 @@ import argparse +from typing import Any from bemani.common import GameConstants from bemani.frontend import app, config # noqa: F401 @@ -17,6 +18,7 @@ from bemani.frontend.reflec import reflec_pages from bemani.frontend.museca import museca_pages from bemani.utils.config import ( load_config as base_load_config, + instantiate_cache as base_instantiate_cache, register_games as base_register_games, ) @@ -60,6 +62,11 @@ def load_config(filename: str) -> None: app.secret_key = config.secret_key +def instantiate_cache(app: Any) -> None: + global config + base_instantiate_cache(app, config) + + def main() -> None: parser = argparse.ArgumentParser( description="A front end services provider for eAmusement games." @@ -105,6 +112,7 @@ def main() -> None: app.wsgi_app = ProfilerMiddleware(app.wsgi_app, profile_dir=".") # type: ignore # Run the app + instantiate_cache(app) app.run(host="0.0.0.0", port=args.port, debug=True) diff --git a/bemani/utils/scheduler.py b/bemani/utils/scheduler.py index fbec82a..f9fb1d4 100644 --- a/bemani/utils/scheduler.py +++ b/bemani/utils/scheduler.py @@ -1,4 +1,5 @@ import argparse +from flask import Flask from typing import Any, List from bemani.backend.popn import PopnMusicFactory @@ -21,7 +22,7 @@ from bemani.frontend.reflec import ReflecBeatCache from bemani.frontend.museca import MusecaCache from bemani.common import GameConstants, Time from bemani.data import Config, Data -from bemani.utils.config import load_config +from bemani.utils.config import load_config, instantiate_cache def run_scheduled_work(config: Config) -> None: @@ -99,5 +100,8 @@ if __name__ == "__main__": if args.read_only: config["database"]["read_only"] = True + # Set up production cache, with a dummy app context because flask-caching needs it. + instantiate_cache(Flask(__name__), config) + # Run out of band work run_scheduled_work(config) diff --git a/bemani/utils/services.py b/bemani/utils/services.py index 7a70e75..180cd5a 100644 --- a/bemani/utils/services.py +++ b/bemani/utils/services.py @@ -1,12 +1,15 @@ import argparse import traceback from flask import Flask, request, redirect, Response, make_response +from typing import Any + from bemani.protocol import EAmuseProtocol from bemani.backend import Dispatch, UnrecognizedPCBIDException from bemani.data import Config, Data from bemani.utils.config import ( load_config as base_load_config, + instantiate_cache as base_instantiate_cache, register_games as base_register_games, ) @@ -125,6 +128,11 @@ def load_config(filename: str) -> None: base_load_config(filename, config) +def instantiate_cache(app: Any) -> None: + global config + base_instantiate_cache(app, config) + + if __name__ == "__main__": parser = argparse.ArgumentParser( description="A backend services provider for eAmusement games" @@ -171,4 +179,5 @@ if __name__ == "__main__": app.wsgi_app = ProfilerMiddleware(app.wsgi_app, profile_dir=".") # type: ignore # Run the app + instantiate_cache(app) app.run(host="0.0.0.0", port=args.port, debug=True) diff --git a/bemani/wsgi/api.wsgi b/bemani/wsgi/api.wsgi index 823a8a8..94dafe2 100644 --- a/bemani/wsgi/api.wsgi +++ b/bemani/wsgi/api.wsgi @@ -1,6 +1,7 @@ -from bemani.utils.api import app, load_config +from bemani.utils.api import app, load_config, instantiate_cache # Assumes a production server yaml in the same directory as this WSGI # file. Also assumes that your uWSGI instance is configured with a # virtualenv that includes the installed version of this repo. load_config('server.yaml') +instantiate_cache(app) diff --git a/bemani/wsgi/frontend.wsgi b/bemani/wsgi/frontend.wsgi index c688329..e597379 100644 --- a/bemani/wsgi/frontend.wsgi +++ b/bemani/wsgi/frontend.wsgi @@ -1,4 +1,4 @@ -from bemani.utils.frontend import app, load_config, register_blueprints, register_games +from bemani.utils.frontend import app, load_config, instantiate_cache, register_blueprints, register_games # Assumes a production server yaml in the same directory as this WSGI # file. Also assumes that your uWSGI instance is configured with a @@ -6,3 +6,4 @@ from bemani.utils.frontend import app, load_config, register_blueprints, registe load_config('server.yaml') register_blueprints() register_games() +instantiate_cache(app) diff --git a/bemani/wsgi/proxy.wsgi b/bemani/wsgi/proxy.wsgi index 770a6df..1771aaa 100644 --- a/bemani/wsgi/proxy.wsgi +++ b/bemani/wsgi/proxy.wsgi @@ -1,6 +1,7 @@ -from bemani.utils.proxy import app, load_config +from bemani.utils.proxy import app, load_config, instantiate_cache # Assumes a proxy yaml in the same directory as this WSGI # file. Also assumes that your uWSGI instance is configured with a # virtualenv that includes the installed version of this repo. load_config('proxy.yaml') +instantiate_cache(app) diff --git a/bemani/wsgi/services.wsgi b/bemani/wsgi/services.wsgi index 6839037..8249986 100644 --- a/bemani/wsgi/services.wsgi +++ b/bemani/wsgi/services.wsgi @@ -1,7 +1,8 @@ -from bemani.utils.services import app, load_config, register_games +from bemani.utils.services import app, load_config, instantiate_cache, register_games # Assumes a production server yaml in the same directory as this WSGI # file. Also assumes that your uWSGI instance is configured with a # virtualenv that includes the installed version of this repo. load_config('server.yaml') register_games() +instantiate_cache(app) diff --git a/config/server.yaml b/config/server.yaml index ad03ff8..eb2b46c 100644 --- a/config/server.yaml +++ b/config/server.yaml @@ -94,8 +94,12 @@ secret_key: 'this_is_a_secret_please_change_me' name: 'e-AMUSEMENT Network' # Administrative contact for this network. email: 'nobody@nowhere.com' -# Cache DIR, should point somewhere other than /tmp for production instances. +# Cache DIR, should point somewhere other than /tmp for production instances that wish +# to use filesystem caching. For memcached, delete this value. cache_dir: '/tmp' +# memcached server, should point somewhere other than this bogus value for production +# instances that wish to use memcached backend. For filesystem caching, delete this value. +memcached_server: 1.2.3.4:5678 # Number of seconds to preserve event logs before deleting them. # Set to zero or delete to disable deleting logs. event_log_duration: 2592000