1
0
mirror of synced 2024-11-23 22:10:59 +01:00

Promote frontend-only cache to a system-wide context, tweak application entrypoints to work with new bemani.common.cache

This commit is contained in:
Jennifer Taylor 2023-08-13 18:56:43 +00:00
parent 8d08fb1ff0
commit 2491eb0767
22 changed files with 123 additions and 90 deletions

View File

@ -15,6 +15,7 @@ from bemani.common.aes import AESCipher
from bemani.common.time import Time from bemani.common.time import Time
from bemani.common.parallel import Parallel from bemani.common.parallel import Parallel
from bemani.common.pe import PEFile, InvalidOffsetException from bemani.common.pe import PEFile, InvalidOffsetException
from bemani.common.cache import cache
__all__ = [ __all__ = [
@ -38,4 +39,5 @@ __all__ = [
"intish", "intish",
"PEFile", "PEFile",
"InvalidOffsetException", "InvalidOffsetException",
"cache",
] ]

17
bemani/common/cache.py Normal file
View File

@ -0,0 +1,17 @@
from flask import Flask
from flask_caching import Cache
# This somewhat breaks convention of trying to keep flask stuff in only the application
# routing layer, but flask-caching itself is a useful wrapper that supports a ton of
# backends like in-memory, filesystem, redis, memcached, etc. So, we centralize the
# object's ownership here and call into this to initialize it during application setup
# so that anywhere in the codebase can assume an initialized cache object for decorators.
app = Flask(__name__)
cache = Cache(
app,
# We should overwrite this in any reasonable entrypoint to the system, but for simple
# utilities that don't want to set up the entire infrastructure, provide a sane default.
config={"CACHE_TYPE": "SimpleCache"},
)

View File

@ -237,8 +237,18 @@ class Config(dict):
return str(self.get("email", "nobody@nowhere.com")) return str(self.get("email", "nobody@nowhere.com"))
@property @property
def cache_dir(self) -> str: def cache_dir(self) -> Optional[str]:
return os.path.abspath(str(self.get("cache_dir", "/tmp"))) cache_dir = self.get("cache_dir")
if cache_dir is None:
return None
return os.path.abspath(str(cache_dir))
@property
def memcached_server(self) -> Optional[str]:
server = self.get("memcached_server")
if server is None:
return None
return str(server)
@property @property
def theme(self) -> str: def theme(self) -> str:

View File

@ -16,10 +16,9 @@ from flask import (
got_request_exception, got_request_exception,
jsonify as flask_jsonify, jsonify as flask_jsonify,
) )
from flask_caching import Cache
from functools import wraps from functools import wraps
from bemani.common import AESCipher, GameConstants from bemani.common import AESCipher, GameConstants, cache
from bemani.data import Config, Data from bemani.data import Config, Data
from bemani.frontend.types import g from bemani.frontend.types import g
from bemani.frontend.templates import templates_location from bemani.frontend.templates import templates_location
@ -40,18 +39,14 @@ FRONTEND_CACHE_BUST: str = "site.1.3.react.16.14"
@app.before_request @app.before_request
def before_request() -> None: def before_request() -> None:
global config global config
g.cache = Cache(
app, g.cache = cache
config={ g.config = config
"CACHE_TYPE": "filesystem",
"CACHE_DIR": config.cache_dir,
},
)
if request.endpoint in ["jsx", "static"]: if request.endpoint in ["jsx", "static"]:
# This is just serving cached compiled frontends, skip loading from DB # This is just serving cached compiled frontends, skip loading from DB
return return
g.config = config
g.data = Data(config) g.data = Data(config)
g.sessionID = None g.sessionID = None
g.userID = None g.userID = None

View File

@ -1,19 +1,10 @@
from flask_caching import Cache from bemani.common import cache
from bemani.data import Config, Data from bemani.data import Config, Data
from bemani.frontend.app import app
from bemani.frontend.ddr.ddr import DDRFrontend from bemani.frontend.ddr.ddr import DDRFrontend
class DDRCache: class DDRCache:
@classmethod @classmethod
def preload(cls, data: Data, config: Config) -> None: def preload(cls, data: Data, config: Config) -> None:
cache = Cache(
app,
config={
"CACHE_TYPE": "filesystem",
"CACHE_DIR": config.cache_dir,
},
)
frontend = DDRFrontend(data, config, cache) frontend = DDRFrontend(data, config, cache)
frontend.get_all_songs(force_db_load=True) frontend.get_all_songs(force_db_load=True)

View File

@ -1,19 +1,10 @@
from flask_caching import Cache from bemani.common import cache
from bemani.data import Config, Data from bemani.data import Config, Data
from bemani.frontend.app import app
from bemani.frontend.iidx.iidx import IIDXFrontend from bemani.frontend.iidx.iidx import IIDXFrontend
class IIDXCache: class IIDXCache:
@classmethod @classmethod
def preload(cls, data: Data, config: Config) -> None: def preload(cls, data: Data, config: Config) -> None:
cache = Cache(
app,
config={
"CACHE_TYPE": "filesystem",
"CACHE_DIR": config.cache_dir,
},
)
frontend = IIDXFrontend(data, config, cache) frontend = IIDXFrontend(data, config, cache)
frontend.get_all_songs(force_db_load=True) frontend.get_all_songs(force_db_load=True)

View File

@ -1,19 +1,10 @@
from flask_caching import Cache from bemani.common import cache
from bemani.data import Config, Data from bemani.data import Config, Data
from bemani.frontend.app import app
from bemani.frontend.jubeat.jubeat import JubeatFrontend from bemani.frontend.jubeat.jubeat import JubeatFrontend
class JubeatCache: class JubeatCache:
@classmethod @classmethod
def preload(cls, data: Data, config: Config) -> None: def preload(cls, data: Data, config: Config) -> None:
cache = Cache(
app,
config={
"CACHE_TYPE": "filesystem",
"CACHE_DIR": config.cache_dir,
},
)
frontend = JubeatFrontend(data, config, cache) frontend = JubeatFrontend(data, config, cache)
frontend.get_all_songs(force_db_load=True) frontend.get_all_songs(force_db_load=True)

View File

@ -1,19 +1,10 @@
from flask_caching import Cache from bemani.common import cache
from bemani.data import Config, Data from bemani.data import Config, Data
from bemani.frontend.app import app
from bemani.frontend.museca.museca import MusecaFrontend from bemani.frontend.museca.museca import MusecaFrontend
class MusecaCache: class MusecaCache:
@classmethod @classmethod
def preload(cls, data: Data, config: Config) -> None: def preload(cls, data: Data, config: Config) -> None:
cache = Cache(
app,
config={
"CACHE_TYPE": "filesystem",
"CACHE_DIR": config.cache_dir,
},
)
frontend = MusecaFrontend(data, config, cache) frontend = MusecaFrontend(data, config, cache)
frontend.get_all_songs(force_db_load=True) frontend.get_all_songs(force_db_load=True)

View File

@ -1,19 +1,10 @@
from flask_caching import Cache from bemani.common import cache
from bemani.data import Config, Data from bemani.data import Config, Data
from bemani.frontend.app import app
from bemani.frontend.popn.popn import PopnMusicFrontend from bemani.frontend.popn.popn import PopnMusicFrontend
class PopnMusicCache: class PopnMusicCache:
@classmethod @classmethod
def preload(cls, data: Data, config: Config) -> None: def preload(cls, data: Data, config: Config) -> None:
cache = Cache(
app,
config={
"CACHE_TYPE": "filesystem",
"CACHE_DIR": config.cache_dir,
},
)
frontend = PopnMusicFrontend(data, config, cache) frontend = PopnMusicFrontend(data, config, cache)
frontend.get_all_songs(force_db_load=True) frontend.get_all_songs(force_db_load=True)

View File

@ -1,19 +1,10 @@
from flask_caching import Cache from bemani.common import cache
from bemani.data import Config, Data from bemani.data import Config, Data
from bemani.frontend.app import app
from bemani.frontend.reflec.reflec import ReflecBeatFrontend from bemani.frontend.reflec.reflec import ReflecBeatFrontend
class ReflecBeatCache: class ReflecBeatCache:
@classmethod @classmethod
def preload(cls, data: Data, config: Config) -> None: def preload(cls, data: Data, config: Config) -> None:
cache = Cache(
app,
config={
"CACHE_TYPE": "filesystem",
"CACHE_DIR": config.cache_dir,
},
)
frontend = ReflecBeatFrontend(data, config, cache) frontend = ReflecBeatFrontend(data, config, cache)
frontend.get_all_songs(force_db_load=True) frontend.get_all_songs(force_db_load=True)

View File

@ -1,19 +1,10 @@
from flask_caching import Cache from bemani.common import cache
from bemani.data import Config, Data from bemani.data import Config, Data
from bemani.frontend.app import app
from bemani.frontend.sdvx.sdvx import SoundVoltexFrontend from bemani.frontend.sdvx.sdvx import SoundVoltexFrontend
class SoundVoltexCache: class SoundVoltexCache:
@classmethod @classmethod
def preload(cls, data: Data, config: Config) -> None: def preload(cls, data: Data, config: Config) -> None:
cache = Cache(
app,
config={
"CACHE_TYPE": "filesystem",
"CACHE_DIR": config.cache_dir,
},
)
frontend = SoundVoltexFrontend(data, config, cache) frontend = SoundVoltexFrontend(data, config, cache)
frontend.get_all_songs(force_db_load=True) frontend.get_all_songs(force_db_load=True)

View File

@ -37,6 +37,10 @@
<dt>Cache Directory</dt> <dt>Cache Directory</dt>
<dd>{{ config.cache_dir }}</dd> <dd>{{ config.cache_dir }}</dd>
</dl> </dl>
<dl>
<dt>memcached Server</dt>
<dd>{{ config.memcached_server }}</dd>
</dl>
</div> </div>
<div class="section"> <div class="section">
<h3>Frontend Settings</h3> <h3>Frontend Settings</h3>

View File

@ -1,7 +1,11 @@
import argparse import argparse
from typing import Any
from bemani.api import app, config # noqa: F401 from bemani.api import app, config # noqa: F401
from bemani.utils.config import load_config as base_load_config from bemani.utils.config import (
load_config as base_load_config,
instantiate_cache as base_instantiate_cache,
)
def load_config(filename: str) -> None: def load_config(filename: str) -> None:
@ -9,6 +13,11 @@ def load_config(filename: str) -> None:
base_load_config(filename, config) base_load_config(filename, config)
def instantiate_cache(app: Any) -> None:
global config
base_instantiate_cache(app, config)
def main() -> None: def main() -> None:
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="An API services provider for eAmusement games, conforming to BEMAPI specs." description="An API services provider for eAmusement games, conforming to BEMAPI specs."
@ -48,6 +57,7 @@ def main() -> None:
app.wsgi_app = ProfilerMiddleware(app.wsgi_app, profile_dir=".") # type: ignore app.wsgi_app = ProfilerMiddleware(app.wsgi_app, profile_dir=".") # type: ignore
# Run the app # Run the app
instantiate_cache(app)
app.run(host="0.0.0.0", port=args.port, debug=True) app.run(host="0.0.0.0", port=args.port, debug=True)

View File

@ -1,4 +1,5 @@
import yaml import yaml
from flask import Flask
from typing import Set from typing import Set
from bemani.backend.iidx import IIDXFactory from bemani.backend.iidx import IIDXFactory
@ -10,7 +11,7 @@ from bemani.backend.sdvx import SoundVoltexFactory
from bemani.backend.reflec import ReflecBeatFactory from bemani.backend.reflec import ReflecBeatFactory
from bemani.backend.museca import MusecaFactory from bemani.backend.museca import MusecaFactory
from bemani.backend.mga import MetalGearArcadeFactory from bemani.backend.mga import MetalGearArcadeFactory
from bemani.common import GameConstants from bemani.common import GameConstants, cache
from bemani.data import Config, Data from bemani.data import Config, Data
@ -26,6 +27,34 @@ def load_config(filename: str, config: Config) -> None:
config["support"] = supported_series config["support"] = supported_series
def instantiate_cache(app: Flask, config: Config) -> None:
# This could easily be extended to add support for any other backend that flask-caching
# supports but right now the only demand is for in-memory, filesystem and memcached.
if config.memcached_server is not None:
cache.init_app(
app,
config={
"CACHE_TYPE": "MemcachedCache",
"CACHE_MEMCACHED_SERVERS": [config.memcached_server],
},
)
elif config.cache_dir is not None:
cache.init_app(
app,
config={
"CACHE_TYPE": "FileSystemCache",
"CACHE_DIR": config.cache_dir,
},
)
else:
cache.init_app(
app,
config={
"CACHE_TYPE": "SimpleCache",
},
)
def register_games(config: Config) -> None: def register_games(config: Config) -> None:
if GameConstants.POPN_MUSIC in config.support: if GameConstants.POPN_MUSIC in config.support:
PopnMusicFactory.register_all() PopnMusicFactory.register_all()

View File

@ -1,4 +1,5 @@
import argparse import argparse
from typing import Any
from bemani.common import GameConstants from bemani.common import GameConstants
from bemani.frontend import app, config # noqa: F401 from bemani.frontend import app, config # noqa: F401
@ -17,6 +18,7 @@ from bemani.frontend.reflec import reflec_pages
from bemani.frontend.museca import museca_pages from bemani.frontend.museca import museca_pages
from bemani.utils.config import ( from bemani.utils.config import (
load_config as base_load_config, load_config as base_load_config,
instantiate_cache as base_instantiate_cache,
register_games as base_register_games, register_games as base_register_games,
) )
@ -60,6 +62,11 @@ def load_config(filename: str) -> None:
app.secret_key = config.secret_key app.secret_key = config.secret_key
def instantiate_cache(app: Any) -> None:
global config
base_instantiate_cache(app, config)
def main() -> None: def main() -> None:
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="A front end services provider for eAmusement games." description="A front end services provider for eAmusement games."
@ -105,6 +112,7 @@ def main() -> None:
app.wsgi_app = ProfilerMiddleware(app.wsgi_app, profile_dir=".") # type: ignore app.wsgi_app = ProfilerMiddleware(app.wsgi_app, profile_dir=".") # type: ignore
# Run the app # Run the app
instantiate_cache(app)
app.run(host="0.0.0.0", port=args.port, debug=True) app.run(host="0.0.0.0", port=args.port, debug=True)

View File

@ -1,4 +1,5 @@
import argparse import argparse
from flask import Flask
from typing import Any, List from typing import Any, List
from bemani.backend.popn import PopnMusicFactory from bemani.backend.popn import PopnMusicFactory
@ -21,7 +22,7 @@ from bemani.frontend.reflec import ReflecBeatCache
from bemani.frontend.museca import MusecaCache from bemani.frontend.museca import MusecaCache
from bemani.common import GameConstants, Time from bemani.common import GameConstants, Time
from bemani.data import Config, Data from bemani.data import Config, Data
from bemani.utils.config import load_config from bemani.utils.config import load_config, instantiate_cache
def run_scheduled_work(config: Config) -> None: def run_scheduled_work(config: Config) -> None:
@ -99,5 +100,8 @@ if __name__ == "__main__":
if args.read_only: if args.read_only:
config["database"]["read_only"] = True config["database"]["read_only"] = True
# Set up production cache, with a dummy app context because flask-caching needs it.
instantiate_cache(Flask(__name__), config)
# Run out of band work # Run out of band work
run_scheduled_work(config) run_scheduled_work(config)

View File

@ -1,12 +1,15 @@
import argparse import argparse
import traceback import traceback
from flask import Flask, request, redirect, Response, make_response from flask import Flask, request, redirect, Response, make_response
from typing import Any
from bemani.protocol import EAmuseProtocol from bemani.protocol import EAmuseProtocol
from bemani.backend import Dispatch, UnrecognizedPCBIDException from bemani.backend import Dispatch, UnrecognizedPCBIDException
from bemani.data import Config, Data from bemani.data import Config, Data
from bemani.utils.config import ( from bemani.utils.config import (
load_config as base_load_config, load_config as base_load_config,
instantiate_cache as base_instantiate_cache,
register_games as base_register_games, register_games as base_register_games,
) )
@ -125,6 +128,11 @@ def load_config(filename: str) -> None:
base_load_config(filename, config) base_load_config(filename, config)
def instantiate_cache(app: Any) -> None:
global config
base_instantiate_cache(app, config)
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="A backend services provider for eAmusement games" description="A backend services provider for eAmusement games"
@ -171,4 +179,5 @@ if __name__ == "__main__":
app.wsgi_app = ProfilerMiddleware(app.wsgi_app, profile_dir=".") # type: ignore app.wsgi_app = ProfilerMiddleware(app.wsgi_app, profile_dir=".") # type: ignore
# Run the app # Run the app
instantiate_cache(app)
app.run(host="0.0.0.0", port=args.port, debug=True) app.run(host="0.0.0.0", port=args.port, debug=True)

View File

@ -1,6 +1,7 @@
from bemani.utils.api import app, load_config from bemani.utils.api import app, load_config, instantiate_cache
# Assumes a production server yaml in the same directory as this WSGI # Assumes a production server yaml in the same directory as this WSGI
# file. Also assumes that your uWSGI instance is configured with a # file. Also assumes that your uWSGI instance is configured with a
# virtualenv that includes the installed version of this repo. # virtualenv that includes the installed version of this repo.
load_config('server.yaml') load_config('server.yaml')
instantiate_cache(app)

View File

@ -1,4 +1,4 @@
from bemani.utils.frontend import app, load_config, register_blueprints, register_games from bemani.utils.frontend import app, load_config, instantiate_cache, register_blueprints, register_games
# Assumes a production server yaml in the same directory as this WSGI # Assumes a production server yaml in the same directory as this WSGI
# file. Also assumes that your uWSGI instance is configured with a # file. Also assumes that your uWSGI instance is configured with a
@ -6,3 +6,4 @@ from bemani.utils.frontend import app, load_config, register_blueprints, registe
load_config('server.yaml') load_config('server.yaml')
register_blueprints() register_blueprints()
register_games() register_games()
instantiate_cache(app)

View File

@ -1,6 +1,7 @@
from bemani.utils.proxy import app, load_config from bemani.utils.proxy import app, load_config, instantiate_cache
# Assumes a proxy yaml in the same directory as this WSGI # Assumes a proxy yaml in the same directory as this WSGI
# file. Also assumes that your uWSGI instance is configured with a # file. Also assumes that your uWSGI instance is configured with a
# virtualenv that includes the installed version of this repo. # virtualenv that includes the installed version of this repo.
load_config('proxy.yaml') load_config('proxy.yaml')
instantiate_cache(app)

View File

@ -1,7 +1,8 @@
from bemani.utils.services import app, load_config, register_games from bemani.utils.services import app, load_config, instantiate_cache, register_games
# Assumes a production server yaml in the same directory as this WSGI # Assumes a production server yaml in the same directory as this WSGI
# file. Also assumes that your uWSGI instance is configured with a # file. Also assumes that your uWSGI instance is configured with a
# virtualenv that includes the installed version of this repo. # virtualenv that includes the installed version of this repo.
load_config('server.yaml') load_config('server.yaml')
register_games() register_games()
instantiate_cache(app)

View File

@ -94,8 +94,12 @@ secret_key: 'this_is_a_secret_please_change_me'
name: 'e-AMUSEMENT Network' name: 'e-AMUSEMENT Network'
# Administrative contact for this network. # Administrative contact for this network.
email: 'nobody@nowhere.com' email: 'nobody@nowhere.com'
# Cache DIR, should point somewhere other than /tmp for production instances. # Cache DIR, should point somewhere other than /tmp for production instances that wish
# to use filesystem caching. For memcached, delete this value.
cache_dir: '/tmp' cache_dir: '/tmp'
# memcached server, should point somewhere other than this bogus value for production
# instances that wish to use memcached backend. For filesystem caching, delete this value.
memcached_server: 1.2.3.4:5678
# Number of seconds to preserve event logs before deleting them. # Number of seconds to preserve event logs before deleting them.
# Set to zero or delete to disable deleting logs. # Set to zero or delete to disable deleting logs.
event_log_duration: 2592000 event_log_duration: 2592000