move to alembic

core/data/alembic/alembic.ini

@@ -1,25 +1,14 @@
 # A generic, single database configuration.
 
 [alembic]
-# path to migration scripts
+script_location=.
-script_location = .
 
 # template used to generate migration files
 # file_template = %%(rev)s_%%(slug)s
 
-# sys.path path, will be prepended to sys.path if present.
-# defaults to the current working directory.
-prepend_sys_path = .
-
-# timezone to use when rendering the date
-# within the migration file as well as the filename.
-# string value is passed to dateutil.tz.gettz()
-# leave blank for localtime
-# timezone =
-
 # max length of characters to apply to the
 # "slug" field
-# truncate_slug_length = 40
+#truncate_slug_length = 40
 
 # set to 'true' to run the environment during
 # the 'revision' command, regardless of autogenerate
@@ -31,28 +20,14 @@ prepend_sys_path = .
 # sourceless = false
 
 # version location specification; this defaults
-# to ./versions.  When using multiple version
+# to migrations//versions.  When using multiple version
 # directories, initial revisions must be specified with --version-path
-# version_locations = %(here)s/bar %(here)s/bat ./versions
+# version_locations = %(here)s/bar %(here)s/bat migrations//versions
 
 # the output encoding used when revision files
 # are written from script.py.mako
 # output_encoding = utf-8
 
-sqlalchemy.url = driver://user:pass@localhost/dbname
-
-
-[post_write_hooks]
-# post_write_hooks defines scripts or Python functions that are run
-# on newly generated revision scripts.  See the documentation for further
-# detail and examples
-
-# format using "black" - use the console_scripts runner, against the "black" entrypoint
-# hooks=black
-# black.type=console_scripts
-# black.entrypoint=black
-# black.options=-l 79
-
 # Logging configuration
 [loggers]
 keys = root,sqlalchemy,alembic

core/data/alembic/env.py

@@ -1,9 +1,9 @@
+from __future__ import with_statement
+from alembic import context
+from sqlalchemy import engine_from_config, pool
 from logging.config import fileConfig
 
-from sqlalchemy import engine_from_config
+from core.data.schema.base import metadata
-from sqlalchemy import pool
-
-from alembic import context
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
@@ -17,7 +17,7 @@ fileConfig(config.config_file_name)
 # for 'autogenerate' support
 # from myapp import mymodel
 # target_metadata = mymodel.Base.metadata
-target_metadata = None
+target_metadata = metadata
 
 # other values from the config, defined by the needs of env.py,
 # can be acquired:
@@ -37,13 +37,11 @@ def run_migrations_offline():
     script output.
 
     """
+    raise Exception('Not implemented or configured!')
+
     url = config.get_main_option("sqlalchemy.url")
     context.configure(
-        url=url,
+        url=url, target_metadata=target_metadata, literal_binds=True)
-        target_metadata=target_metadata,
-        literal_binds=True,
-        dialect_opts={"paramstyle": "named"},
-    )
 
     with context.begin_transaction():
         context.run_migrations()
@@ -56,21 +54,27 @@ def run_migrations_online():
     and associate a connection with the context.
 
     """
+    ini_section = config.get_section(config.config_ini_section)
+    overrides = context.get_x_argument(as_dictionary=True)
+    for override in overrides:
+        ini_section[override] = overrides[override]
+
     connectable = engine_from_config(
-        config.get_section(config.config_ini_section),
+        ini_section,
-        prefix="sqlalchemy.",
+        prefix='sqlalchemy.',
-        poolclass=pool.NullPool,
+        poolclass=pool.NullPool)
-    )
 
     with connectable.connect() as connection:
         context.configure(
-            connection=connection, target_metadata=target_metadata
+            connection=connection,
+            target_metadata=target_metadata,
+            compare_type=True,
+            compare_server_default=True,
         )
 
         with context.begin_transaction():
             context.run_migrations()
 
-
 if context.is_offline_mode():
     run_migrations_offline()
 else:

core/data/alembic/versions/835b862f9bf0_initial_migration.py

@@ -0,0 +1,24 @@
+"""Initial Migration
+
+Revision ID: 835b862f9bf0
+Revises: 
+Create Date: 2024-01-09 13:06:10.787432
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '835b862f9bf0'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    pass
+
+
+def downgrade():
+    pass

core/data/alembic/versions/d8950c7ce2fc_remove_old_db_mgmt_system.py

@@ -0,0 +1,29 @@
+"""Remove old db mgmt system
+
+Revision ID: d8950c7ce2fc
+Revises: 835b862f9bf0
+Create Date: 2024-01-09 13:43:51.381175
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'd8950c7ce2fc'
+down_revision = '835b862f9bf0'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.drop_table("schema_versions")
+
+
+def downgrade():
+    op.create_table(
+        "schema_versions",
+        sa.Column("game", sa.String(4), primary_key=True, nullable=False),
+        sa.Column("version", sa.Integer, nullable=False, server_default="1"),
+        mysql_charset="utf8mb4",
+    )

core/data/database.py

@@ -1,13 +1,13 @@
 import logging, coloredlogs
-from typing import Optional, Dict, List
+from typing import Optional
 from sqlalchemy.orm import scoped_session, sessionmaker
-from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy import create_engine
 from logging.handlers import TimedRotatingFileHandler
-import importlib, os
+import os
 import secrets, string
 import bcrypt
 from hashlib import sha256
+import alembic.config
 
 from core.config import CoreConfig
 from core.data.schema import *
@@ -15,7 +15,6 @@ from core.utils import Utils
 
 
 class Data:
-    current_schema_version = 6
     engine = None
     session = None
     user = None
@@ -77,281 +76,85 @@ class Data:
             )
             self.logger.handler_set = True  # type: ignore
 
+    def __alembic_cmd(self, command: str, *args: str) -> None:
+        old_dir = os.path.abspath(os.path.curdir)
+        base_dir = os.path.join(os.path.abspath(os.path.curdir), 'core', 'data', 'alembic')
+        alembicArgs = [
+            "-c",
+            os.path.join(base_dir, "alembic.ini"),
+            "-x",
+            f"script_location={base_dir}",
+            "-x",
+            f"sqlalchemy.url={self.__url}",
+            command,
+        ]
+        alembicArgs.extend(args)
+        os.chdir(base_dir)
+        alembic.config.main(argv=alembicArgs)
+        os.chdir(old_dir)
+
     def create_database(self):
         self.logger.info("Creating databases...")
-        try:
+        metadata.create_all(
-            metadata.create_all(self.__engine.connect())
+            self.engine,
-        except SQLAlchemyError as e:
+            checkfirst=True,
-            self.logger.error(f"Failed to create databases! {e}")
-            return
-
-        games = Utils.get_all_titles()
-        for game_dir, game_mod in games.items():
-            try:
-                if hasattr(game_mod, "database") and hasattr(
-                    game_mod, "current_schema_version"
-                ):
-                    game_mod.database(self.config)
-                metadata.create_all(self.__engine.connect())
-
-                self.base.touch_schema_ver(
-                    game_mod.current_schema_version, game_mod.game_codes[0]
-                )
-
-            except Exception as e:
-                self.logger.warning(
-                    f"Could not load database schema from {game_dir} - {e}"
-                )
-
-        self.logger.info(f"Setting base_schema_ver to {self.current_schema_version}")
-        self.base.set_schema_ver(self.current_schema_version)
-
-        self.logger.info(
-            f"Setting user auto_incrememnt to {self.config.database.user_table_autoincrement_start}"
-        )
-        self.user.reset_autoincrement(
-            self.config.database.user_table_autoincrement_start
         )
 
-    def recreate_database(self):
+        for _, mod in Utils.get_all_titles().items():
-        self.logger.info("Dropping all databases...")
+            if hasattr(mod, "database"):
-        self.base.execute("SET FOREIGN_KEY_CHECKS=0")
+                mod.database(self.config)
-        try:
+                metadata.create_all(
-            metadata.drop_all(self.__engine.connect())
+                    self.engine,
-        except SQLAlchemyError as e:
+                    checkfirst=True,
-            self.logger.error(f"Failed to drop databases! {e}")
+                )
-            return
 
-        for root, dirs, files in os.walk("./titles"):
+        # Stamp the end revision as if alembic had created it, so it can take off after this.
-            for dir in dirs:
+        self.__alembic_cmd(
-                if not dir.startswith("__"):
+            "stamp",
-                    try:
+            "head",
-                        mod = importlib.import_module(f"titles.{dir}")
+        )
 
-                        try:
+    def schema_upgrade(self, ver: str = None):
-                            if hasattr(mod, "database"):
+        self.__alembic_cmd(
-                                mod.database(self.config)
+            "upgrade",
-                            metadata.drop_all(self.__engine.connect())
+            "head",
+        )
 
-                        except Exception as e:
+    async def create_owner(self, email: Optional[str] = None, code: Optional[str] = "00000000000000000000") -> None:
-                            self.logger.warning(
-                                f"Could not load database schema from {dir} - {e}"
-                            )
-
-                    except ImportError as e:
-                        self.logger.warning(
-                            f"Failed to load database schema dir {dir} - {e}"
-                        )
-            break
-
-        self.base.execute("SET FOREIGN_KEY_CHECKS=1")
-
-        self.create_database()
-
-    def migrate_database(self, game: str, version: Optional[int], action: str) -> None:
-        old_ver = self.base.get_schema_ver(game)
-        sql = ""
-        if version is None:
-            if not game == "CORE":
-                titles = Utils.get_all_titles()
-
-                for folder, mod in titles.items():
-                    if not mod.game_codes[0] == game:
-                        continue
-
-                    if hasattr(mod, "current_schema_version"):
-                        version = mod.current_schema_version
-
-                    else:
-                        self.logger.warning(
-                            f"current_schema_version not found for {folder}"
-                        )
-
-            else:
-                version = self.current_schema_version
-
-        if version is None:
-            self.logger.warning(
-                f"Could not determine latest version for {game}, please specify --version"
-            )
-
-        if old_ver is None:
-            self.logger.error(
-                f"Schema for game {game} does not exist, did you run the creation script?"
-            )
-            return
-
-        if old_ver == version:
-            self.logger.info(
-                f"Schema for game {game} is already version {old_ver}, nothing to do"
-            )
-            return
-
-        if action == "upgrade":
-            for x in range(old_ver, version):
-                if not os.path.exists(
-                    f"core/data/schema/versions/{game.upper()}_{x + 1}_{action}.sql"
-                ):
-                    self.logger.error(
-                        f"Could not find {action} script {game.upper()}_{x + 1}_{action}.sql in core/data/schema/versions folder"
-                    )
-                    return
-
-                with open(
-                    f"core/data/schema/versions/{game.upper()}_{x + 1}_{action}.sql",
-                    "r",
-                    encoding="utf-8",
-                ) as f:
-                    sql = f.read()
-
-                result = self.base.execute(sql)
-                if result is None:
-                    self.logger.error("Error execuing sql script!")
-                    return None
-
-        else:
-            for x in range(old_ver, version, -1):
-                if not os.path.exists(
-                    f"core/data/schema/versions/{game.upper()}_{x - 1}_{action}.sql"
-                ):
-                    self.logger.error(
-                        f"Could not find {action} script {game.upper()}_{x - 1}_{action}.sql in core/data/schema/versions folder"
-                    )
-                    return
-
-                with open(
-                    f"core/data/schema/versions/{game.upper()}_{x - 1}_{action}.sql",
-                    "r",
-                    encoding="utf-8",
-                ) as f:
-                    sql = f.read()
-
-                result = self.base.execute(sql)
-                if result is None:
-                    self.logger.error("Error execuing sql script!")
-                    return None
-
-        result = self.base.set_schema_ver(version, game)
-        if result is None:
-            self.logger.error("Error setting version in schema_version table!")
-            return None
-
-        self.logger.info(f"Successfully migrated {game} to schema version {version}")
-
-    def create_owner(self, email: Optional[str] = None) -> None:
         pw = "".join(
             secrets.choice(string.ascii_letters + string.digits) for i in range(20)
         )
         hash = bcrypt.hashpw(pw.encode(), bcrypt.gensalt())
 
-        user_id = self.user.create_user(email=email, permission=255, password=hash)
+        user_id = await self.user.create_user("sysowner", email, hash.decode(), 255)
         if user_id is None:
             self.logger.error(f"Failed to create owner with email {email}")
             return
 
-        card_id = self.card.create_card(user_id, "00000000000000000000")
+        card_id = await self.card.create_card(user_id, code)
         if card_id is None:
             self.logger.error(f"Failed to create card for owner with id {user_id}")
             return
 
         self.logger.warning(
-            f"Successfully created owner with email {email}, access code 00000000000000000000, and password {pw} Make sure to change this password and assign a real card ASAP!"
+            f"Successfully created owner with email {email}, access code {code}, and password {pw} Make sure to change this password and assign a real card ASAP!"
         )
-
-    def migrate_card(self, old_ac: str, new_ac: str, should_force: bool) -> None:
-        if old_ac == new_ac:
-            self.logger.error("Both access codes are the same!")
-            return
-
-        new_card = self.card.get_card_by_access_code(new_ac)
-        if new_card is None:
-            self.card.update_access_code(old_ac, new_ac)
-            return
-
-        if not should_force:
-            self.logger.warning(
-                f"Card already exists for access code {new_ac} (id {new_card['id']}). If you wish to continue, rerun with the '--force' flag."
-                f" All exiting data on the target card {new_ac} will be perminently erased and replaced with data from card {old_ac}."
-            )
-            return
-
-        self.logger.info(
-            f"All exiting data on the target card {new_ac} will be perminently erased and replaced with data from card {old_ac}."
-        )
-        self.card.delete_card(new_card["id"])
-        self.card.update_access_code(old_ac, new_ac)
-
-        hanging_user = self.user.get_user(new_card["user"])
-        if hanging_user["password"] is None:
-            self.logger.info(f"Delete hanging user {hanging_user['id']}")
-            self.user.delete_user(hanging_user["id"])
-
-    def delete_hanging_users(self) -> None:
-        """
-        Finds and deletes users that have not registered for the webui that have no cards assocated with them.
-        """
-        unreg_users = self.user.get_unregistered_users()
-        if unreg_users is None:
-            self.logger.error("Error occoured finding unregistered users")
-
-        for user in unreg_users:
-            cards = self.card.get_user_cards(user["id"])
-            if cards is None:
-                self.logger.error(f"Error getting cards for user {user['id']}")
-                continue
-
-            if not cards:
-                self.logger.info(f"Delete hanging user {user['id']}")
-                self.user.delete_user(user["id"])
-
-    def autoupgrade(self) -> None:
-        all_game_versions = self.base.get_all_schema_vers()
-        if all_game_versions is None:
-            self.logger.warning("Failed to get schema versions")
-            return
-
-        all_games = Utils.get_all_titles()
-        all_games_list: Dict[str, int] = {}
-        for _, mod in all_games.items():
-            if hasattr(mod, "current_schema_version"):
-                all_games_list[mod.game_codes[0]] = mod.current_schema_version
-
-        for x in all_game_versions:
-            failed = False
-            game = x["game"].upper()
-            update_ver = int(x["version"])
-            latest_ver = all_games_list.get(game, 1)
-            if game == "CORE":
-                latest_ver = self.current_schema_version
-
-            if update_ver == latest_ver:
-                self.logger.info(f"{game} is already latest version")
-                continue
-
-            for y in range(update_ver + 1, latest_ver + 1):
-                if os.path.exists(f"core/data/schema/versions/{game}_{y}_upgrade.sql"):
-                    with open(
-                        f"core/data/schema/versions/{game}_{y}_upgrade.sql",
-                        "r",
-                        encoding="utf-8",
-                    ) as f:
-                        sql = f.read()
-
-                    result = self.base.execute(sql)
-                    if result is None:
-                        self.logger.error(
-                            f"Error execuing sql script for game {game} v{y}!"
-                        )
-                        failed = True
-                        break
-                else:
-                    self.logger.warning(f"Could not find script {game}_{y}_upgrade.sql")
-                    failed = True
-
-            if not failed:
-                self.base.set_schema_ver(latest_ver, game)
     
-    def show_versions(self) -> None:
+    async def migrate(self) -> None:
-        all_game_versions = self.base.get_all_schema_vers()
+        exist = await self.base.execute("SELECT * FROM alembic_version")
-        for ver in all_game_versions:
+        if exist is not None:
-            self.logger.info(f"{ver['game']} -> v{ver['version']}")
+            self.logger.warn("No need to migrate as you have already migrated to alembic. If you are trying to upgrade the schema, use `upgrade` instead!")
+            return
+        
+        self.logger.info("Stamp with initial revision")
+        self.__alembic_cmd(
+            "stamp",
+            "835b862f9bf0",
+        )
+
+        self.logger.info("Upgrade")
+        self.__alembic_cmd(
+            "upgrade",
+            "head",
+        )
+

core/data/schema/base.py

@@ -43,11 +43,11 @@ class BaseData:
         self.conn = conn
         self.logger = logging.getLogger("database")
 
-    def execute(self, sql: str, opts: Dict[str, Any] = {}) -> Optional[CursorResult]:
+    async def execute(self, sql: str, opts: Dict[str, Any] = {}) -> Optional[CursorResult]:
         res = None
 
         try:
-            self.logger.info(f"SQL Execute: {''.join(str(sql).splitlines())}")
+            self.logger.debug(f"SQL Execute: {''.join(str(sql).splitlines())}")
             res = self.conn.execute(text(sql), opts)
 
         except SQLAlchemyError as e:
@@ -82,52 +82,7 @@ class BaseData:
         """
         return randrange(10000, 9999999)
 
-    def get_all_schema_vers(self) -> Optional[List[Row]]:
+    async def log_event(
-        sql = select(schema_ver)
-
-        result = self.execute(sql)
-        if result is None:
-            return None
-        return result.fetchall()
-
-    def get_schema_ver(self, game: str) -> Optional[int]:
-        sql = select(schema_ver).where(schema_ver.c.game == game)
-
-        result = self.execute(sql)
-        if result is None:
-            return None
-
-        row = result.fetchone()
-        if row is None:
-            return None
-
-        return row["version"]
-    
-    def touch_schema_ver(self, ver: int, game: str = "CORE") -> Optional[int]:
-        sql = insert(schema_ver).values(game=game, version=ver)
-        conflict = sql.on_duplicate_key_update(version=schema_ver.c.version)
-
-        result = self.execute(conflict)
-        if result is None:
-            self.logger.error(
-                f"Failed to update schema version for game {game} (v{ver})"
-            )
-            return None
-        return result.lastrowid
-
-    def set_schema_ver(self, ver: int, game: str = "CORE") -> Optional[int]:
-        sql = insert(schema_ver).values(game=game, version=ver)
-        conflict = sql.on_duplicate_key_update(version=ver)
-
-        result = self.execute(conflict)
-        if result is None:
-            self.logger.error(
-                f"Failed to update schema version for game {game} (v{ver})"
-            )
-            return None
-        return result.lastrowid
-
-    def log_event(
         self, system: str, type: str, severity: int, message: str, details: Dict = {}
     ) -> Optional[int]:
         sql = event_log.insert().values(
@@ -147,7 +102,7 @@ class BaseData:
 
         return result.lastrowid
 
-    def get_event_log(self, entries: int = 100) -> Optional[List[Dict]]:
+    async def get_event_log(self, entries: int = 100) -> Optional[List[Dict]]:
         sql = event_log.select().limit(entries).all()
         result = self.execute(sql)
 

dbutils.py

@@ -5,7 +5,8 @@ from os import mkdir, path, access, W_OK
 import yaml
 import asyncio
 
-from core import Data, CoreConfig
+from core.data import Data
+from core.config import CoreConfig
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Database utilities")
@@ -49,6 +50,11 @@ if __name__ == "__main__":
     elif args.action == "create-owner":
         loop = asyncio.get_event_loop()
         loop.run_until_complete(data.create_owner(args.email, args.access_code))
+        data.schema_upgrade(args.version)
+
+    elif args.action == "migrate":
+        loop = asyncio.get_event_loop()
+        loop.run_until_complete(data.migrate())
 
     else:
         logging.getLogger("database").info(f"Unknown action {args.action}")

example_config/core.yaml

@@ -19,6 +19,7 @@ title:
   reboot_start_time: "04:00"
   reboot_end_time: "05:00"
   ssl_key: "cert/title.key"
+
 database:
   host: "localhost"
   username: "aime"
@@ -27,7 +28,7 @@ database:
   port: 3306
   protocol: "mysql"
   sha2_password: False
-  loglevel: "warn"
+  loglevel: "info"
   enable_memcached: True
   memcached_host: "localhost"
 

titles/chuni/__init__.py

@@ -7,4 +7,3 @@ index = ChuniServlet
 database = ChuniData
 reader = ChuniReader
 game_codes = [ChuniConstants.GAME_CODE, ChuniConstants.GAME_CODE_NEW, ChuniConstants.GAME_CODE_INT]
-current_schema_version = 5

titles/cm/__init__.py

@@ -6,7 +6,4 @@ from titles.cm.database import CardMakerData
 index = CardMakerServlet
 reader = CardMakerReader
 database = CardMakerData
-
 game_codes = [CardMakerConstants.GAME_CODE]
-
-current_schema_version = 1

titles/cxb/__init__.py

@@ -7,4 +7,3 @@ index = CxbServlet
 database = CxbData
 reader = CxbReader
 game_codes = [CxbConstants.GAME_CODE]
-current_schema_version = 1

titles/diva/__init__.py

@@ -7,4 +7,3 @@ index = DivaServlet
 database = DivaData
 reader = DivaReader
 game_codes = [DivaConstants.GAME_CODE]
-current_schema_version = 6

titles/idac/__init__.py

@@ -9,4 +9,3 @@ database = IDACData
 reader = IDACReader
 frontend = IDACFrontend
 game_codes = [IDACConstants.GAME_CODE]
-current_schema_version = 1

titles/idz/__init__.py

@@ -5,4 +5,3 @@ from titles.idz.database import IDZData
 index = IDZServlet
 database = IDZData
 game_codes = [IDZConstants.GAME_CODE]
-current_schema_version = 1

titles/mai2/__init__.py

@@ -16,4 +16,3 @@ game_codes = [
     Mai2Constants.GAME_CODE_GREEN,
     Mai2Constants.GAME_CODE,
 ]
-current_schema_version = 8

titles/ongeki/__init__.py

@@ -9,4 +9,3 @@ database = OngekiData
 reader = OngekiReader
 frontend = OngekiFrontend
 game_codes = [OngekiConstants.GAME_CODE]
-current_schema_version = 6

titles/pokken/__init__.py

@@ -6,5 +6,4 @@ from .frontend import PokkenFrontend
 index = PokkenServlet
 database = PokkenData
 game_codes = [PokkenConstants.GAME_CODE]
-current_schema_version = 1
 frontend = PokkenFrontend

titles/sao/__init__.py

@@ -7,4 +7,3 @@ index = SaoServlet
 database = SaoData
 reader = SaoReader
 game_codes = [SaoConstants.GAME_CODE]
-current_schema_version = 1

titles/wacca/__init__.py

@@ -9,4 +9,3 @@ database = WaccaData
 reader = WaccaReader
 frontend = WaccaFrontend
 game_codes = [WaccaConstants.GAME_CODE]
-current_schema_version = 5