From 0cef797a8a74c6a895fbfaab8035e0bd57a6b65c Mon Sep 17 00:00:00 2001 From: Kevin Trocolli Date: Sun, 6 Oct 2024 03:47:10 -0400 Subject: [PATCH] mai2: rework photo uploads, relates to #67 --- .../versions/d8cd1fa04c2a_mai2_add_photos.py | 38 +++++ titles/mai2/base.py | 61 ++++---- titles/mai2/frontend.py | 130 +++++++++++++++++- titles/mai2/schema/profile.py | 54 +++++++- titles/mai2/templates/mai2_header.jinja | 3 + titles/mai2/templates/mai2_photos.jinja | 28 ++++ 6 files changed, 272 insertions(+), 42 deletions(-) create mode 100644 core/data/alembic/versions/d8cd1fa04c2a_mai2_add_photos.py create mode 100644 titles/mai2/templates/mai2_photos.jinja diff --git a/core/data/alembic/versions/d8cd1fa04c2a_mai2_add_photos.py b/core/data/alembic/versions/d8cd1fa04c2a_mai2_add_photos.py new file mode 100644 index 0000000..312a127 --- /dev/null +++ b/core/data/alembic/versions/d8cd1fa04c2a_mai2_add_photos.py @@ -0,0 +1,38 @@ +"""mai2_add_photos + +Revision ID: d8cd1fa04c2a +Revises: 54a84103b84e +Create Date: 2024-10-06 03:09:15.959817 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +# revision identifiers, used by Alembic. +revision = 'd8cd1fa04c2a' +down_revision = '54a84103b84e' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('mai2_user_photo', + sa.Column('id', sa.VARCHAR(length=36), nullable=False), + sa.Column('user', sa.Integer(), nullable=False), + sa.Column('playlog_num', sa.INTEGER(), nullable=False), + sa.Column('track_num', sa.INTEGER(), nullable=False), + sa.Column('when_upload', sa.TIMESTAMP(), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['user'], ['aime_user.id'], onupdate='cascade', ondelete='cascade'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('user', 'playlog_num', 'track_num', name='mai2_user_photo_uk'), + mysql_charset='utf8mb4' + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('mai2_user_photo') + # ### end Alembic commands ### diff --git a/titles/mai2/base.py b/titles/mai2/base.py index b041028..cb74206 100644 --- a/titles/mai2/base.py +++ b/titles/mai2/base.py @@ -2,7 +2,7 @@ from datetime import datetime, timedelta from typing import Any, Dict, List import logging from base64 import b64decode -from os import path, stat, remove +from os import path, stat, remove, mkdir, access, W_OK from PIL import ImageFile from random import randint @@ -866,46 +866,33 @@ class Mai2Base: self.logger.warning(f"Incorrect data size after decoding (Expected 10240, got {len(photo_chunk)})") return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'} - out_name = f"{self.game_config.uploads.photos_dir}/{user_id}_{playlog_id}_{track_num}" + photo_data = await self.data.profile.get_user_photo_by_user_playlog_track(user_id, playlog_id, track_num) + + if not photo_data: + photo_id = await self.data.profile.put_user_photo(user_id, playlog_id, track_num) + else: + photo_id = photo_data['id'] - if not path.exists(f"{out_name}.bin") and div_num != 0: - self.logger.warning(f"Out of order photo upload (div_num {div_num})") - return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'} - - if path.exists(f"{out_name}.bin") and div_num == 0: - self.logger.warning(f"Duplicate file upload") + out_folder = f"{self.game_config.uploads.photos_dir}/{photo_id}" + out_file = f"{out_folder}/{div_num}_{div_len - 1}.bin" + + if not path.exists(out_folder): + mkdir(out_folder) + + if not access(out_folder, W_OK): + self.logger.error(f"Cannot access {out_folder}") return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'} - elif path.exists(f"{out_name}.bin"): - fstats = stat(f"{out_name}.bin") - if fstats.st_size != 10240 * div_num: - self.logger.warning(f"Out of order photo upload (trying to upload div {div_num}, expected div {fstats.st_size / 10240} for file sized {fstats.st_size} bytes)") + if path.exists(out_file): + self.logger.warning(f"Photo chunk {out_file} already exists, skipping") + + else: + with open(out_file, "wb") as f: + written = f.write(photo_chunk) + + if written != len(photo_chunk): + self.logger.error(f"Writing {out_file} failed! Wrote {written} bytes, expected {photo_chunk} bytes") return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'} - - try: - with open(f"{out_name}.bin", "ab") as f: - f.write(photo_chunk) - - except Exception: - self.logger.error(f"Failed writing to {out_name}.bin") - return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'} - - if div_num + 1 == div_len and path.exists(f"{out_name}.bin"): - try: - p = ImageFile.Parser() - with open(f"{out_name}.bin", "rb") as f: - p.feed(f.read()) - - im = p.close() - im.save(f"{out_name}.jpeg") - except Exception: - self.logger.error(f"File {out_name}.bin failed image validation") - - try: - remove(f"{out_name}.bin") - - except Exception: - self.logger.error(f"Failed to delete {out_name}.bin, please remove it manually") return {'returnCode': ret_code, 'apiName': 'UploadUserPhotoApi'} diff --git a/titles/mai2/frontend.py b/titles/mai2/frontend.py index 976e2c4..f1f961a 100644 --- a/titles/mai2/frontend.py +++ b/titles/mai2/frontend.py @@ -1,11 +1,14 @@ from typing import List from starlette.routing import Route, Mount from starlette.requests import Request -from starlette.responses import Response, RedirectResponse -from os import path +from starlette.responses import Response, RedirectResponse, FileResponse +from os import path, walk, remove import yaml import jinja2 -from datetime import datetime +from datetime import datetime, timedelta +from PIL import ImageFile +import re +import shutil from core.frontend import FE_Base, UserSession, PermissionOffset from core.config import CoreConfig @@ -31,7 +34,8 @@ class Mai2Frontend(FE_Base): Route("/", self.render_GET, methods=['GET']), Mount("/playlog", routes=[ Route("/", self.render_GET_playlog, methods=['GET']), - Route("/{index}", self.render_GET_playlog, methods=['GET']), + Route("/{index:int}", self.render_GET_playlog, methods=['GET']), + Route("/photos", self.render_GET_photos, methods=['GET']), ]), Mount("/events", routes=[ Route("/", self.render_events, methods=['GET']), @@ -41,6 +45,7 @@ class Mai2Frontend(FE_Base): ]), Route("/update.name", self.update_name, methods=['POST']), Route("/version.change", self.version_change, methods=['POST']), + Route("/photo/{photo_id}", self.get_photo, methods=['GET']), ] async def render_GET(self, request: Request) -> bytes: @@ -140,6 +145,50 @@ class Mai2Frontend(FE_Base): else: return RedirectResponse("/gate/", 303) + async def render_GET_photos(self, request: Request) -> bytes: + template = self.environment.get_template( + "titles/mai2/templates/mai2_photos.jinja" + ) + usr_sesh = self.validate_session(request) + if not usr_sesh: + usr_sesh = UserSession() + + if usr_sesh.user_id > 0: + if usr_sesh.maimai_version < 0: + return RedirectResponse("/game/mai2/", 303) + + photos = await self.data.profile.get_user_photos_by_user(usr_sesh.user_id) + + photos_fixed = [] + for photo in photos: + if datetime.now().timestamp() > (photo['when_upload'] + timedelta(days=7)).timestamp(): + await self.data.profile.delete_user_photo_by_id(photo['id']) + + if path.exists(f"{self.game_cfg.uploads.photos_dir}/{photo['id']}.jpeg"): + remove(f"{self.game_cfg.uploads.photos_dir}/{photo['id']}.jpeg") + + if path.exists(f"{self.game_cfg.uploads.photos_dir}/{photo['id']}"): + shutil.rmtree(f"{self.game_cfg.uploads.photos_dir}/{photo['id']}") + + continue + + photos_fixed.append({ + "id": photo['id'], + "playlog_num": photo['playlog_num'], + "track_num": photo['track_num'], + "when_upload": photo['when_upload'], + }) + + return Response(template.render( + title=f"{self.core_config.server.name} | {self.nav_name}", + game_list=self.environment.globals["game_list"], + sesh=vars(usr_sesh), + photos=photos_fixed, + expire_days=7, + ), media_type="text/html; charset=utf-8") + else: + return RedirectResponse("/gate/", 303) + async def update_name(self, request: Request) -> bytes: usr_sesh = self.validate_session(request) if not usr_sesh: @@ -299,3 +348,76 @@ class Mai2Frontend(FE_Base): await self.data.static.update_event_by_id(int(event_id), new_enabled, new_start_date) return RedirectResponse("/game/mai2/events/?s=1", 303) + + async def get_photo(self, request: Request) -> RedirectResponse: + usr_sesh = self.validate_session(request) + if not usr_sesh: + return RedirectResponse("/gate/", 303) + + photo_jpeg = request.path_params.get("photo_id", None) + if not photo_jpeg: + return Response(status_code=400) + + matcher = re.match(r"^([0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}).jpeg$", photo_jpeg) + if not matcher: + return Response(status_code=400) + + photo_id = matcher.groups()[0] + photo_info = await self.data.profile.get_user_photo_by_id(photo_id) + if not photo_info: + return Response(status_code=404) + + if photo_info["user"] != usr_sesh.user_id: + return Response(status_code=403) + + out_folder = f"{self.game_cfg.uploads.photos_dir}/{photo_id}" + + if datetime.now().timestamp() > (photo_info['when_upload'] + timedelta(days=7)).timestamp(): + await self.data.profile.delete_user_photo_by_id(photo_info['id']) + if path.exists(f"{out_folder}.jpeg"): + remove(f"{out_folder}.jpeg") + + if path.exists(f"{out_folder}"): + shutil.rmtree(out_folder) + + return Response(status_code=404) + + if path.exists(f"{out_folder}"): + print("path exists") + max_idx = 0 + p = ImageFile.Parser() + for _, _, files in walk("out_folder"): + if not files: + break + + matcher = re.match("^(\d+)_(\d+)$", files[0]) + if not matcher: + break + + max_idx = int(matcher.groups()[1]) + + if max_idx + 1 != len(files): + self.logger.error(f"Expected {max_idx + 1} files, found {len(files)}") + max_idx = 0 + break + + if max_idx == 0: + return Response(status_code=500) + + for i in range(max_idx + 1): + with open(f"{out_folder}/{i}_{max_idx}", "rb") as f: + p.feed(f.read()) + try: + im = p.close() + im.save(f"{out_folder}.jpeg") + + except Exception as e: + self.logger.error(f"{photo_id} failed PIL validation! - {e}") + + shutil.rmtree(out_folder) + + if path.exists(f"{out_folder}.jpeg"): + print(f"{out_folder}.jpeg exists") + return FileResponse(f"{out_folder}.jpeg") + + return Response(status_code=404) diff --git a/titles/mai2/schema/profile.py b/titles/mai2/schema/profile.py index 3ff85d2..ede0adf 100644 --- a/titles/mai2/schema/profile.py +++ b/titles/mai2/schema/profile.py @@ -1,9 +1,10 @@ from core.data.schema import BaseData, metadata from titles.mai2.const import Mai2Constants +from uuid import uuid4 from typing import Optional, Dict, List from sqlalchemy import Table, Column, UniqueConstraint, PrimaryKeyConstraint, and_ -from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON, BigInteger, SmallInteger +from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON, BigInteger, SmallInteger, VARCHAR, INTEGER from sqlalchemy.schema import ForeignKey from sqlalchemy.sql import func, select from sqlalchemy.engine import Row @@ -529,6 +530,22 @@ intimacy = Table( mysql_charset="utf8mb4", ) +photo = Table( # end-of-credit memorial photos, NOT user portraits + "mai2_user_photo", + metadata, + Column("id", VARCHAR(36), primary_key=True, nullable=False), + Column( + "user", + ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"), + nullable=False, + ), + Column("playlog_num", INTEGER, nullable=False), + Column("track_num", INTEGER, nullable=False), + Column("when_upload", TIMESTAMP, nullable=False, server_default=func.now()), + UniqueConstraint("user", "playlog_num", "track_num", name="mai2_user_photo_uk"), + mysql_charset="utf8mb4", +) + class Mai2ProfileData(BaseData): async def get_all_profile_versions(self, user_id: int) -> Optional[List[Row]]: result = await self.execute(detail.select(detail.c.user == user_id)) @@ -945,6 +962,41 @@ class Mai2ProfileData(BaseData): self.logger.error(f"Failed to update intimacy for user {user_id} and partner {partner_id}!") + async def put_user_photo(self, user_id: int, playlog_num: int, track_num: int) -> Optional[str]: + photo_id = str(uuid4()) + sql = insert(photo).values( + id = photo_id, + user = user_id, + playlog_num = playlog_num, + track_num = track_num, + ) + + conflict = sql.on_duplicate_key_update(user = user_id) + + result = await self.execute(conflict) + if result: + return photo_id + + async def get_user_photo_by_id(self, photo_id: str) -> Optional[Row]: + result = await self.execute(photo.select(photo.c.id.like(photo_id))) + if result: + return result.fetchone() + + async def get_user_photo_by_user_playlog_track(self, user_id: int, playlog_num: int, track_num: int) -> Optional[Row]: + result = await self.execute(photo.select(and_(and_(photo.c.user == user_id, photo.c.playlog_num == playlog_num), photo.c.track_num == track_num))) + if result: + return result.fetchone() + + async def get_user_photos_by_user(self, user_id: int) -> Optional[List[Row]]: + result = await self.execute(photo.select(photo.c.user == user_id)) + if result: + return result.fetchall() + + async def delete_user_photo_by_id(self, photo_id: str) -> Optional[List[Row]]: + result = await self.execute(photo.delete(photo.c.id.like(photo_id))) + if not result: + self.logger.error(f"Failed to delete photo {photo_id}") + async def update_name(self, user_id: int, new_name: str) -> bool: sql = detail.update(detail.c.user == user_id).values( userName=new_name diff --git a/titles/mai2/templates/mai2_header.jinja b/titles/mai2/templates/mai2_header.jinja index f226fbe..7e6757f 100644 --- a/titles/mai2/templates/mai2_header.jinja +++ b/titles/mai2/templates/mai2_header.jinja @@ -3,6 +3,7 @@