Tendokyu/artemis
1
0
Fork 0
mirror of synced 2025-02-22 20:50:07 +01:00
Code Issues Packages Projects Releases Wiki Activity
artemis/titles/mai2/schema/score.py
beerpsi 58a5177a30 use SQL's limit/offset pagination for nextIndex/maxCount requests (#185) 
Instead of retrieving the entire list of items/characters/scores/etc. at once (and even store them in memory), use SQL's `LIMIT ... OFFSET ...` pagination so we only take what we need.

Currently only CHUNITHM uses this, but this will also affect maimai DX and O.N.G.E.K.I. once the PR is ready.

Also snuck in a fix for CHUNITHM/maimai DX's `GetUserRivalMusicApi` to respect the `userRivalMusicLevelList` sent by the client.

### How this works

Say we have a `GetUserCharacterApi` request:

```json
{
    "userId": 10000,
    "maxCount": 700,
    "nextIndex": 0
}
```

Instead of getting the entire character list from the database (which can be very large if the user force unlocked everything), add limit/offset to the query:

```python
select(character)
.where(character.c.user == user_id)
.order_by(character.c.id.asc())
.limit(max_count + 1)
.offset(next_index)
```

The query takes `maxCount + 1` items from the database to determine if there is more items than can be returned:

```python
rows = ...

if len(rows) > max_count:
    # return only max_count rows
    next_index += max_count
else:
    # return everything left
    next_index = -1
```

This has the benefit of not needing to load everything into memory (and also having to store server state, as seen in the [`SCORE_BUFFER` list](2274b42358/titles/chuni/base.py (L13)).)

Reviewed-on: https://gitea.tendokyu.moe/Hay1tsme/artemis/pulls/185
Co-authored-by: beerpsi <beerpsi@duck.com>
Co-committed-by: beerpsi <beerpsi@duck.com>
2024-11-16 19:10:29 +00:00

453 lines
15 KiB
Python
Raw Blame History

from typing import Dict, List, Optional
from sqlalchemy import Column, Table, UniqueConstraint, and_
from sqlalchemy.dialects.mysql import insert
from sqlalchemy.engine import Row
from sqlalchemy.schema import ForeignKey
from sqlalchemy.sql import func, select
from sqlalchemy.types import JSON, BigInteger, Boolean, Integer, String
from core.data.schema import BaseData, metadata
best_score: Table = Table(
"mai2_score_best",
metadata,
Column("id", Integer, primary_key=True, nullable=False),
Column(
"user",
ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
nullable=False,
),
Column("musicId", Integer),
Column("level", Integer),
Column("playCount", Integer),
Column("achievement", Integer),
Column("comboStatus", Integer),
Column("syncStatus", Integer),
Column("deluxscoreMax", Integer),
Column("scoreRank", Integer),
Column("extNum1", Integer, server_default="0"),
UniqueConstraint("user", "musicId", "level", name="mai2_score_best_uk"),
mysql_charset="utf8mb4",
)
playlog = Table(
"mai2_playlog",
metadata,
Column("id", Integer, primary_key=True, nullable=False),
Column(
"user",
ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
nullable=False,
),
Column("userId", BigInteger),
Column("orderId", Integer),
Column("playlogId", BigInteger),
Column("version", Integer),
Column("placeId", Integer),
Column("placeName", String(255)),
Column("loginDate", BigInteger),
Column("playDate", String(255)),
Column("userPlayDate", String(255)),
Column("type", Integer),
Column("musicId", Integer),
Column("level", Integer),
Column("trackNo", Integer),
Column("vsMode", Integer),
Column("vsUserName", String(255)),
Column("vsStatus", Integer),
Column("vsUserRating", Integer),
Column("vsUserAchievement", Integer),
Column("vsUserGradeRank", Integer),
Column("vsRank", Integer),
Column("playerNum", Integer),
Column("playedUserId1", BigInteger),
Column("playedUserName1", String(255)),
Column("playedMusicLevel1", Integer),
Column("playedUserId2", BigInteger),
Column("playedUserName2", String(255)),
Column("playedMusicLevel2", Integer),
Column("playedUserId3", BigInteger),
Column("playedUserName3", String(255)),
Column("playedMusicLevel3", Integer),
Column("characterId1", Integer),
Column("characterLevel1", Integer),
Column("characterAwakening1", Integer),
Column("characterId2", Integer),
Column("characterLevel2", Integer),
Column("characterAwakening2", Integer),
Column("characterId3", Integer),
Column("characterLevel3", Integer),
Column("characterAwakening3", Integer),
Column("characterId4", Integer),
Column("characterLevel4", Integer),
Column("characterAwakening4", Integer),
Column("characterId5", Integer),
Column("characterLevel5", Integer),
Column("characterAwakening5", Integer),
Column("achievement", Integer),
Column("deluxscore", Integer),
Column("scoreRank", Integer),
Column("maxCombo", Integer),
Column("totalCombo", Integer),
Column("maxSync", Integer),
Column("totalSync", Integer),
Column("tapCriticalPerfect", Integer),
Column("tapPerfect", Integer),
Column("tapGreat", Integer),
Column("tapGood", Integer),
Column("tapMiss", Integer),
Column("holdCriticalPerfect", Integer),
Column("holdPerfect", Integer),
Column("holdGreat", Integer),
Column("holdGood", Integer),
Column("holdMiss", Integer),
Column("slideCriticalPerfect", Integer),
Column("slidePerfect", Integer),
Column("slideGreat", Integer),
Column("slideGood", Integer),
Column("slideMiss", Integer),
Column("touchCriticalPerfect", Integer),
Column("touchPerfect", Integer),
Column("touchGreat", Integer),
Column("touchGood", Integer),
Column("touchMiss", Integer),
Column("breakCriticalPerfect", Integer),
Column("breakPerfect", Integer),
Column("breakGreat", Integer),
Column("breakGood", Integer),
Column("breakMiss", Integer),
Column("isTap", Boolean),
Column("isHold", Boolean),
Column("isSlide", Boolean),
Column("isTouch", Boolean),
Column("isBreak", Boolean),
Column("isCriticalDisp", Boolean),
Column("isFastLateDisp", Boolean),
Column("fastCount", Integer),
Column("lateCount", Integer),
Column("isAchieveNewRecord", Boolean),
Column("isDeluxscoreNewRecord", Boolean),
Column("comboStatus", Integer),
Column("syncStatus", Integer),
Column("isClear", Boolean),
Column("beforeRating", Integer),
Column("afterRating", Integer),
Column("beforeGrade", Integer),
Column("afterGrade", Integer),
Column("afterGradeRank", Integer),
Column("beforeDeluxRating", Integer),
Column("afterDeluxRating", Integer),
Column("isPlayTutorial", Boolean),
Column("isEventMode", Boolean),
Column("isFreedomMode", Boolean),
Column("playMode", Integer),
Column("isNewFree", Boolean),
Column("extNum1", Integer),
Column("extNum2", Integer),
Column("extNum4", Integer),
Column("extBool1", Boolean), # new with buddies
Column("trialPlayAchievement", Integer),
mysql_charset="utf8mb4",
)
# new with buddies
playlog_2p = Table(
"mai2_playlog_2p",
metadata,
Column("id", Integer, primary_key=True, nullable=False),
Column(
"user",
ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
nullable=False,
),
# TODO: ForeignKey to aime_user?
Column("userId1", Integer),
Column("userId2", Integer),
# TODO: ForeignKey to mai2_profile_detail?
Column("userName1", String(25)),
Column("userName2", String(25)),
Column("regionId", Integer),
Column("placeId", Integer),
Column("user2pPlaylogDetailList", JSON),
mysql_charset="utf8mb4",
)
course = Table(
"mai2_score_course",
metadata,
Column("id", Integer, primary_key=True, nullable=False),
Column(
"user",
ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
nullable=False,
),
Column("courseId", Integer),
Column("isLastClear", Boolean),
Column("totalRestlife", Integer),
Column("totalAchievement", Integer),
Column("totalDeluxscore", Integer),
Column("playCount", Integer),
Column("clearDate", String(25)),
Column("lastPlayDate", String(25)),
Column("bestAchievement", Integer),
Column("bestAchievementDate", String(25)),
Column("bestDeluxscore", Integer),
Column("bestDeluxscoreDate", String(25)),
UniqueConstraint("user", "courseId", name="mai2_score_best_uk"),
mysql_charset="utf8mb4",
)
playlog_old = Table(
"maimai_playlog",
metadata,
Column("id", Integer, primary_key=True, nullable=False),
Column(
"user",
ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
nullable=False,
),
Column("version", Integer),
# Pop access code
Column("orderId", Integer),
Column("sortNumber", Integer),
Column("placeId", Integer),
Column("placeName", String(255)),
Column("country", String(255)),
Column("regionId", Integer),
Column("playDate", String(255)),
Column("userPlayDate", String(255)),
Column("musicId", Integer),
Column("level", Integer),
Column("gameMode", Integer),
Column("rivalNum", Integer),
Column("track", Integer),
Column("eventId", Integer),
Column("isFreeToPlay", Boolean),
Column("playerRating", Integer),
Column("playedUserId1", Integer),
Column("playedUserId2", Integer),
Column("playedUserId3", Integer),
Column("playedUserName1", String(255)),
Column("playedUserName2", String(255)),
Column("playedUserName3", String(255)),
Column("playedMusicLevel1", Integer),
Column("playedMusicLevel2", Integer),
Column("playedMusicLevel3", Integer),
Column("achievement", Integer),
Column("score", Integer),
Column("tapScore", Integer),
Column("holdScore", Integer),
Column("slideScore", Integer),
Column("breakScore", Integer),
Column("syncRate", Integer),
Column("vsWin", Integer),
Column("isAllPerfect", Boolean),
Column("fullCombo", Integer),
Column("maxFever", Integer),
Column("maxCombo", Integer),
Column("tapPerfect", Integer),
Column("tapGreat", Integer),
Column("tapGood", Integer),
Column("tapBad", Integer),
Column("holdPerfect", Integer),
Column("holdGreat", Integer),
Column("holdGood", Integer),
Column("holdBad", Integer),
Column("slidePerfect", Integer),
Column("slideGreat", Integer),
Column("slideGood", Integer),
Column("slideBad", Integer),
Column("breakPerfect", Integer),
Column("breakGreat", Integer),
Column("breakGood", Integer),
Column("breakBad", Integer),
Column("judgeStyle", Integer),
Column("isTrackSkip", Boolean),
Column("isHighScore", Boolean),
Column("isChallengeTrack", Boolean),
Column("challengeLife", Integer),
Column("challengeRemain", Integer),
Column("isAllPerfectPlus", Integer),
mysql_charset="utf8mb4",
)
best_score_old: Table = Table(
"maimai_score_best",
metadata,
Column("id", Integer, primary_key=True, nullable=False),
Column(
"user",
ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
nullable=False,
),
Column("musicId", Integer),
Column("level", Integer),
Column("playCount", Integer),
Column("achievement", Integer),
Column("scoreMax", Integer),
Column("syncRateMax", Integer),
Column("isAllPerfect", Boolean),
Column("isAllPerfectPlus", Integer),
Column("fullCombo", Integer),
Column("maxFever", Integer),
UniqueConstraint("user", "musicId", "level", name="maimai_score_best_uk"),
mysql_charset="utf8mb4",
)
class Mai2ScoreData(BaseData):
async def put_best_score(self, user_id: int, score_data: Dict, is_dx: bool = True) -> Optional[int]:
score_data["user"] = user_id
if is_dx:
sql = insert(best_score).values(**score_data)
else:
sql = insert(best_score_old).values(**score_data)
conflict = sql.on_duplicate_key_update(**score_data)
result = await self.execute(conflict)
if result is None:
self.logger.error(
f"put_best_score: Failed to insert best score! user_id {user_id} is_dx {is_dx}"
)
return None
return result.lastrowid
async def get_best_scores(
self,
user_id: int,
song_id: Optional[int] = None,
is_dx: bool = True,
limit: Optional[int] = None,
offset: Optional[int] = None,
levels: Optional[list[int]] = None,
) -> Optional[List[Row]]:
if is_dx:
table = best_score
else:
table = best_score_old
cond = table.c.user == user_id
if song_id is not None:
cond &= table.c.musicId == song_id
if levels is not None:
cond &= table.c.level.in_(levels)
if limit is None and offset is None:
sql = (
select(table)
.where(cond)
.order_by(table.c.musicId, table.c.level)
)
else:
subq = (
select(table.c.musicId)
.distinct()
.where(cond)
.order_by(table.c.musicId)
)
if limit is not None:
subq = subq.limit(limit)
if offset is not None:
subq = subq.offset(offset)
subq = subq.subquery()
sql = (
select(table)
.join(subq, table.c.musicId == subq.c.musicId)
.where(cond)
.order_by(table.c.musicId, table.c.level)
)
result = await self.execute(sql)
if result is None:
return None
return result.fetchall()
async def get_best_score(
self, user_id: int, song_id: int, chart_id: int
) -> Optional[Row]:
sql = best_score.select(
and_(
best_score.c.user == user_id,
best_score.c.song_id == song_id,
best_score.c.chart_id == chart_id,
)
)
result = await self.execute(sql)
if result is None:
return None
return result.fetchone()
async def put_playlog(self, user_id: int, playlog_data: Dict, is_dx: bool = True) -> Optional[int]:
playlog_data["user"] = user_id
if is_dx:
sql = insert(playlog).values(**playlog_data)
else:
sql = insert(playlog_old).values(**playlog_data)
result = await self.execute(sql)
if result is None:
self.logger.error(f"put_playlog: Failed to insert! user_id {user_id} is_dx {is_dx}")
return None
return result.lastrowid
async def put_playlog_2p(self, user_id: int, playlog_2p_data: Dict) -> Optional[int]:
playlog_2p_data["user"] = user_id
sql = insert(playlog_2p).values(**playlog_2p_data)
result = await self.execute(sql)
if result is None:
self.logger.error(f"put_playlog_2p: Failed to insert! user_id {user_id}")
return None
return result.lastrowid
async def put_course(self, user_id: int, course_data: Dict) -> Optional[int]:
course_data["user"] = user_id
sql = insert(course).values(**course_data)
conflict = sql.on_duplicate_key_update(**course_data)
result = await self.execute(conflict)
if result is None:
self.logger.error(f"put_course: Failed to insert! user_id {user_id}")
return None
return result.lastrowid
async def get_courses(self, user_id: int) -> Optional[List[Row]]:
sql = course.select(course.c.user == user_id)
result = await self.execute(sql)
if result is None:
return None
return result.fetchall()
async def get_playlogs(self, user_id: int, idx: int = 0, limit: int = 0) -> Optional[List[Row]]:
if user_id is not None:
sql = playlog.select(playlog.c.user == user_id)
else:
sql = playlog.select()
if limit:
sql = sql.limit(limit)
if idx:
sql = sql.offset(idx * limit)
result = await self.execute(sql)
if result:
return result.fetchall()
async def get_user_playlogs_count(self, aime_id: int) -> Optional[Row]:
sql = select(func.count()).where(playlog.c.user == aime_id)
result = await self.execute(sql)
if result is None:
self.logger.warning(f"aime_id {aime_id} has no playlog ")
return None
return result.scalar()
Reference in New Issue View Git Blame Copy Permalink