Cool_Tools/bemaniutils
1
0
Fork 0
mirror of synced 2024-11-23 22:10:59 +01:00
Code Releases Activity

Use cache layer to properly shard Jubeat music data responses.

Browse Source
This commit is contained in:
Jennifer Taylor 2023-08-19 20:08:35 +00:00
parent d162d57024
commit 387ec1a272
7 changed files with 271 additions and 257 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
bemani/backend/jubeat/base.py
View File

@ -176,14 +176,43 @@ class JubeatBase(CoreHandler, CardManagerHandler, PASELIHandler, Base):
if profile is None: if profile is None:
return None return None
if partition != 1: cache_key = f"get_scores_by_extid-{extid}"
scores = [] score: Optional[List[Score]]
else:
if partition == 1:
# We fetch all scores on the first partition and then divy up
# the scores across total_partitions fetches. If it is small
# enough, we don't bother.
scores = self.data.remote.music.get_scores( scores = self.data.remote.music.get_scores(
self.game, self.music_version, userid self.game, self.music_version, userid
) )
if scores is None: else:
return None # We will want to fetch the remaining scores that were in our
# cache.
scores = self.cache.get(cache_key) # type: ignore
if len(scores) < 50:
# We simply return the whole amount for this, and cache nothing.
rest = []
else:
groups = (total_partitions - partition) + 1
pivot = len(scores) // groups
rest = scores[pivot:]
scores = scores[:pivot]
# Cache the rest of the scores for next iteration, unless we're on the
# last iteration.
if partition == total_partitions:
if rest:
raise Exception(
"Logic error, should not have gotten additional scores to cache on last iteration!"
)
self.cache.delete(cache_key)
else:
self.cache.set(cache_key, rest, timeout=60)
# Format the chunk of scores we have to send back to the client.
return self.format_scores(userid, profile, scores) return self.format_scores(userid, profile, scores)
def update_score( def update_score(

78
bemani/client/jubeat/clan.py
View File

@ -478,50 +478,48 @@ class JubeatClanClient(BaseClient):
return self.__verify_profile(resp) return self.__verify_profile(resp)
def verify_gametop_get_mdata(self, jid: int) -> Dict[str, List[Dict[str, Any]]]: def verify_gametop_get_mdata(self, jid: int) -> Dict[str, List[Dict[str, Any]]]:
call = self.call_node()
# Construct node
gametop = Node.void("gametop")
call.add_child(gametop)
gametop.set_attribute("method", "get_mdata")
retry = Node.s32("retry", 0)
gametop.add_child(retry)
data = Node.void("data")
gametop.add_child(data)
player = Node.void("player")
data.add_child(player)
player.add_child(Node.s32("jid", jid))
# Technically the game sends this same packet 3 times, one with
# each value 1, 2, 3 here. Unclear why, but we won't emulate it.
player.add_child(Node.s8("mdata_ver", 1))
player.add_child(Node.bool("rival", False))
# Swap with server
resp = self.exchange("", call)
# Parse out scores
self.assert_path(resp, "response/gametop/data/player/mdata_list")
ret = {} ret = {}
for musicdata in resp.child("gametop/data/player/mdata_list").children: for ver in [1, 2, 3]:
if musicdata.name != "musicdata": # Construct node
raise Exception("Unexpected node in playdata!") call = self.call_node()
gametop = Node.void("gametop")
call.add_child(gametop)
gametop.set_attribute("method", "get_mdata")
retry = Node.s32("retry", 0)
gametop.add_child(retry)
data = Node.void("data")
gametop.add_child(data)
player = Node.void("player")
data.add_child(player)
player.add_child(Node.s32("jid", jid))
player.add_child(Node.s8("mdata_ver", ver))
player.add_child(Node.bool("rival", False))
music_id = musicdata.attribute("music_id") # Swap with server
scores_by_chart: List[Dict[str, int]] = [{}, {}, {}] resp = self.exchange("", call)
def extract_cnts(name: str, val: List[int]) -> None: # Parse out scores
scores_by_chart[0][name] = val[0] self.assert_path(resp, "response/gametop/data/player/mdata_list")
scores_by_chart[1][name] = val[1]
scores_by_chart[2][name] = val[2]
extract_cnts("plays", musicdata.child_value("play_cnt")) for musicdata in resp.child("gametop/data/player/mdata_list").children:
extract_cnts("clears", musicdata.child_value("clear_cnt")) if musicdata.name != "musicdata":
extract_cnts("full_combos", musicdata.child_value("fc_cnt")) raise Exception("Unexpected node in playdata!")
extract_cnts("excellents", musicdata.child_value("ex_cnt"))
extract_cnts("score", musicdata.child_value("score")) music_id = musicdata.attribute("music_id")
extract_cnts("medal", musicdata.child_value("clear")) scores_by_chart: List[Dict[str, int]] = [{}, {}, {}]
ret[music_id] = scores_by_chart
def extract_cnts(name: str, val: List[int]) -> None:
scores_by_chart[0][name] = val[0]
scores_by_chart[1][name] = val[1]
scores_by_chart[2][name] = val[2]
extract_cnts("plays", musicdata.child_value("play_cnt"))
extract_cnts("clears", musicdata.child_value("clear_cnt"))
extract_cnts("full_combos", musicdata.child_value("fc_cnt"))
extract_cnts("excellents", musicdata.child_value("ex_cnt"))
extract_cnts("score", musicdata.child_value("score"))
extract_cnts("medal", musicdata.child_value("clear"))
ret[music_id] = scores_by_chart
return ret return ret

103
bemani/client/jubeat/festo.py
View File

@ -582,66 +582,61 @@ class JubeatFestoClient(BaseClient):
return self.__verify_profile(resp, False) return self.__verify_profile(resp, False)
def verify_gametop_get_mdata(self, jid: int) -> Dict[str, List[Dict[str, Any]]]: def verify_gametop_get_mdata(self, jid: int) -> Dict[str, List[Dict[str, Any]]]:
call = self.call_node()
# Construct node
gametop = Node.void("gametop")
call.add_child(gametop)
gametop.set_attribute("method", "get_mdata")
retry = Node.s32("retry", 0)
gametop.add_child(retry)
data = Node.void("data")
gametop.add_child(data)
player = Node.void("player")
data.add_child(player)
player.add_child(Node.s32("jid", jid))
# Technically the game sends this same packet 3 times, one with
# each value 1, 2, 3 here. This is for sharding across 3 requests,
# and the game will combine all 3 responses. Its up to the server to
# handle this the way it wants, and we just send everything back in the
# first request and ignore the rest.
player.add_child(Node.s8("mdata_ver", 1))
player.add_child(Node.bool("rival", False))
# Swap with server
resp = self.exchange("", call)
# Parse out scores
self.assert_path(resp, "response/gametop/data/player/jid")
self.assert_path(resp, "response/gametop/data/player/mdata_list")
if resp.child_value("gametop/data/player/jid") != jid:
raise Exception("Unexpected jid received from server!")
ret = {} ret = {}
for musicdata in resp.child("gametop/data/player/mdata_list").children: for ver in [1, 2, 3]:
if musicdata.name != "musicdata": # Construct node
raise Exception("Unexpected node in playdata!") call = self.call_node()
gametop = Node.void("gametop")
call.add_child(gametop)
gametop.set_attribute("method", "get_mdata")
retry = Node.s32("retry", 0)
gametop.add_child(retry)
data = Node.void("data")
gametop.add_child(data)
player = Node.void("player")
data.add_child(player)
player.add_child(Node.s32("jid", jid))
player.add_child(Node.s8("mdata_ver", ver))
player.add_child(Node.bool("rival", False))
music_id = musicdata.attribute("music_id") # Swap with server
scores_by_chart: List[Dict[str, int]] = [{}, {}, {}, {}, {}, {}] resp = self.exchange("", call)
def extract_cnts(name: str, offset: int, val: List[int]) -> None: # Parse out scores
scores_by_chart[offset + 0][name] = val[0] self.assert_path(resp, "response/gametop/data/player/jid")
scores_by_chart[offset + 1][name] = val[1] self.assert_path(resp, "response/gametop/data/player/mdata_list")
scores_by_chart[offset + 2][name] = val[2] if resp.child_value("gametop/data/player/jid") != jid:
raise Exception("Unexpected jid received from server!")
for subdata in musicdata.children: for musicdata in resp.child("gametop/data/player/mdata_list").children:
if subdata.name == "normal": if musicdata.name != "musicdata":
offset = 0 raise Exception("Unexpected node in playdata!")
elif subdata.name == "hard":
offset = 3
else:
raise Exception(f"Unexpected chart type {subdata.name}!")
extract_cnts("plays", offset, subdata.child_value("play_cnt")) music_id = musicdata.attribute("music_id")
extract_cnts("clears", offset, subdata.child_value("clear_cnt")) scores_by_chart: List[Dict[str, int]] = [{}, {}, {}, {}, {}, {}]
extract_cnts("full_combos", offset, subdata.child_value("fc_cnt"))
extract_cnts("excellents", offset, subdata.child_value("ex_cnt"))
extract_cnts("score", offset, subdata.child_value("score"))
extract_cnts("medal", offset, subdata.child_value("clear"))
extract_cnts("rate", offset, subdata.child_value("music_rate"))
ret[music_id] = scores_by_chart def extract_cnts(name: str, offset: int, val: List[int]) -> None:
scores_by_chart[offset + 0][name] = val[0]
scores_by_chart[offset + 1][name] = val[1]
scores_by_chart[offset + 2][name] = val[2]
for subdata in musicdata.children:
if subdata.name == "normal":
offset = 0
elif subdata.name == "hard":
offset = 3
else:
raise Exception(f"Unexpected chart type {subdata.name}!")
extract_cnts("plays", offset, subdata.child_value("play_cnt"))
extract_cnts("clears", offset, subdata.child_value("clear_cnt"))
extract_cnts("full_combos", offset, subdata.child_value("fc_cnt"))
extract_cnts("excellents", offset, subdata.child_value("ex_cnt"))
extract_cnts("score", offset, subdata.child_value("score"))
extract_cnts("medal", offset, subdata.child_value("clear"))
extract_cnts("rate", offset, subdata.child_value("music_rate"))
ret[music_id] = scores_by_chart
return ret return ret

78
bemani/client/jubeat/prop.py
View File

@ -392,50 +392,48 @@ class JubeatPropClient(BaseClient):
return self.__verify_profile(resp) return self.__verify_profile(resp)
def verify_gametop_get_mdata(self, jid: int) -> Dict[str, List[Dict[str, Any]]]: def verify_gametop_get_mdata(self, jid: int) -> Dict[str, List[Dict[str, Any]]]:
call = self.call_node()
# Construct node
gametop = Node.void("gametop")
call.add_child(gametop)
gametop.set_attribute("method", "get_mdata")
retry = Node.s32("retry", 0)
gametop.add_child(retry)
data = Node.void("data")
gametop.add_child(data)
player = Node.void("player")
data.add_child(player)
player.add_child(Node.s32("jid", jid))
# Technically the game sends this same packet 3 times, one with
# each value 1, 2, 3 here. Unclear why, but we won't emulate it.
player.add_child(Node.s8("mdata_ver", 1))
player.add_child(Node.bool("rival", False))
# Swap with server
resp = self.exchange("", call)
# Parse out scores
self.assert_path(resp, "response/gametop/data/player/mdata_list")
ret = {} ret = {}
for musicdata in resp.child("gametop/data/player/mdata_list").children: for ver in [1, 2, 3]:
if musicdata.name != "musicdata": # Construct node
raise Exception("Unexpected node in playdata!") call = self.call_node()
gametop = Node.void("gametop")
call.add_child(gametop)
gametop.set_attribute("method", "get_mdata")
retry = Node.s32("retry", 0)
gametop.add_child(retry)
data = Node.void("data")
gametop.add_child(data)
player = Node.void("player")
data.add_child(player)
player.add_child(Node.s32("jid", jid))
player.add_child(Node.s8("mdata_ver", ver))
player.add_child(Node.bool("rival", False))
music_id = musicdata.attribute("music_id") # Swap with server
scores_by_chart: List[Dict[str, int]] = [{}, {}, {}] resp = self.exchange("", call)
def extract_cnts(name: str, val: List[int]) -> None: # Parse out scores
scores_by_chart[0][name] = val[0] self.assert_path(resp, "response/gametop/data/player/mdata_list")
scores_by_chart[1][name] = val[1]
scores_by_chart[2][name] = val[2]
extract_cnts("plays", musicdata.child_value("play_cnt")) for musicdata in resp.child("gametop/data/player/mdata_list").children:
extract_cnts("clears", musicdata.child_value("clear_cnt")) if musicdata.name != "musicdata":
extract_cnts("full_combos", musicdata.child_value("fc_cnt")) raise Exception("Unexpected node in playdata!")
extract_cnts("excellents", musicdata.child_value("ex_cnt"))
extract_cnts("score", musicdata.child_value("score")) music_id = musicdata.attribute("music_id")
extract_cnts("medal", musicdata.child_value("clear")) scores_by_chart: List[Dict[str, int]] = [{}, {}, {}]
ret[music_id] = scores_by_chart
def extract_cnts(name: str, val: List[int]) -> None:
scores_by_chart[0][name] = val[0]
scores_by_chart[1][name] = val[1]
scores_by_chart[2][name] = val[2]
extract_cnts("plays", musicdata.child_value("play_cnt"))
extract_cnts("clears", musicdata.child_value("clear_cnt"))
extract_cnts("full_combos", musicdata.child_value("fc_cnt"))
extract_cnts("excellents", musicdata.child_value("ex_cnt"))
extract_cnts("score", musicdata.child_value("score"))
extract_cnts("medal", musicdata.child_value("clear"))
ret[music_id] = scores_by_chart
return ret return ret

78
bemani/client/jubeat/qubell.py
View File

@ -357,50 +357,48 @@ class JubeatQubellClient(BaseClient):
return self.__verify_profile(resp) return self.__verify_profile(resp)
def verify_gametop_get_mdata(self, jid: int) -> Dict[str, List[Dict[str, Any]]]: def verify_gametop_get_mdata(self, jid: int) -> Dict[str, List[Dict[str, Any]]]:
call = self.call_node()
# Construct node
gametop = Node.void("gametop")
call.add_child(gametop)
gametop.set_attribute("method", "get_mdata")
retry = Node.s32("retry", 0)
gametop.add_child(retry)
data = Node.void("data")
gametop.add_child(data)
player = Node.void("player")
data.add_child(player)
player.add_child(Node.s32("jid", jid))
# Technically the game sends this same packet 3 times, one with
# each value 1, 2, 3 here. Unclear why, but we won't emulate it.
player.add_child(Node.s8("mdata_ver", 1))
player.add_child(Node.bool("rival", False))
# Swap with server
resp = self.exchange("", call)
# Parse out scores
self.assert_path(resp, "response/gametop/data/player/mdata_list")
ret = {} ret = {}
for musicdata in resp.child("gametop/data/player/mdata_list").children: for ver in [1, 2, 3]:
if musicdata.name != "musicdata": # Construct node
raise Exception("Unexpected node in playdata!") call = self.call_node()
gametop = Node.void("gametop")
call.add_child(gametop)
gametop.set_attribute("method", "get_mdata")
retry = Node.s32("retry", 0)
gametop.add_child(retry)
data = Node.void("data")
gametop.add_child(data)
player = Node.void("player")
data.add_child(player)
player.add_child(Node.s32("jid", jid))
player.add_child(Node.s8("mdata_ver", ver))
player.add_child(Node.bool("rival", False))
music_id = musicdata.attribute("music_id") # Swap with server
scores_by_chart: List[Dict[str, int]] = [{}, {}, {}] resp = self.exchange("", call)
def extract_cnts(name: str, val: List[int]) -> None: # Parse out scores
scores_by_chart[0][name] = val[0] self.assert_path(resp, "response/gametop/data/player/mdata_list")
scores_by_chart[1][name] = val[1]
scores_by_chart[2][name] = val[2]
extract_cnts("plays", musicdata.child_value("play_cnt")) for musicdata in resp.child("gametop/data/player/mdata_list").children:
extract_cnts("clears", musicdata.child_value("clear_cnt")) if musicdata.name != "musicdata":
extract_cnts("full_combos", musicdata.child_value("fc_cnt")) raise Exception("Unexpected node in playdata!")
extract_cnts("excellents", musicdata.child_value("ex_cnt"))
extract_cnts("score", musicdata.child_value("score")) music_id = musicdata.attribute("music_id")
extract_cnts("medal", musicdata.child_value("clear")) scores_by_chart: List[Dict[str, int]] = [{}, {}, {}]
ret[music_id] = scores_by_chart
def extract_cnts(name: str, val: List[int]) -> None:
scores_by_chart[0][name] = val[0]
scores_by_chart[1][name] = val[1]
scores_by_chart[2][name] = val[2]
extract_cnts("plays", musicdata.child_value("play_cnt"))
extract_cnts("clears", musicdata.child_value("clear_cnt"))
extract_cnts("full_combos", musicdata.child_value("fc_cnt"))
extract_cnts("excellents", musicdata.child_value("ex_cnt"))
extract_cnts("score", musicdata.child_value("score"))
extract_cnts("medal", musicdata.child_value("clear"))
ret[music_id] = scores_by_chart
return ret return ret

76
bemani/client/jubeat/saucer.py
View File

@ -275,49 +275,47 @@ class JubeatSaucerClient(BaseClient):
return self.__verify_profile(resp) return self.__verify_profile(resp)
def verify_gametop_get_mdata(self, jid: int) -> Dict[str, List[Dict[str, Any]]]: def verify_gametop_get_mdata(self, jid: int) -> Dict[str, List[Dict[str, Any]]]:
call = self.call_node()
# Construct node
gametop = Node.void("gametop")
call.add_child(gametop)
gametop.set_attribute("method", "get_mdata")
retry = Node.s32("retry", 0)
gametop.add_child(retry)
data = Node.void("data")
gametop.add_child(data)
player = Node.void("player")
data.add_child(player)
player.add_child(Node.s32("jid", jid))
# Technically the game sends this same packet 3 times, one with
# each value 1, 2, 3 here. Unclear why, but we won't emulate it.
player.add_child(Node.s8("mdata_ver", 1))
# Swap with server
resp = self.exchange("", call)
# Parse out scores
self.assert_path(resp, "response/gametop/data/player/playdata")
ret = {} ret = {}
for musicdata in resp.child("gametop/data/player/playdata").children: for ver in [1, 2, 3]:
if musicdata.name != "musicdata": # Construct node
raise Exception("Unexpected node in playdata!") call = self.call_node()
gametop = Node.void("gametop")
call.add_child(gametop)
gametop.set_attribute("method", "get_mdata")
retry = Node.s32("retry", 0)
gametop.add_child(retry)
data = Node.void("data")
gametop.add_child(data)
player = Node.void("player")
data.add_child(player)
player.add_child(Node.s32("jid", jid))
player.add_child(Node.s8("mdata_ver", ver))
music_id = musicdata.attribute("music_id") # Swap with server
scores_by_chart: List[Dict[str, int]] = [{}, {}, {}] resp = self.exchange("", call)
def extract_cnts(name: str, val: List[int]) -> None: # Parse out scores
scores_by_chart[0][name] = val[0] self.assert_path(resp, "response/gametop/data/player/playdata")
scores_by_chart[1][name] = val[1]
scores_by_chart[2][name] = val[2]
extract_cnts("plays", musicdata.child_value("play_cnt")) for musicdata in resp.child("gametop/data/player/playdata").children:
extract_cnts("clears", musicdata.child_value("clear_cnt")) if musicdata.name != "musicdata":
extract_cnts("full_combos", musicdata.child_value("fc_cnt")) raise Exception("Unexpected node in playdata!")
extract_cnts("excellents", musicdata.child_value("ex_cnt"))
extract_cnts("score", musicdata.child_value("score")) music_id = musicdata.attribute("music_id")
extract_cnts("medal", musicdata.child_value("clear")) scores_by_chart: List[Dict[str, int]] = [{}, {}, {}]
ret[music_id] = scores_by_chart
def extract_cnts(name: str, val: List[int]) -> None:
scores_by_chart[0][name] = val[0]
scores_by_chart[1][name] = val[1]
scores_by_chart[2][name] = val[2]
extract_cnts("plays", musicdata.child_value("play_cnt"))
extract_cnts("clears", musicdata.child_value("clear_cnt"))
extract_cnts("full_combos", musicdata.child_value("fc_cnt"))
extract_cnts("excellents", musicdata.child_value("ex_cnt"))
extract_cnts("score", musicdata.child_value("score"))
extract_cnts("medal", musicdata.child_value("clear"))
ret[music_id] = scores_by_chart
return ret return ret

76
bemani/client/jubeat/saucerfulfill.py
View File

@ -310,49 +310,47 @@ class JubeatSaucerFulfillClient(BaseClient):
return self.__verify_profile(resp) return self.__verify_profile(resp)
def verify_gametop_get_mdata(self, jid: int) -> Dict[str, List[Dict[str, Any]]]: def verify_gametop_get_mdata(self, jid: int) -> Dict[str, List[Dict[str, Any]]]:
call = self.call_node()
# Construct node
gametop = Node.void("gametop")
call.add_child(gametop)
gametop.set_attribute("method", "get_mdata")
retry = Node.s32("retry", 0)
gametop.add_child(retry)
data = Node.void("data")
gametop.add_child(data)
player = Node.void("player")
data.add_child(player)
player.add_child(Node.s32("jid", jid))
# Technically the game sends this same packet 3 times, one with
# each value 1, 2, 3 here. Unclear why, but we won't emulate it.
player.add_child(Node.s8("mdata_ver", 1))
# Swap with server
resp = self.exchange("", call)
# Parse out scores
self.assert_path(resp, "response/gametop/data/player/playdata")
ret = {} ret = {}
for musicdata in resp.child("gametop/data/player/playdata").children: for ver in [1, 2, 3]:
if musicdata.name != "musicdata": # Construct node
raise Exception("Unexpected node in playdata!") call = self.call_node()
gametop = Node.void("gametop")
call.add_child(gametop)
gametop.set_attribute("method", "get_mdata")
retry = Node.s32("retry", 0)
gametop.add_child(retry)
data = Node.void("data")
gametop.add_child(data)
player = Node.void("player")
data.add_child(player)
player.add_child(Node.s32("jid", jid))
player.add_child(Node.s8("mdata_ver", ver))
music_id = musicdata.attribute("music_id") # Swap with server
scores_by_chart: List[Dict[str, int]] = [{}, {}, {}] resp = self.exchange("", call)
def extract_cnts(name: str, val: List[int]) -> None: # Parse out scores
scores_by_chart[0][name] = val[0] self.assert_path(resp, "response/gametop/data/player/playdata")
scores_by_chart[1][name] = val[1]
scores_by_chart[2][name] = val[2]
extract_cnts("plays", musicdata.child_value("play_cnt")) for musicdata in resp.child("gametop/data/player/playdata").children:
extract_cnts("clears", musicdata.child_value("clear_cnt")) if musicdata.name != "musicdata":
extract_cnts("full_combos", musicdata.child_value("fc_cnt")) raise Exception("Unexpected node in playdata!")
extract_cnts("excellents", musicdata.child_value("ex_cnt"))
extract_cnts("score", musicdata.child_value("score")) music_id = musicdata.attribute("music_id")
extract_cnts("medal", musicdata.child_value("clear")) scores_by_chart: List[Dict[str, int]] = [{}, {}, {}]
ret[music_id] = scores_by_chart
def extract_cnts(name: str, val: List[int]) -> None:
scores_by_chart[0][name] = val[0]
scores_by_chart[1][name] = val[1]
scores_by_chart[2][name] = val[2]
extract_cnts("plays", musicdata.child_value("play_cnt"))
extract_cnts("clears", musicdata.child_value("clear_cnt"))
extract_cnts("full_combos", musicdata.child_value("fc_cnt"))
extract_cnts("excellents", musicdata.child_value("ex_cnt"))
extract_cnts("score", musicdata.child_value("score"))
extract_cnts("medal", musicdata.child_value("clear"))
ret[music_id] = scores_by_chart
return ret return ret