diff --git a/bemani/backend/jubeat/base.py b/bemani/backend/jubeat/base.py index e1371a9..49f6ba7 100644 --- a/bemani/backend/jubeat/base.py +++ b/bemani/backend/jubeat/base.py @@ -176,14 +176,43 @@ class JubeatBase(CoreHandler, CardManagerHandler, PASELIHandler, Base): if profile is None: return None - if partition != 1: - scores = [] - else: + cache_key = f"get_scores_by_extid-{extid}" + score: Optional[List[Score]] + + if partition == 1: + # We fetch all scores on the first partition and then divy up + # the scores across total_partitions fetches. If it is small + # enough, we don't bother. scores = self.data.remote.music.get_scores( self.game, self.music_version, userid ) - if scores is None: - return None + else: + # We will want to fetch the remaining scores that were in our + # cache. + scores = self.cache.get(cache_key) # type: ignore + + if len(scores) < 50: + # We simply return the whole amount for this, and cache nothing. + rest = [] + else: + groups = (total_partitions - partition) + 1 + pivot = len(scores) // groups + + rest = scores[pivot:] + scores = scores[:pivot] + + # Cache the rest of the scores for next iteration, unless we're on the + # last iteration. + if partition == total_partitions: + if rest: + raise Exception( + "Logic error, should not have gotten additional scores to cache on last iteration!" + ) + self.cache.delete(cache_key) + else: + self.cache.set(cache_key, rest, timeout=60) + + # Format the chunk of scores we have to send back to the client. return self.format_scores(userid, profile, scores) def update_score( diff --git a/bemani/client/jubeat/clan.py b/bemani/client/jubeat/clan.py index e16cae8..ebbbd70 100644 --- a/bemani/client/jubeat/clan.py +++ b/bemani/client/jubeat/clan.py @@ -478,50 +478,48 @@ class JubeatClanClient(BaseClient): return self.__verify_profile(resp) def verify_gametop_get_mdata(self, jid: int) -> Dict[str, List[Dict[str, Any]]]: - call = self.call_node() - - # Construct node - gametop = Node.void("gametop") - call.add_child(gametop) - gametop.set_attribute("method", "get_mdata") - retry = Node.s32("retry", 0) - gametop.add_child(retry) - data = Node.void("data") - gametop.add_child(data) - player = Node.void("player") - data.add_child(player) - player.add_child(Node.s32("jid", jid)) - # Technically the game sends this same packet 3 times, one with - # each value 1, 2, 3 here. Unclear why, but we won't emulate it. - player.add_child(Node.s8("mdata_ver", 1)) - player.add_child(Node.bool("rival", False)) - - # Swap with server - resp = self.exchange("", call) - - # Parse out scores - self.assert_path(resp, "response/gametop/data/player/mdata_list") - ret = {} - for musicdata in resp.child("gametop/data/player/mdata_list").children: - if musicdata.name != "musicdata": - raise Exception("Unexpected node in playdata!") + for ver in [1, 2, 3]: + # Construct node + call = self.call_node() + gametop = Node.void("gametop") + call.add_child(gametop) + gametop.set_attribute("method", "get_mdata") + retry = Node.s32("retry", 0) + gametop.add_child(retry) + data = Node.void("data") + gametop.add_child(data) + player = Node.void("player") + data.add_child(player) + player.add_child(Node.s32("jid", jid)) + player.add_child(Node.s8("mdata_ver", ver)) + player.add_child(Node.bool("rival", False)) - music_id = musicdata.attribute("music_id") - scores_by_chart: List[Dict[str, int]] = [{}, {}, {}] + # Swap with server + resp = self.exchange("", call) - def extract_cnts(name: str, val: List[int]) -> None: - scores_by_chart[0][name] = val[0] - scores_by_chart[1][name] = val[1] - scores_by_chart[2][name] = val[2] + # Parse out scores + self.assert_path(resp, "response/gametop/data/player/mdata_list") - extract_cnts("plays", musicdata.child_value("play_cnt")) - extract_cnts("clears", musicdata.child_value("clear_cnt")) - extract_cnts("full_combos", musicdata.child_value("fc_cnt")) - extract_cnts("excellents", musicdata.child_value("ex_cnt")) - extract_cnts("score", musicdata.child_value("score")) - extract_cnts("medal", musicdata.child_value("clear")) - ret[music_id] = scores_by_chart + for musicdata in resp.child("gametop/data/player/mdata_list").children: + if musicdata.name != "musicdata": + raise Exception("Unexpected node in playdata!") + + music_id = musicdata.attribute("music_id") + scores_by_chart: List[Dict[str, int]] = [{}, {}, {}] + + def extract_cnts(name: str, val: List[int]) -> None: + scores_by_chart[0][name] = val[0] + scores_by_chart[1][name] = val[1] + scores_by_chart[2][name] = val[2] + + extract_cnts("plays", musicdata.child_value("play_cnt")) + extract_cnts("clears", musicdata.child_value("clear_cnt")) + extract_cnts("full_combos", musicdata.child_value("fc_cnt")) + extract_cnts("excellents", musicdata.child_value("ex_cnt")) + extract_cnts("score", musicdata.child_value("score")) + extract_cnts("medal", musicdata.child_value("clear")) + ret[music_id] = scores_by_chart return ret diff --git a/bemani/client/jubeat/festo.py b/bemani/client/jubeat/festo.py index 9f5e20e..c6de1d8 100644 --- a/bemani/client/jubeat/festo.py +++ b/bemani/client/jubeat/festo.py @@ -582,66 +582,61 @@ class JubeatFestoClient(BaseClient): return self.__verify_profile(resp, False) def verify_gametop_get_mdata(self, jid: int) -> Dict[str, List[Dict[str, Any]]]: - call = self.call_node() - - # Construct node - gametop = Node.void("gametop") - call.add_child(gametop) - gametop.set_attribute("method", "get_mdata") - retry = Node.s32("retry", 0) - gametop.add_child(retry) - data = Node.void("data") - gametop.add_child(data) - player = Node.void("player") - data.add_child(player) - player.add_child(Node.s32("jid", jid)) - # Technically the game sends this same packet 3 times, one with - # each value 1, 2, 3 here. This is for sharding across 3 requests, - # and the game will combine all 3 responses. Its up to the server to - # handle this the way it wants, and we just send everything back in the - # first request and ignore the rest. - player.add_child(Node.s8("mdata_ver", 1)) - player.add_child(Node.bool("rival", False)) - - # Swap with server - resp = self.exchange("", call) - - # Parse out scores - self.assert_path(resp, "response/gametop/data/player/jid") - self.assert_path(resp, "response/gametop/data/player/mdata_list") - if resp.child_value("gametop/data/player/jid") != jid: - raise Exception("Unexpected jid received from server!") - ret = {} - for musicdata in resp.child("gametop/data/player/mdata_list").children: - if musicdata.name != "musicdata": - raise Exception("Unexpected node in playdata!") + for ver in [1, 2, 3]: + # Construct node + call = self.call_node() + gametop = Node.void("gametop") + call.add_child(gametop) + gametop.set_attribute("method", "get_mdata") + retry = Node.s32("retry", 0) + gametop.add_child(retry) + data = Node.void("data") + gametop.add_child(data) + player = Node.void("player") + data.add_child(player) + player.add_child(Node.s32("jid", jid)) + player.add_child(Node.s8("mdata_ver", ver)) + player.add_child(Node.bool("rival", False)) - music_id = musicdata.attribute("music_id") - scores_by_chart: List[Dict[str, int]] = [{}, {}, {}, {}, {}, {}] + # Swap with server + resp = self.exchange("", call) - def extract_cnts(name: str, offset: int, val: List[int]) -> None: - scores_by_chart[offset + 0][name] = val[0] - scores_by_chart[offset + 1][name] = val[1] - scores_by_chart[offset + 2][name] = val[2] + # Parse out scores + self.assert_path(resp, "response/gametop/data/player/jid") + self.assert_path(resp, "response/gametop/data/player/mdata_list") + if resp.child_value("gametop/data/player/jid") != jid: + raise Exception("Unexpected jid received from server!") - for subdata in musicdata.children: - if subdata.name == "normal": - offset = 0 - elif subdata.name == "hard": - offset = 3 - else: - raise Exception(f"Unexpected chart type {subdata.name}!") + for musicdata in resp.child("gametop/data/player/mdata_list").children: + if musicdata.name != "musicdata": + raise Exception("Unexpected node in playdata!") - extract_cnts("plays", offset, subdata.child_value("play_cnt")) - extract_cnts("clears", offset, subdata.child_value("clear_cnt")) - extract_cnts("full_combos", offset, subdata.child_value("fc_cnt")) - extract_cnts("excellents", offset, subdata.child_value("ex_cnt")) - extract_cnts("score", offset, subdata.child_value("score")) - extract_cnts("medal", offset, subdata.child_value("clear")) - extract_cnts("rate", offset, subdata.child_value("music_rate")) + music_id = musicdata.attribute("music_id") + scores_by_chart: List[Dict[str, int]] = [{}, {}, {}, {}, {}, {}] - ret[music_id] = scores_by_chart + def extract_cnts(name: str, offset: int, val: List[int]) -> None: + scores_by_chart[offset + 0][name] = val[0] + scores_by_chart[offset + 1][name] = val[1] + scores_by_chart[offset + 2][name] = val[2] + + for subdata in musicdata.children: + if subdata.name == "normal": + offset = 0 + elif subdata.name == "hard": + offset = 3 + else: + raise Exception(f"Unexpected chart type {subdata.name}!") + + extract_cnts("plays", offset, subdata.child_value("play_cnt")) + extract_cnts("clears", offset, subdata.child_value("clear_cnt")) + extract_cnts("full_combos", offset, subdata.child_value("fc_cnt")) + extract_cnts("excellents", offset, subdata.child_value("ex_cnt")) + extract_cnts("score", offset, subdata.child_value("score")) + extract_cnts("medal", offset, subdata.child_value("clear")) + extract_cnts("rate", offset, subdata.child_value("music_rate")) + + ret[music_id] = scores_by_chart return ret diff --git a/bemani/client/jubeat/prop.py b/bemani/client/jubeat/prop.py index e9caa29..47b52af 100644 --- a/bemani/client/jubeat/prop.py +++ b/bemani/client/jubeat/prop.py @@ -392,50 +392,48 @@ class JubeatPropClient(BaseClient): return self.__verify_profile(resp) def verify_gametop_get_mdata(self, jid: int) -> Dict[str, List[Dict[str, Any]]]: - call = self.call_node() - - # Construct node - gametop = Node.void("gametop") - call.add_child(gametop) - gametop.set_attribute("method", "get_mdata") - retry = Node.s32("retry", 0) - gametop.add_child(retry) - data = Node.void("data") - gametop.add_child(data) - player = Node.void("player") - data.add_child(player) - player.add_child(Node.s32("jid", jid)) - # Technically the game sends this same packet 3 times, one with - # each value 1, 2, 3 here. Unclear why, but we won't emulate it. - player.add_child(Node.s8("mdata_ver", 1)) - player.add_child(Node.bool("rival", False)) - - # Swap with server - resp = self.exchange("", call) - - # Parse out scores - self.assert_path(resp, "response/gametop/data/player/mdata_list") - ret = {} - for musicdata in resp.child("gametop/data/player/mdata_list").children: - if musicdata.name != "musicdata": - raise Exception("Unexpected node in playdata!") + for ver in [1, 2, 3]: + # Construct node + call = self.call_node() + gametop = Node.void("gametop") + call.add_child(gametop) + gametop.set_attribute("method", "get_mdata") + retry = Node.s32("retry", 0) + gametop.add_child(retry) + data = Node.void("data") + gametop.add_child(data) + player = Node.void("player") + data.add_child(player) + player.add_child(Node.s32("jid", jid)) + player.add_child(Node.s8("mdata_ver", ver)) + player.add_child(Node.bool("rival", False)) - music_id = musicdata.attribute("music_id") - scores_by_chart: List[Dict[str, int]] = [{}, {}, {}] + # Swap with server + resp = self.exchange("", call) - def extract_cnts(name: str, val: List[int]) -> None: - scores_by_chart[0][name] = val[0] - scores_by_chart[1][name] = val[1] - scores_by_chart[2][name] = val[2] + # Parse out scores + self.assert_path(resp, "response/gametop/data/player/mdata_list") - extract_cnts("plays", musicdata.child_value("play_cnt")) - extract_cnts("clears", musicdata.child_value("clear_cnt")) - extract_cnts("full_combos", musicdata.child_value("fc_cnt")) - extract_cnts("excellents", musicdata.child_value("ex_cnt")) - extract_cnts("score", musicdata.child_value("score")) - extract_cnts("medal", musicdata.child_value("clear")) - ret[music_id] = scores_by_chart + for musicdata in resp.child("gametop/data/player/mdata_list").children: + if musicdata.name != "musicdata": + raise Exception("Unexpected node in playdata!") + + music_id = musicdata.attribute("music_id") + scores_by_chart: List[Dict[str, int]] = [{}, {}, {}] + + def extract_cnts(name: str, val: List[int]) -> None: + scores_by_chart[0][name] = val[0] + scores_by_chart[1][name] = val[1] + scores_by_chart[2][name] = val[2] + + extract_cnts("plays", musicdata.child_value("play_cnt")) + extract_cnts("clears", musicdata.child_value("clear_cnt")) + extract_cnts("full_combos", musicdata.child_value("fc_cnt")) + extract_cnts("excellents", musicdata.child_value("ex_cnt")) + extract_cnts("score", musicdata.child_value("score")) + extract_cnts("medal", musicdata.child_value("clear")) + ret[music_id] = scores_by_chart return ret diff --git a/bemani/client/jubeat/qubell.py b/bemani/client/jubeat/qubell.py index 8a9484e..2509f3a 100644 --- a/bemani/client/jubeat/qubell.py +++ b/bemani/client/jubeat/qubell.py @@ -357,50 +357,48 @@ class JubeatQubellClient(BaseClient): return self.__verify_profile(resp) def verify_gametop_get_mdata(self, jid: int) -> Dict[str, List[Dict[str, Any]]]: - call = self.call_node() - - # Construct node - gametop = Node.void("gametop") - call.add_child(gametop) - gametop.set_attribute("method", "get_mdata") - retry = Node.s32("retry", 0) - gametop.add_child(retry) - data = Node.void("data") - gametop.add_child(data) - player = Node.void("player") - data.add_child(player) - player.add_child(Node.s32("jid", jid)) - # Technically the game sends this same packet 3 times, one with - # each value 1, 2, 3 here. Unclear why, but we won't emulate it. - player.add_child(Node.s8("mdata_ver", 1)) - player.add_child(Node.bool("rival", False)) - - # Swap with server - resp = self.exchange("", call) - - # Parse out scores - self.assert_path(resp, "response/gametop/data/player/mdata_list") - ret = {} - for musicdata in resp.child("gametop/data/player/mdata_list").children: - if musicdata.name != "musicdata": - raise Exception("Unexpected node in playdata!") + for ver in [1, 2, 3]: + # Construct node + call = self.call_node() + gametop = Node.void("gametop") + call.add_child(gametop) + gametop.set_attribute("method", "get_mdata") + retry = Node.s32("retry", 0) + gametop.add_child(retry) + data = Node.void("data") + gametop.add_child(data) + player = Node.void("player") + data.add_child(player) + player.add_child(Node.s32("jid", jid)) + player.add_child(Node.s8("mdata_ver", ver)) + player.add_child(Node.bool("rival", False)) - music_id = musicdata.attribute("music_id") - scores_by_chart: List[Dict[str, int]] = [{}, {}, {}] + # Swap with server + resp = self.exchange("", call) - def extract_cnts(name: str, val: List[int]) -> None: - scores_by_chart[0][name] = val[0] - scores_by_chart[1][name] = val[1] - scores_by_chart[2][name] = val[2] + # Parse out scores + self.assert_path(resp, "response/gametop/data/player/mdata_list") - extract_cnts("plays", musicdata.child_value("play_cnt")) - extract_cnts("clears", musicdata.child_value("clear_cnt")) - extract_cnts("full_combos", musicdata.child_value("fc_cnt")) - extract_cnts("excellents", musicdata.child_value("ex_cnt")) - extract_cnts("score", musicdata.child_value("score")) - extract_cnts("medal", musicdata.child_value("clear")) - ret[music_id] = scores_by_chart + for musicdata in resp.child("gametop/data/player/mdata_list").children: + if musicdata.name != "musicdata": + raise Exception("Unexpected node in playdata!") + + music_id = musicdata.attribute("music_id") + scores_by_chart: List[Dict[str, int]] = [{}, {}, {}] + + def extract_cnts(name: str, val: List[int]) -> None: + scores_by_chart[0][name] = val[0] + scores_by_chart[1][name] = val[1] + scores_by_chart[2][name] = val[2] + + extract_cnts("plays", musicdata.child_value("play_cnt")) + extract_cnts("clears", musicdata.child_value("clear_cnt")) + extract_cnts("full_combos", musicdata.child_value("fc_cnt")) + extract_cnts("excellents", musicdata.child_value("ex_cnt")) + extract_cnts("score", musicdata.child_value("score")) + extract_cnts("medal", musicdata.child_value("clear")) + ret[music_id] = scores_by_chart return ret diff --git a/bemani/client/jubeat/saucer.py b/bemani/client/jubeat/saucer.py index 090795a..8127959 100644 --- a/bemani/client/jubeat/saucer.py +++ b/bemani/client/jubeat/saucer.py @@ -275,49 +275,47 @@ class JubeatSaucerClient(BaseClient): return self.__verify_profile(resp) def verify_gametop_get_mdata(self, jid: int) -> Dict[str, List[Dict[str, Any]]]: - call = self.call_node() - - # Construct node - gametop = Node.void("gametop") - call.add_child(gametop) - gametop.set_attribute("method", "get_mdata") - retry = Node.s32("retry", 0) - gametop.add_child(retry) - data = Node.void("data") - gametop.add_child(data) - player = Node.void("player") - data.add_child(player) - player.add_child(Node.s32("jid", jid)) - # Technically the game sends this same packet 3 times, one with - # each value 1, 2, 3 here. Unclear why, but we won't emulate it. - player.add_child(Node.s8("mdata_ver", 1)) - - # Swap with server - resp = self.exchange("", call) - - # Parse out scores - self.assert_path(resp, "response/gametop/data/player/playdata") - ret = {} - for musicdata in resp.child("gametop/data/player/playdata").children: - if musicdata.name != "musicdata": - raise Exception("Unexpected node in playdata!") + for ver in [1, 2, 3]: + # Construct node + call = self.call_node() + gametop = Node.void("gametop") + call.add_child(gametop) + gametop.set_attribute("method", "get_mdata") + retry = Node.s32("retry", 0) + gametop.add_child(retry) + data = Node.void("data") + gametop.add_child(data) + player = Node.void("player") + data.add_child(player) + player.add_child(Node.s32("jid", jid)) + player.add_child(Node.s8("mdata_ver", ver)) - music_id = musicdata.attribute("music_id") - scores_by_chart: List[Dict[str, int]] = [{}, {}, {}] + # Swap with server + resp = self.exchange("", call) - def extract_cnts(name: str, val: List[int]) -> None: - scores_by_chart[0][name] = val[0] - scores_by_chart[1][name] = val[1] - scores_by_chart[2][name] = val[2] + # Parse out scores + self.assert_path(resp, "response/gametop/data/player/playdata") - extract_cnts("plays", musicdata.child_value("play_cnt")) - extract_cnts("clears", musicdata.child_value("clear_cnt")) - extract_cnts("full_combos", musicdata.child_value("fc_cnt")) - extract_cnts("excellents", musicdata.child_value("ex_cnt")) - extract_cnts("score", musicdata.child_value("score")) - extract_cnts("medal", musicdata.child_value("clear")) - ret[music_id] = scores_by_chart + for musicdata in resp.child("gametop/data/player/playdata").children: + if musicdata.name != "musicdata": + raise Exception("Unexpected node in playdata!") + + music_id = musicdata.attribute("music_id") + scores_by_chart: List[Dict[str, int]] = [{}, {}, {}] + + def extract_cnts(name: str, val: List[int]) -> None: + scores_by_chart[0][name] = val[0] + scores_by_chart[1][name] = val[1] + scores_by_chart[2][name] = val[2] + + extract_cnts("plays", musicdata.child_value("play_cnt")) + extract_cnts("clears", musicdata.child_value("clear_cnt")) + extract_cnts("full_combos", musicdata.child_value("fc_cnt")) + extract_cnts("excellents", musicdata.child_value("ex_cnt")) + extract_cnts("score", musicdata.child_value("score")) + extract_cnts("medal", musicdata.child_value("clear")) + ret[music_id] = scores_by_chart return ret diff --git a/bemani/client/jubeat/saucerfulfill.py b/bemani/client/jubeat/saucerfulfill.py index e3e37df..297e8a9 100644 --- a/bemani/client/jubeat/saucerfulfill.py +++ b/bemani/client/jubeat/saucerfulfill.py @@ -310,49 +310,47 @@ class JubeatSaucerFulfillClient(BaseClient): return self.__verify_profile(resp) def verify_gametop_get_mdata(self, jid: int) -> Dict[str, List[Dict[str, Any]]]: - call = self.call_node() - - # Construct node - gametop = Node.void("gametop") - call.add_child(gametop) - gametop.set_attribute("method", "get_mdata") - retry = Node.s32("retry", 0) - gametop.add_child(retry) - data = Node.void("data") - gametop.add_child(data) - player = Node.void("player") - data.add_child(player) - player.add_child(Node.s32("jid", jid)) - # Technically the game sends this same packet 3 times, one with - # each value 1, 2, 3 here. Unclear why, but we won't emulate it. - player.add_child(Node.s8("mdata_ver", 1)) - - # Swap with server - resp = self.exchange("", call) - - # Parse out scores - self.assert_path(resp, "response/gametop/data/player/playdata") - ret = {} - for musicdata in resp.child("gametop/data/player/playdata").children: - if musicdata.name != "musicdata": - raise Exception("Unexpected node in playdata!") + for ver in [1, 2, 3]: + # Construct node + call = self.call_node() + gametop = Node.void("gametop") + call.add_child(gametop) + gametop.set_attribute("method", "get_mdata") + retry = Node.s32("retry", 0) + gametop.add_child(retry) + data = Node.void("data") + gametop.add_child(data) + player = Node.void("player") + data.add_child(player) + player.add_child(Node.s32("jid", jid)) + player.add_child(Node.s8("mdata_ver", ver)) - music_id = musicdata.attribute("music_id") - scores_by_chart: List[Dict[str, int]] = [{}, {}, {}] + # Swap with server + resp = self.exchange("", call) - def extract_cnts(name: str, val: List[int]) -> None: - scores_by_chart[0][name] = val[0] - scores_by_chart[1][name] = val[1] - scores_by_chart[2][name] = val[2] + # Parse out scores + self.assert_path(resp, "response/gametop/data/player/playdata") - extract_cnts("plays", musicdata.child_value("play_cnt")) - extract_cnts("clears", musicdata.child_value("clear_cnt")) - extract_cnts("full_combos", musicdata.child_value("fc_cnt")) - extract_cnts("excellents", musicdata.child_value("ex_cnt")) - extract_cnts("score", musicdata.child_value("score")) - extract_cnts("medal", musicdata.child_value("clear")) - ret[music_id] = scores_by_chart + for musicdata in resp.child("gametop/data/player/playdata").children: + if musicdata.name != "musicdata": + raise Exception("Unexpected node in playdata!") + + music_id = musicdata.attribute("music_id") + scores_by_chart: List[Dict[str, int]] = [{}, {}, {}] + + def extract_cnts(name: str, val: List[int]) -> None: + scores_by_chart[0][name] = val[0] + scores_by_chart[1][name] = val[1] + scores_by_chart[2][name] = val[2] + + extract_cnts("plays", musicdata.child_value("play_cnt")) + extract_cnts("clears", musicdata.child_value("clear_cnt")) + extract_cnts("full_combos", musicdata.child_value("fc_cnt")) + extract_cnts("excellents", musicdata.child_value("ex_cnt")) + extract_cnts("score", musicdata.child_value("score")) + extract_cnts("medal", musicdata.child_value("clear")) + ret[music_id] = scores_by_chart return ret