Use cache layer to properly shard Jubeat music data responses.

bemani/backend/jubeat/base.py

@@ -176,14 +176,43 @@ class JubeatBase(CoreHandler, CardManagerHandler, PASELIHandler, Base):
         if profile is None:
             return None
 
-        if partition != 1:
+        cache_key = f"get_scores_by_extid-{extid}"
-            scores = []
+        score: Optional[List[Score]]
-        else:
+
+        if partition == 1:
+            # We fetch all scores on the first partition and then divy up
+            # the scores across total_partitions fetches. If it is small
+            # enough, we don't bother.
             scores = self.data.remote.music.get_scores(
                 self.game, self.music_version, userid
             )
-            if scores is None:
+        else:
-                return None
+            # We will want to fetch the remaining scores that were in our
+            # cache.
+            scores = self.cache.get(cache_key)  # type: ignore
+
+        if len(scores) < 50:
+            # We simply return the whole amount for this, and cache nothing.
+            rest = []
+        else:
+            groups = (total_partitions - partition) + 1
+            pivot = len(scores) // groups
+
+            rest = scores[pivot:]
+            scores = scores[:pivot]
+
+        # Cache the rest of the scores for next iteration, unless we're on the
+        # last iteration.
+        if partition == total_partitions:
+            if rest:
+                raise Exception(
+                    "Logic error, should not have gotten additional scores to cache on last iteration!"
+                )
+            self.cache.delete(cache_key)
+        else:
+            self.cache.set(cache_key, rest, timeout=60)
+
+        # Format the chunk of scores we have to send back to the client.
         return self.format_scores(userid, profile, scores)
 
     def update_score(

bemani/client/jubeat/clan.py

@@ -478,9 +478,10 @@ class JubeatClanClient(BaseClient):
         return self.__verify_profile(resp)
 
     def verify_gametop_get_mdata(self, jid: int) -> Dict[str, List[Dict[str, Any]]]:
-        call = self.call_node()
+        ret = {}
-
+        for ver in [1, 2, 3]:
             # Construct node
+            call = self.call_node()
             gametop = Node.void("gametop")
             call.add_child(gametop)
             gametop.set_attribute("method", "get_mdata")
@@ -491,9 +492,7 @@ class JubeatClanClient(BaseClient):
             player = Node.void("player")
             data.add_child(player)
             player.add_child(Node.s32("jid", jid))
-        # Technically the game sends this same packet 3 times, one with
+            player.add_child(Node.s8("mdata_ver", ver))
-        # each value 1, 2, 3 here. Unclear why, but we won't emulate it.
-        player.add_child(Node.s8("mdata_ver", 1))
             player.add_child(Node.bool("rival", False))
 
             # Swap with server
@@ -502,7 +501,6 @@ class JubeatClanClient(BaseClient):
             # Parse out scores
             self.assert_path(resp, "response/gametop/data/player/mdata_list")
 
-        ret = {}
             for musicdata in resp.child("gametop/data/player/mdata_list").children:
                 if musicdata.name != "musicdata":
                     raise Exception("Unexpected node in playdata!")

bemani/client/jubeat/festo.py

@@ -582,9 +582,10 @@ class JubeatFestoClient(BaseClient):
         return self.__verify_profile(resp, False)
 
     def verify_gametop_get_mdata(self, jid: int) -> Dict[str, List[Dict[str, Any]]]:
-        call = self.call_node()
+        ret = {}
-
+        for ver in [1, 2, 3]:
             # Construct node
+            call = self.call_node()
             gametop = Node.void("gametop")
             call.add_child(gametop)
             gametop.set_attribute("method", "get_mdata")
@@ -595,12 +596,7 @@ class JubeatFestoClient(BaseClient):
             player = Node.void("player")
             data.add_child(player)
             player.add_child(Node.s32("jid", jid))
-        # Technically the game sends this same packet 3 times, one with
+            player.add_child(Node.s8("mdata_ver", ver))
-        # each value 1, 2, 3 here. This is for sharding across 3 requests,
-        # and the game will combine all 3 responses. Its up to the server to
-        # handle this the way it wants, and we just send everything back in the
-        # first request and ignore the rest.
-        player.add_child(Node.s8("mdata_ver", 1))
             player.add_child(Node.bool("rival", False))
 
             # Swap with server
@@ -612,7 +608,6 @@ class JubeatFestoClient(BaseClient):
             if resp.child_value("gametop/data/player/jid") != jid:
                 raise Exception("Unexpected jid received from server!")
 
-        ret = {}
             for musicdata in resp.child("gametop/data/player/mdata_list").children:
                 if musicdata.name != "musicdata":
                     raise Exception("Unexpected node in playdata!")

bemani/client/jubeat/prop.py

@@ -392,9 +392,10 @@ class JubeatPropClient(BaseClient):
         return self.__verify_profile(resp)
 
     def verify_gametop_get_mdata(self, jid: int) -> Dict[str, List[Dict[str, Any]]]:
-        call = self.call_node()
+        ret = {}
-
+        for ver in [1, 2, 3]:
             # Construct node
+            call = self.call_node()
             gametop = Node.void("gametop")
             call.add_child(gametop)
             gametop.set_attribute("method", "get_mdata")
@@ -405,9 +406,7 @@ class JubeatPropClient(BaseClient):
             player = Node.void("player")
             data.add_child(player)
             player.add_child(Node.s32("jid", jid))
-        # Technically the game sends this same packet 3 times, one with
+            player.add_child(Node.s8("mdata_ver", ver))
-        # each value 1, 2, 3 here. Unclear why, but we won't emulate it.
-        player.add_child(Node.s8("mdata_ver", 1))
             player.add_child(Node.bool("rival", False))
 
             # Swap with server
@@ -416,7 +415,6 @@ class JubeatPropClient(BaseClient):
             # Parse out scores
             self.assert_path(resp, "response/gametop/data/player/mdata_list")
 
-        ret = {}
             for musicdata in resp.child("gametop/data/player/mdata_list").children:
                 if musicdata.name != "musicdata":
                     raise Exception("Unexpected node in playdata!")

bemani/client/jubeat/qubell.py

@@ -357,9 +357,10 @@ class JubeatQubellClient(BaseClient):
         return self.__verify_profile(resp)
 
     def verify_gametop_get_mdata(self, jid: int) -> Dict[str, List[Dict[str, Any]]]:
-        call = self.call_node()
+        ret = {}
-
+        for ver in [1, 2, 3]:
             # Construct node
+            call = self.call_node()
             gametop = Node.void("gametop")
             call.add_child(gametop)
             gametop.set_attribute("method", "get_mdata")
@@ -370,9 +371,7 @@ class JubeatQubellClient(BaseClient):
             player = Node.void("player")
             data.add_child(player)
             player.add_child(Node.s32("jid", jid))
-        # Technically the game sends this same packet 3 times, one with
+            player.add_child(Node.s8("mdata_ver", ver))
-        # each value 1, 2, 3 here. Unclear why, but we won't emulate it.
-        player.add_child(Node.s8("mdata_ver", 1))
             player.add_child(Node.bool("rival", False))
 
             # Swap with server
@@ -381,7 +380,6 @@ class JubeatQubellClient(BaseClient):
             # Parse out scores
             self.assert_path(resp, "response/gametop/data/player/mdata_list")
 
-        ret = {}
             for musicdata in resp.child("gametop/data/player/mdata_list").children:
                 if musicdata.name != "musicdata":
                     raise Exception("Unexpected node in playdata!")

bemani/client/jubeat/saucer.py

@@ -275,9 +275,10 @@ class JubeatSaucerClient(BaseClient):
         return self.__verify_profile(resp)
 
     def verify_gametop_get_mdata(self, jid: int) -> Dict[str, List[Dict[str, Any]]]:
-        call = self.call_node()
+        ret = {}
-
+        for ver in [1, 2, 3]:
             # Construct node
+            call = self.call_node()
             gametop = Node.void("gametop")
             call.add_child(gametop)
             gametop.set_attribute("method", "get_mdata")
@@ -288,9 +289,7 @@ class JubeatSaucerClient(BaseClient):
             player = Node.void("player")
             data.add_child(player)
             player.add_child(Node.s32("jid", jid))
-        # Technically the game sends this same packet 3 times, one with
+            player.add_child(Node.s8("mdata_ver", ver))
-        # each value 1, 2, 3 here. Unclear why, but we won't emulate it.
-        player.add_child(Node.s8("mdata_ver", 1))
 
             # Swap with server
             resp = self.exchange("", call)
@@ -298,7 +297,6 @@ class JubeatSaucerClient(BaseClient):
             # Parse out scores
             self.assert_path(resp, "response/gametop/data/player/playdata")
 
-        ret = {}
             for musicdata in resp.child("gametop/data/player/playdata").children:
                 if musicdata.name != "musicdata":
                     raise Exception("Unexpected node in playdata!")

bemani/client/jubeat/saucerfulfill.py

@@ -310,9 +310,10 @@ class JubeatSaucerFulfillClient(BaseClient):
         return self.__verify_profile(resp)
 
     def verify_gametop_get_mdata(self, jid: int) -> Dict[str, List[Dict[str, Any]]]:
-        call = self.call_node()
+        ret = {}
-
+        for ver in [1, 2, 3]:
             # Construct node
+            call = self.call_node()
             gametop = Node.void("gametop")
             call.add_child(gametop)
             gametop.set_attribute("method", "get_mdata")
@@ -323,9 +324,7 @@ class JubeatSaucerFulfillClient(BaseClient):
             player = Node.void("player")
             data.add_child(player)
             player.add_child(Node.s32("jid", jid))
-        # Technically the game sends this same packet 3 times, one with
+            player.add_child(Node.s8("mdata_ver", ver))
-        # each value 1, 2, 3 here. Unclear why, but we won't emulate it.
-        player.add_child(Node.s8("mdata_ver", 1))
 
             # Swap with server
             resp = self.exchange("", call)
@@ -333,7 +332,6 @@ class JubeatSaucerFulfillClient(BaseClient):
             # Parse out scores
             self.assert_path(resp, "response/gametop/data/player/playdata")
 
-        ret = {}
             for musicdata in resp.child("gametop/data/player/playdata").children:
                 if musicdata.name != "musicdata":
                     raise Exception("Unexpected node in playdata!")