CommonClient: revamp DataPackage handling

CommonClient.py

@@ -56,7 +56,7 @@ class ClientCommandProcessor(CommandProcessor):
         """List all received items"""
         logger.info(f'{len(self.ctx.items_received)} received items:')
         for index, item in enumerate(self.ctx.items_received, 1):
-            self.output(f"{self.ctx.item_name_getter(item.item)} from {self.ctx.player_names[item.player]}")
+            self.output(f"{self.ctx.item_names(item.item)} from {self.ctx.player_names[item.player]}")
         return True
 
     def _cmd_missing(self) -> bool:
@@ -120,6 +120,11 @@ class CommonContext:
     game: typing.Optional[str] = None
     items_handling: typing.Optional[int] = None
 
+    # datapackage
+    # Contents in flux until connection to server is made, to download correct data for this multiworld.
+    item_names: typing.Dict[int, str] = Utils.KeyedDefaultDict(lambda code: f'Unknown item (ID:{code})')
+    location_names: typing.Dict[int, str] = Utils.KeyedDefaultDict(lambda code: f'Unknown location (ID:{code})')
+
     # defaults
     starting_reconnect_delay: int = 5
     current_reconnect_delay: int = starting_reconnect_delay
@@ -140,7 +145,6 @@ class CommonContext:
     server_address: str
     password: typing.Optional[str]
     hint_cost: typing.Optional[int]
-    games: typing.Dict[int, str]
     player_names: typing.Dict[int, str]
 
     # locations
@@ -150,16 +154,15 @@ class CommonContext:
     checked_locations: typing.Set[int]  # server state
     locations_info: typing.Dict[int, NetworkItem]
 
+    # internals
     # current message box through kvui
     _messagebox = None
 
-
     def __init__(self, server_address, password):
         # server state
         self.server_address = server_address
         self.password = password
         self.hint_cost = None
-        self.games = {}
         self.slot_info = {}
         self.permissions = {
             "forfeit": "disabled",
@@ -191,7 +194,7 @@ class CommonContext:
         self.watcher_event = asyncio.Event()
 
         self.jsontotextparser = JSONtoTextParser(self)
-        self.set_getters(network_data_package)
+        self.update_datapackage(network_data_package)
 
         # execution
         self.keep_alive_task = asyncio.create_task(keep_alive(self), name="Bouncy")
@@ -216,7 +219,6 @@ class CommonContext:
         self.server_version = Version(0, 0, 0)
         self.server = None
         self.server_task = None
-        self.games = {}
         self.hint_cost = None
         self.permissions = {
             "forfeit": "disabled",
@@ -224,35 +226,6 @@ class CommonContext:
             "remaining": "disabled",
         }
 
-    # noinspection PyAttributeOutsideInit
-    def set_getters(self, data_package: dict, network=False):
-        if not network:  # local data; check if newer data was already downloaded
-            local_package = Utils.persistent_load().get("datapackage", {}).get("latest", {})
-            if local_package and local_package["version"] > network_data_package["version"]:
-                data_package: dict = local_package
-        elif network:  # check if data from server is newer
-
-            if data_package["version"] > network_data_package["version"]:
-                Utils.persistent_store("datapackage", "latest", network_data_package)
-
-        item_lookup: dict = {}
-        locations_lookup: dict = {}
-        for game, gamedata in data_package["games"].items():
-            for item_name, item_id in gamedata["item_name_to_id"].items():
-                item_lookup[item_id] = item_name
-            for location_name, location_id in gamedata["location_name_to_id"].items():
-                locations_lookup[location_id] = location_name
-
-        def get_item_name_from_id(code: int) -> str:
-            return item_lookup.get(code, f'Unknown item (ID:{code})')
-
-        self.item_name_getter = get_item_name_from_id
-
-        def get_location_name_from_address(address: int) -> str:
-            return locations_lookup.get(address, f'Unknown location (ID:{address})')
-
-        self.location_name_getter = get_location_name_from_address
-
     async def disconnect(self):
         if self.server and not self.server.socket.closed:
             await self.server.socket.close()
@@ -335,7 +308,7 @@ class CommonContext:
                 logger.exception(e)
 
     async def shutdown(self):
-        self.server_address = None
+        self.server_address = ""
         if self.server and not self.server.socket.closed:
             await self.server.socket.close()
         if self.server_task:
@@ -350,6 +323,47 @@ class CommonContext:
         if self.input_task:
             self.input_task.cancel()
 
+    # DataPackage
+    async def prepare_datapackage(self, relevant_games: typing.Set[str],
+                                  remote_datepackage_versions: typing.Dict[str, int]):
+        """Validate that all data is present for the current multiworld.
+        Download, assimilate and cache missing data from the server."""
+        cache_package = Utils.persistent_load().get("datapackage", {}).get("games", {})
+        needed_updates: typing.Set[str] = set()
+        for game in relevant_games:
+            remote_version: int = remote_datepackage_versions[game]
+
+            if remote_version == 0:  # custom datapackage for this game
+                needed_updates.add(game)
+                continue
+            local_version: int = network_data_package["games"].get(game, {}).get("version", 0)
+            # no action required if local version is new enough
+            if remote_version > local_version:
+                cache_version: int = cache_package.get(game, {}).get("version", 0)
+                # download remote version if cache is not new enough
+                if remote_version > cache_version:
+                    needed_updates.add(game)
+                else:
+                    self.update_game(cache_package[game])
+        if needed_updates:
+            await self.send_msgs([{"cmd": "GetDataPackage", "games": list(needed_updates)}])
+
+    def update_game(self, game_package: dict):
+        for item_name, item_id in game_package["item_name_to_id"].items():
+            self.item_names[item_id] = item_name
+        for location_name, location_id in game_package["location_name_to_id"].items():
+            self.location_names[location_id] = location_name
+
+    def update_datapackage(self, data_package: dict):
+        for game, gamedata in data_package["games"].items():
+            self.update_game(gamedata)
+
+    def consume_network_datapackage(self, data_package: dict):
+        self.update_datapackage(data_package)
+        current_cache = Utils.persistent_load().get("datapackage", {}).get("games", {})
+        current_cache.update(data_package["games"])
+        Utils.persistent_store("datapackage", "games", current_cache)
+
     # DeathLink hooks
 
     def on_deathlink(self, data: dict):
@@ -520,8 +534,6 @@ async def process_server_cmd(ctx: CommonContext, args: dict):
             if args['password']:
                 logger.info('Password required')
             ctx.update_permissions(args.get("permissions", {}))
-            if "games" in args:
-                ctx.games = {x: game for x, game in enumerate(args["games"], start=1)}
             logger.info(
                 f"A !hint costs {args['hint_cost']}% of your total location count as points"
                 f" and you get {args['location_check_points']}"
@@ -540,13 +552,14 @@ async def process_server_cmd(ctx: CommonContext, args: dict):
                         logger.info(f'  Team #{network_player.team + 1}')
                         current_team = network_player.team
                     logger.info('    %s (Player %d)' % (network_player.alias, network_player.slot))
-            if args["datapackage_version"] > network_data_package["version"] or args["datapackage_version"] == 0:
+            # update datapackage
-                await ctx.send_msgs([{"cmd": "GetDataPackage"}])
+            await ctx.prepare_datapackage(set(args["games"]), args["datapackage_versions"])
+
             await ctx.server_auth(args['password'])
 
     elif cmd == 'DataPackage':
-        logger.info("Got new ID/Name Datapackage")
+        logger.info("Got new ID/Name DataPackage")
-        ctx.set_getters(args['data'], network=True)
+        ctx.consume_network_datapackage(args['data'])
 
     elif cmd == 'ConnectionRefused':
         errors = args["errors"]
@@ -700,7 +713,7 @@ if __name__ == '__main__':
 
         def on_package(self, cmd: str, args: dict):
             if cmd == "Connected":
-                self.game = self.games.get(self.slot, None)
+                self.game = self.slot_info[self.slot].game
 
 
     async def main(args):

FF1Client.py

@@ -39,6 +39,7 @@ class FF1CommandProcessor(ClientCommandProcessor):
 
 class FF1Context(CommonContext):
     command_processor = FF1CommandProcessor
+    game = 'Final Fantasy'
     items_handling = 0b111  # full remote
 
     def __init__(self, server_address, password):
@@ -48,7 +49,6 @@ class FF1Context(CommonContext):
         self.messages = {}
         self.locations_array = None
         self.nes_status = CONNECTION_INITIAL_STATUS
-        self.game = 'Final Fantasy'
         self.awaiting_rom = False
         self.display_msgs = True
 
@@ -68,14 +68,13 @@ class FF1Context(CommonContext):
 
     def on_package(self, cmd: str, args: dict):
         if cmd == 'Connected':
-            self.game = self.games.get(self.slot, None)
             asyncio.create_task(parse_locations(self.locations_array, self, True))
         elif cmd == 'Print':
             msg = args['text']
             if ': !' not in msg:
                 self._set_message(msg, SYSTEM_MESSAGE_ID)
         elif cmd == "ReceivedItems":
-            msg = f"Received {', '.join([self.item_name_getter(item.item) for item in args['items']])}"
+            msg = f"Received {', '.join([self.item_names[item.item] for item in args['items']])}"
             self._set_message(msg, SYSTEM_MESSAGE_ID)
         elif cmd == 'PrintJSON':
             print_type = args['type']
@@ -85,20 +84,20 @@ class FF1Context(CommonContext):
             sending_player_id = item.player
             sending_player_name = self.player_names[item.player]
             if print_type == 'Hint':
-                msg = f"Hint: Your {self.item_name_getter(item.item)} is at" \
+                msg = f"Hint: Your {self.item_names[item.item]} is at" \
-                      f" {self.player_names[item.player]}'s {self.location_name_getter(item.location)}"
+                      f" {self.player_names[item.player]}'s {self.location_names[item.location]}"
                 self._set_message(msg, item.item)
             elif print_type == 'ItemSend' and receiving_player_id != self.slot:
                 if sending_player_id == self.slot:
                     if receiving_player_id == self.slot:
-                        msg = f"You found your own {self.item_name_getter(item.item)}"
+                        msg = f"You found your own {self.item_names[item.item]}"
                     else:
-                        msg = f"You sent {self.item_name_getter(item.item)} to {receiving_player_name}"
+                        msg = f"You sent {self.item_names[item.item]} to {receiving_player_name}"
                 else:
                     if receiving_player_id == sending_player_id:
-                        msg = f"{sending_player_name} found their {self.item_name_getter(item.item)}"
+                        msg = f"{sending_player_name} found their {self.item_names[item.item]}"
                     else:
-                        msg = f"{sending_player_name} sent {self.item_name_getter(item.item)} to " \
+                        msg = f"{sending_player_name} sent {self.item_names[item.item]} to " \
                               f"{receiving_player_name}"
                 self._set_message(msg, item.item)
 
@@ -151,13 +150,13 @@ async def parse_locations(locations_array: List[int], ctx: FF1Context, force: bo
                 index -= 0x200
                 flag = 0x02
 
-            # print(f"Location: {ctx.location_name_getter(location)}")
+            # print(f"Location: {ctx.location_names[location]}")
             # print(f"Index: {str(hex(index))}")
             # print(f"value: {locations_array[index] & flag != 0}")
             if locations_array[index] & flag != 0:
                 locations_checked.append(location)
         if locations_checked:
-            # print([ctx.location_name_getter(location) for location in locations_checked])
+            # print([ctx.location_names[location] for location in locations_checked])
             await ctx.send_msgs([
                 {"cmd": "LocationChecks",
                  "locations": locations_checked}

MultiServer.py

@@ -631,8 +631,7 @@ async def on_client_connected(ctx: Context, client: Client):
         'cmd': 'RoomInfo',
         'password': bool(ctx.password),
         'players': players,
-        # TODO remove around 0.2.5 in favor of slot_info ?
+        # TODO convert to list of games present in 0.4
-        #  Maybe convert into a list of games that are present to fetch relevant datapackage entries before Connect?
         'games': [ctx.games[x] for x in range(1, len(ctx.games) + 1)],
         # tags are for additional features in the communication.
         # Name them by feature or fork, as you feel is appropriate.

NetUtils.py

@@ -245,7 +245,7 @@ class JSONtoTextParser(metaclass=HandlerMeta):
 
     def _handle_item_id(self, node: JSONMessagePart):
         item_id = int(node["text"])
-        node["text"] = self.ctx.item_name_getter(item_id)
+        node["text"] = self.ctx.item_names[item_id]
         return self._handle_item_name(node)
 
     def _handle_location_name(self, node: JSONMessagePart):
@@ -254,7 +254,7 @@ class JSONtoTextParser(metaclass=HandlerMeta):
 
     def _handle_location_id(self, node: JSONMessagePart):
         item_id = int(node["text"])
-        node["text"] = self.ctx.location_name_getter(item_id)
+        node["text"] = self.ctx.location_names[item_id]
         return self._handle_location_name(node)
 
     def _handle_entrance_name(self, node: JSONMessagePart):

SNIClient.py

@@ -893,7 +893,7 @@ async def track_locations(ctx: Context, roomid, roomdata):
     def new_check(location_id):
         new_locations.append(location_id)
         ctx.locations_checked.add(location_id)
-        location = ctx.location_name_getter(location_id)
+        location = ctx.location_names[location_id]
         snes_logger.info(
             f'New Check: {location} ({len(ctx.locations_checked)}/{len(ctx.missing_locations) + len(ctx.checked_locations)})')
 
@@ -1126,9 +1126,9 @@ async def game_watcher(ctx: Context):
                 item = ctx.items_received[recv_index]
                 recv_index += 1
                 logging.info('Received %s from %s (%s) (%d/%d in list)' % (
-                    color(ctx.item_name_getter(item.item), 'red', 'bold'),
+                    color(ctx.item_names[item.item], 'red', 'bold'),
                     color(ctx.player_names[item.player], 'yellow'),
-                    ctx.location_name_getter(item.location), recv_index, len(ctx.items_received)))
+                    ctx.location_names[item.location], recv_index, len(ctx.items_received)))
 
                 snes_buffered_write(ctx, RECV_PROGRESS_ADDR,
                                     bytes([recv_index & 0xFF, (recv_index >> 8) & 0xFF]))
@@ -1183,7 +1183,7 @@ async def game_watcher(ctx: Context):
                 location_id = locations_start_id + itemIndex
 
                 ctx.locations_checked.add(location_id)
-                location = ctx.location_name_getter(location_id)
+                location = ctx.location_names[location_id]
                 snes_logger.info(
                     f'New Check: {location} ({len(ctx.locations_checked)}/{len(ctx.missing_locations) + len(ctx.checked_locations)})')
                 await ctx.send_msgs([{"cmd": 'LocationChecks', "locations": [location_id]}])
@@ -1212,9 +1212,9 @@ async def game_watcher(ctx: Context):
                 snes_buffered_write(ctx, SM_RECV_PROGRESS_ADDR + 0x602,
                                     bytes([itemOutPtr & 0xFF, (itemOutPtr >> 8) & 0xFF]))
                 logging.info('Received %s from %s (%s) (%d/%d in list)' % (
-                    color(ctx.item_name_getter(item.item), 'red', 'bold'),
+                    color(ctx.item_names[item.item], 'red', 'bold'),
                     color(ctx.player_names[item.player], 'yellow'),
-                    ctx.location_name_getter(item.location), itemOutPtr, len(ctx.items_received)))
+                    ctx.location_names[item.location], itemOutPtr, len(ctx.items_received)))
             await snes_flush_writes(ctx)
         elif ctx.game == GAME_SMZ3:
             currentGame = await snes_read(ctx, SRAM_START + 0x33FE, 2)
@@ -1255,7 +1255,7 @@ async def game_watcher(ctx: Context):
                 location_id = locations_start_id + itemIndex
 
                 ctx.locations_checked.add(location_id)
-                location = ctx.location_name_getter(location_id)
+                location = ctx.location_names[location_id]
                 snes_logger.info(f'New Check: {location} ({len(ctx.locations_checked)}/{len(ctx.missing_locations) + len(ctx.checked_locations)})')
                 await ctx.send_msgs([{"cmd": 'LocationChecks', "locations": [location_id]}])
 
@@ -1276,8 +1276,8 @@ async def game_watcher(ctx: Context):
                 itemOutPtr += 1
                 snes_buffered_write(ctx, SMZ3_RECV_PROGRESS_ADDR + 0x602, bytes([itemOutPtr & 0xFF, (itemOutPtr >> 8) & 0xFF]))
                 logging.info('Received %s from %s (%s) (%d/%d in list)' % (
-                    color(ctx.item_name_getter(item.item), 'red', 'bold'), color(ctx.player_names[item.player], 'yellow'),
+                    color(ctx.item_names[item.item], 'red', 'bold'), color(ctx.player_names[item.player], 'yellow'),
-                    ctx.location_name_getter(item.location), itemOutPtr, len(ctx.items_received)))
+                    ctx.location_names[item.location], itemOutPtr, len(ctx.items_received)))
             await snes_flush_writes(ctx)
 
 

Starcraft2Client.py

@@ -592,8 +592,8 @@ def calc_objectives_completed(mission, missions_info, locations_done, unfinished
             if (missions_info[mission].id * 100 + SC2WOL_LOC_ID_OFFSET + i) in locations_done:
                 objectives_complete += 1
             else:
-                unfinished_locations[mission].append(ctx.location_name_getter(
+                unfinished_locations[mission].append(ctx.location_names[
-                    missions_info[mission].id * 100 + SC2WOL_LOC_ID_OFFSET + i))
+                    missions_info[mission].id * 100 + SC2WOL_LOC_ID_OFFSET + i])
 
         return objectives_complete
 

docs/network protocol.md

@@ -64,7 +64,7 @@ Sent to clients when they connect to an Archipelago server.
 | hint_cost | int | The amount of points it costs to receive a hint from the server. |
 | location_check_points | int | The amount of hint points you receive per item/location check completed. ||
 | players | list\[[NetworkPlayer](#NetworkPlayer)\] | Sent only if the client is properly authenticated (see [Archipelago Connection Handshake](#Archipelago-Connection-Handshake)). Information on the players currently connected to the server. |
-| games | list\[str\] | sorted list of game names for the players, so first player's game will be games\[0\]. Matches game names in datapackage. |
+| games | list\[str\] | List of games present in this multiworld. |
 | datapackage_version | int | Sum of individual games' datapackage version. Deprecated. Use `datapackage_versions` instead. |
 | datapackage_versions | dict\[str, int\] | Data versions of the individual games' data packages the server will send. Used to decide which games' caches are outdated. See [Data Package Contents](#Data-Package-Contents). |
 | seed_name | str | uniquely identifying name of this generation |