From d81dbbd95150844784c2c46509a2c952ad65609b Mon Sep 17 00:00:00 2001 From: Fabian Dill Date: Thu, 9 Jun 2022 12:54:03 +0200 Subject: [PATCH] CommonClient: revamp DataPackage handling --- CommonClient.py | 101 ++++++++++++++++++++++----------------- FF1Client.py | 21 ++++---- MultiServer.py | 3 +- NetUtils.py | 4 +- SNIClient.py | 18 +++---- Starcraft2Client.py | 4 +- docs/network protocol.md | 2 +- 7 files changed, 82 insertions(+), 71 deletions(-) diff --git a/CommonClient.py b/CommonClient.py index 5fb42971..ec774d09 100644 --- a/CommonClient.py +++ b/CommonClient.py @@ -56,7 +56,7 @@ class ClientCommandProcessor(CommandProcessor): """List all received items""" logger.info(f'{len(self.ctx.items_received)} received items:') for index, item in enumerate(self.ctx.items_received, 1): - self.output(f"{self.ctx.item_name_getter(item.item)} from {self.ctx.player_names[item.player]}") + self.output(f"{self.ctx.item_names(item.item)} from {self.ctx.player_names[item.player]}") return True def _cmd_missing(self) -> bool: @@ -120,6 +120,11 @@ class CommonContext: game: typing.Optional[str] = None items_handling: typing.Optional[int] = None + # datapackage + # Contents in flux until connection to server is made, to download correct data for this multiworld. + item_names: typing.Dict[int, str] = Utils.KeyedDefaultDict(lambda code: f'Unknown item (ID:{code})') + location_names: typing.Dict[int, str] = Utils.KeyedDefaultDict(lambda code: f'Unknown location (ID:{code})') + # defaults starting_reconnect_delay: int = 5 current_reconnect_delay: int = starting_reconnect_delay @@ -140,7 +145,6 @@ class CommonContext: server_address: str password: typing.Optional[str] hint_cost: typing.Optional[int] - games: typing.Dict[int, str] player_names: typing.Dict[int, str] # locations @@ -149,17 +153,16 @@ class CommonContext: missing_locations: typing.Set[int] checked_locations: typing.Set[int] # server state locations_info: typing.Dict[int, NetworkItem] - + + # internals # current message box through kvui _messagebox = None - def __init__(self, server_address, password): # server state self.server_address = server_address self.password = password self.hint_cost = None - self.games = {} self.slot_info = {} self.permissions = { "forfeit": "disabled", @@ -191,7 +194,7 @@ class CommonContext: self.watcher_event = asyncio.Event() self.jsontotextparser = JSONtoTextParser(self) - self.set_getters(network_data_package) + self.update_datapackage(network_data_package) # execution self.keep_alive_task = asyncio.create_task(keep_alive(self), name="Bouncy") @@ -216,7 +219,6 @@ class CommonContext: self.server_version = Version(0, 0, 0) self.server = None self.server_task = None - self.games = {} self.hint_cost = None self.permissions = { "forfeit": "disabled", @@ -224,35 +226,6 @@ class CommonContext: "remaining": "disabled", } - # noinspection PyAttributeOutsideInit - def set_getters(self, data_package: dict, network=False): - if not network: # local data; check if newer data was already downloaded - local_package = Utils.persistent_load().get("datapackage", {}).get("latest", {}) - if local_package and local_package["version"] > network_data_package["version"]: - data_package: dict = local_package - elif network: # check if data from server is newer - - if data_package["version"] > network_data_package["version"]: - Utils.persistent_store("datapackage", "latest", network_data_package) - - item_lookup: dict = {} - locations_lookup: dict = {} - for game, gamedata in data_package["games"].items(): - for item_name, item_id in gamedata["item_name_to_id"].items(): - item_lookup[item_id] = item_name - for location_name, location_id in gamedata["location_name_to_id"].items(): - locations_lookup[location_id] = location_name - - def get_item_name_from_id(code: int) -> str: - return item_lookup.get(code, f'Unknown item (ID:{code})') - - self.item_name_getter = get_item_name_from_id - - def get_location_name_from_address(address: int) -> str: - return locations_lookup.get(address, f'Unknown location (ID:{address})') - - self.location_name_getter = get_location_name_from_address - async def disconnect(self): if self.server and not self.server.socket.closed: await self.server.socket.close() @@ -335,7 +308,7 @@ class CommonContext: logger.exception(e) async def shutdown(self): - self.server_address = None + self.server_address = "" if self.server and not self.server.socket.closed: await self.server.socket.close() if self.server_task: @@ -350,6 +323,47 @@ class CommonContext: if self.input_task: self.input_task.cancel() + # DataPackage + async def prepare_datapackage(self, relevant_games: typing.Set[str], + remote_datepackage_versions: typing.Dict[str, int]): + """Validate that all data is present for the current multiworld. + Download, assimilate and cache missing data from the server.""" + cache_package = Utils.persistent_load().get("datapackage", {}).get("games", {}) + needed_updates: typing.Set[str] = set() + for game in relevant_games: + remote_version: int = remote_datepackage_versions[game] + + if remote_version == 0: # custom datapackage for this game + needed_updates.add(game) + continue + local_version: int = network_data_package["games"].get(game, {}).get("version", 0) + # no action required if local version is new enough + if remote_version > local_version: + cache_version: int = cache_package.get(game, {}).get("version", 0) + # download remote version if cache is not new enough + if remote_version > cache_version: + needed_updates.add(game) + else: + self.update_game(cache_package[game]) + if needed_updates: + await self.send_msgs([{"cmd": "GetDataPackage", "games": list(needed_updates)}]) + + def update_game(self, game_package: dict): + for item_name, item_id in game_package["item_name_to_id"].items(): + self.item_names[item_id] = item_name + for location_name, location_id in game_package["location_name_to_id"].items(): + self.location_names[location_id] = location_name + + def update_datapackage(self, data_package: dict): + for game, gamedata in data_package["games"].items(): + self.update_game(gamedata) + + def consume_network_datapackage(self, data_package: dict): + self.update_datapackage(data_package) + current_cache = Utils.persistent_load().get("datapackage", {}).get("games", {}) + current_cache.update(data_package["games"]) + Utils.persistent_store("datapackage", "games", current_cache) + # DeathLink hooks def on_deathlink(self, data: dict): @@ -520,8 +534,6 @@ async def process_server_cmd(ctx: CommonContext, args: dict): if args['password']: logger.info('Password required') ctx.update_permissions(args.get("permissions", {})) - if "games" in args: - ctx.games = {x: game for x, game in enumerate(args["games"], start=1)} logger.info( f"A !hint costs {args['hint_cost']}% of your total location count as points" f" and you get {args['location_check_points']}" @@ -540,13 +552,14 @@ async def process_server_cmd(ctx: CommonContext, args: dict): logger.info(f' Team #{network_player.team + 1}') current_team = network_player.team logger.info(' %s (Player %d)' % (network_player.alias, network_player.slot)) - if args["datapackage_version"] > network_data_package["version"] or args["datapackage_version"] == 0: - await ctx.send_msgs([{"cmd": "GetDataPackage"}]) + # update datapackage + await ctx.prepare_datapackage(set(args["games"]), args["datapackage_versions"]) + await ctx.server_auth(args['password']) elif cmd == 'DataPackage': - logger.info("Got new ID/Name Datapackage") - ctx.set_getters(args['data'], network=True) + logger.info("Got new ID/Name DataPackage") + ctx.consume_network_datapackage(args['data']) elif cmd == 'ConnectionRefused': errors = args["errors"] @@ -700,7 +713,7 @@ if __name__ == '__main__': def on_package(self, cmd: str, args: dict): if cmd == "Connected": - self.game = self.games.get(self.slot, None) + self.game = self.slot_info[self.slot].game async def main(args): diff --git a/FF1Client.py b/FF1Client.py index f6bd0829..c280fa30 100644 --- a/FF1Client.py +++ b/FF1Client.py @@ -39,6 +39,7 @@ class FF1CommandProcessor(ClientCommandProcessor): class FF1Context(CommonContext): command_processor = FF1CommandProcessor + game = 'Final Fantasy' items_handling = 0b111 # full remote def __init__(self, server_address, password): @@ -48,7 +49,6 @@ class FF1Context(CommonContext): self.messages = {} self.locations_array = None self.nes_status = CONNECTION_INITIAL_STATUS - self.game = 'Final Fantasy' self.awaiting_rom = False self.display_msgs = True @@ -68,14 +68,13 @@ class FF1Context(CommonContext): def on_package(self, cmd: str, args: dict): if cmd == 'Connected': - self.game = self.games.get(self.slot, None) asyncio.create_task(parse_locations(self.locations_array, self, True)) elif cmd == 'Print': msg = args['text'] if ': !' not in msg: self._set_message(msg, SYSTEM_MESSAGE_ID) elif cmd == "ReceivedItems": - msg = f"Received {', '.join([self.item_name_getter(item.item) for item in args['items']])}" + msg = f"Received {', '.join([self.item_names[item.item] for item in args['items']])}" self._set_message(msg, SYSTEM_MESSAGE_ID) elif cmd == 'PrintJSON': print_type = args['type'] @@ -85,20 +84,20 @@ class FF1Context(CommonContext): sending_player_id = item.player sending_player_name = self.player_names[item.player] if print_type == 'Hint': - msg = f"Hint: Your {self.item_name_getter(item.item)} is at" \ - f" {self.player_names[item.player]}'s {self.location_name_getter(item.location)}" + msg = f"Hint: Your {self.item_names[item.item]} is at" \ + f" {self.player_names[item.player]}'s {self.location_names[item.location]}" self._set_message(msg, item.item) elif print_type == 'ItemSend' and receiving_player_id != self.slot: if sending_player_id == self.slot: if receiving_player_id == self.slot: - msg = f"You found your own {self.item_name_getter(item.item)}" + msg = f"You found your own {self.item_names[item.item]}" else: - msg = f"You sent {self.item_name_getter(item.item)} to {receiving_player_name}" + msg = f"You sent {self.item_names[item.item]} to {receiving_player_name}" else: if receiving_player_id == sending_player_id: - msg = f"{sending_player_name} found their {self.item_name_getter(item.item)}" + msg = f"{sending_player_name} found their {self.item_names[item.item]}" else: - msg = f"{sending_player_name} sent {self.item_name_getter(item.item)} to " \ + msg = f"{sending_player_name} sent {self.item_names[item.item]} to " \ f"{receiving_player_name}" self._set_message(msg, item.item) @@ -151,13 +150,13 @@ async def parse_locations(locations_array: List[int], ctx: FF1Context, force: bo index -= 0x200 flag = 0x02 - # print(f"Location: {ctx.location_name_getter(location)}") + # print(f"Location: {ctx.location_names[location]}") # print(f"Index: {str(hex(index))}") # print(f"value: {locations_array[index] & flag != 0}") if locations_array[index] & flag != 0: locations_checked.append(location) if locations_checked: - # print([ctx.location_name_getter(location) for location in locations_checked]) + # print([ctx.location_names[location] for location in locations_checked]) await ctx.send_msgs([ {"cmd": "LocationChecks", "locations": locations_checked} diff --git a/MultiServer.py b/MultiServer.py index 06a3bba2..e1a46ace 100644 --- a/MultiServer.py +++ b/MultiServer.py @@ -631,8 +631,7 @@ async def on_client_connected(ctx: Context, client: Client): 'cmd': 'RoomInfo', 'password': bool(ctx.password), 'players': players, - # TODO remove around 0.2.5 in favor of slot_info ? - # Maybe convert into a list of games that are present to fetch relevant datapackage entries before Connect? + # TODO convert to list of games present in 0.4 'games': [ctx.games[x] for x in range(1, len(ctx.games) + 1)], # tags are for additional features in the communication. # Name them by feature or fork, as you feel is appropriate. diff --git a/NetUtils.py b/NetUtils.py index 9b207715..ccc3f6da 100644 --- a/NetUtils.py +++ b/NetUtils.py @@ -245,7 +245,7 @@ class JSONtoTextParser(metaclass=HandlerMeta): def _handle_item_id(self, node: JSONMessagePart): item_id = int(node["text"]) - node["text"] = self.ctx.item_name_getter(item_id) + node["text"] = self.ctx.item_names[item_id] return self._handle_item_name(node) def _handle_location_name(self, node: JSONMessagePart): @@ -254,7 +254,7 @@ class JSONtoTextParser(metaclass=HandlerMeta): def _handle_location_id(self, node: JSONMessagePart): item_id = int(node["text"]) - node["text"] = self.ctx.location_name_getter(item_id) + node["text"] = self.ctx.location_names[item_id] return self._handle_location_name(node) def _handle_entrance_name(self, node: JSONMessagePart): diff --git a/SNIClient.py b/SNIClient.py index 32a0da4b..0563b494 100644 --- a/SNIClient.py +++ b/SNIClient.py @@ -893,7 +893,7 @@ async def track_locations(ctx: Context, roomid, roomdata): def new_check(location_id): new_locations.append(location_id) ctx.locations_checked.add(location_id) - location = ctx.location_name_getter(location_id) + location = ctx.location_names[location_id] snes_logger.info( f'New Check: {location} ({len(ctx.locations_checked)}/{len(ctx.missing_locations) + len(ctx.checked_locations)})') @@ -1126,9 +1126,9 @@ async def game_watcher(ctx: Context): item = ctx.items_received[recv_index] recv_index += 1 logging.info('Received %s from %s (%s) (%d/%d in list)' % ( - color(ctx.item_name_getter(item.item), 'red', 'bold'), + color(ctx.item_names[item.item], 'red', 'bold'), color(ctx.player_names[item.player], 'yellow'), - ctx.location_name_getter(item.location), recv_index, len(ctx.items_received))) + ctx.location_names[item.location], recv_index, len(ctx.items_received))) snes_buffered_write(ctx, RECV_PROGRESS_ADDR, bytes([recv_index & 0xFF, (recv_index >> 8) & 0xFF])) @@ -1183,7 +1183,7 @@ async def game_watcher(ctx: Context): location_id = locations_start_id + itemIndex ctx.locations_checked.add(location_id) - location = ctx.location_name_getter(location_id) + location = ctx.location_names[location_id] snes_logger.info( f'New Check: {location} ({len(ctx.locations_checked)}/{len(ctx.missing_locations) + len(ctx.checked_locations)})') await ctx.send_msgs([{"cmd": 'LocationChecks', "locations": [location_id]}]) @@ -1212,9 +1212,9 @@ async def game_watcher(ctx: Context): snes_buffered_write(ctx, SM_RECV_PROGRESS_ADDR + 0x602, bytes([itemOutPtr & 0xFF, (itemOutPtr >> 8) & 0xFF])) logging.info('Received %s from %s (%s) (%d/%d in list)' % ( - color(ctx.item_name_getter(item.item), 'red', 'bold'), + color(ctx.item_names[item.item], 'red', 'bold'), color(ctx.player_names[item.player], 'yellow'), - ctx.location_name_getter(item.location), itemOutPtr, len(ctx.items_received))) + ctx.location_names[item.location], itemOutPtr, len(ctx.items_received))) await snes_flush_writes(ctx) elif ctx.game == GAME_SMZ3: currentGame = await snes_read(ctx, SRAM_START + 0x33FE, 2) @@ -1255,7 +1255,7 @@ async def game_watcher(ctx: Context): location_id = locations_start_id + itemIndex ctx.locations_checked.add(location_id) - location = ctx.location_name_getter(location_id) + location = ctx.location_names[location_id] snes_logger.info(f'New Check: {location} ({len(ctx.locations_checked)}/{len(ctx.missing_locations) + len(ctx.checked_locations)})') await ctx.send_msgs([{"cmd": 'LocationChecks', "locations": [location_id]}]) @@ -1276,8 +1276,8 @@ async def game_watcher(ctx: Context): itemOutPtr += 1 snes_buffered_write(ctx, SMZ3_RECV_PROGRESS_ADDR + 0x602, bytes([itemOutPtr & 0xFF, (itemOutPtr >> 8) & 0xFF])) logging.info('Received %s from %s (%s) (%d/%d in list)' % ( - color(ctx.item_name_getter(item.item), 'red', 'bold'), color(ctx.player_names[item.player], 'yellow'), - ctx.location_name_getter(item.location), itemOutPtr, len(ctx.items_received))) + color(ctx.item_names[item.item], 'red', 'bold'), color(ctx.player_names[item.player], 'yellow'), + ctx.location_names[item.location], itemOutPtr, len(ctx.items_received))) await snes_flush_writes(ctx) diff --git a/Starcraft2Client.py b/Starcraft2Client.py index 120da014..54cc484a 100644 --- a/Starcraft2Client.py +++ b/Starcraft2Client.py @@ -592,8 +592,8 @@ def calc_objectives_completed(mission, missions_info, locations_done, unfinished if (missions_info[mission].id * 100 + SC2WOL_LOC_ID_OFFSET + i) in locations_done: objectives_complete += 1 else: - unfinished_locations[mission].append(ctx.location_name_getter( - missions_info[mission].id * 100 + SC2WOL_LOC_ID_OFFSET + i)) + unfinished_locations[mission].append(ctx.location_names[ + missions_info[mission].id * 100 + SC2WOL_LOC_ID_OFFSET + i]) return objectives_complete diff --git a/docs/network protocol.md b/docs/network protocol.md index 63b3f7c8..afc19a1f 100644 --- a/docs/network protocol.md +++ b/docs/network protocol.md @@ -64,7 +64,7 @@ Sent to clients when they connect to an Archipelago server. | hint_cost | int | The amount of points it costs to receive a hint from the server. | | location_check_points | int | The amount of hint points you receive per item/location check completed. || | players | list\[[NetworkPlayer](#NetworkPlayer)\] | Sent only if the client is properly authenticated (see [Archipelago Connection Handshake](#Archipelago-Connection-Handshake)). Information on the players currently connected to the server. | -| games | list\[str\] | sorted list of game names for the players, so first player's game will be games\[0\]. Matches game names in datapackage. | +| games | list\[str\] | List of games present in this multiworld. | | datapackage_version | int | Sum of individual games' datapackage version. Deprecated. Use `datapackage_versions` instead. | | datapackage_versions | dict\[str, int\] | Data versions of the individual games' data packages the server will send. Used to decide which games' caches are outdated. See [Data Package Contents](#Data-Package-Contents). | | seed_name | str | uniquely identifying name of this generation |