From 113c54f9bea7138946b9bcc9b583a34254414b4f Mon Sep 17 00:00:00 2001 From: Doug Hoskisson Date: Sun, 3 Mar 2024 13:10:14 -0800 Subject: [PATCH] Zillion: remove rom requirement for generation (#2875) * in the middle of work towards no rom for generation (not working) * no rom needed for Zillion generation * revert core changes --- worlds/zillion/__init__.py | 113 +++++++++++--------------------- worlds/zillion/client.py | 3 +- worlds/zillion/gen_data.py | 35 ++++++++++ worlds/zillion/id_maps.py | 75 +++++++++++++++++++-- worlds/zillion/patch.py | 57 ++++++++++++++-- worlds/zillion/requirements.txt | 2 +- 6 files changed, 200 insertions(+), 85 deletions(-) create mode 100644 worlds/zillion/gen_data.py diff --git a/worlds/zillion/__init__.py b/worlds/zillion/__init__.py index d7e653bb..b4e382e0 100644 --- a/worlds/zillion/__init__.py +++ b/worlds/zillion/__init__.py @@ -4,20 +4,22 @@ import functools import settings import threading import typing -from typing import Any, Dict, List, Set, Tuple, Optional, cast +from typing import Any, Dict, List, Set, Tuple, Optional import os import logging from BaseClasses import ItemClassification, LocationProgressType, \ MultiWorld, Item, CollectionState, Entrance, Tutorial + +from .gen_data import GenData from .logic import cs_to_zz_locs from .region import ZillionLocation, ZillionRegion from .options import ZillionOptions, validate -from .id_maps import item_name_to_id as _item_name_to_id, \ +from .id_maps import ZillionSlotInfo, get_slot_info, item_name_to_id as _item_name_to_id, \ loc_name_to_id as _loc_name_to_id, make_id_to_others, \ zz_reg_name_to_reg_name, base_id from .item import ZillionItem -from .patch import ZillionDeltaPatch, get_base_rom_path +from .patch import ZillionPatch from zilliandomizer.randomizer import Randomizer as ZzRandomizer from zilliandomizer.system import System @@ -33,8 +35,8 @@ class ZillionSettings(settings.Group): """File name of the Zillion US rom""" description = "Zillion US ROM File" copy_to = "Zillion (UE) [!].sms" - assert ZillionDeltaPatch.hash - md5s = [ZillionDeltaPatch.hash] + assert ZillionPatch.hash + md5s = [ZillionPatch.hash] class RomStart(str): """ @@ -134,14 +136,6 @@ class ZillionWorld(World): _id_to_name, _id_to_zz_id, id_to_zz_item = make_id_to_others(start_char) self.id_to_zz_item = id_to_zz_item - @classmethod - def stage_assert_generate(cls, multiworld: MultiWorld) -> None: - """Checks that a game is capable of generating, usually checks for some base file like a ROM. - Not run for unittests since they don't produce output""" - rom_file = get_base_rom_path() - if not os.path.exists(rom_file): - raise FileNotFoundError(rom_file) - def generate_early(self) -> None: if not hasattr(self.multiworld, "zillion_logic_cache"): setattr(self.multiworld, "zillion_logic_cache", {}) @@ -311,7 +305,9 @@ class ZillionWorld(World): if sc != to_stay: group_players.remove(p) assert "world" in group - cast(ZillionWorld, group["world"])._make_item_maps(to_stay) + group_world = group["world"] + assert isinstance(group_world, ZillionWorld) + group_world._make_item_maps(to_stay) def post_fill(self) -> None: """Optional Method that is called after regular fill. Can be used to do adjustments before output generation. @@ -319,27 +315,28 @@ class ZillionWorld(World): self.zz_system.post_fill() - def finalize_item_locations(self) -> None: + def finalize_item_locations(self) -> GenData: """ sync zilliandomizer item locations with AP item locations + + return the data needed to generate output """ - rom_dir_name = os.path.dirname(get_base_rom_path()) - self.zz_system.make_patcher(rom_dir_name) - assert self.zz_system.randomizer and self.zz_system.patcher, "generate_early hasn't been called" - zz_options = self.zz_system.randomizer.options + + assert self.zz_system.randomizer, "generate_early hasn't been called" # debug_zz_loc_ids: Dict[str, int] = {} empty = zz_items[4] multi_item = empty # a different patcher method differentiates empty from ap multi item multi_items: Dict[str, Tuple[str, str]] = {} # zz_loc_name to (item_name, player_name) - for loc in self.multiworld.get_locations(self.player): - z_loc = cast(ZillionLocation, loc) + for z_loc in self.multiworld.get_locations(self.player): + assert isinstance(z_loc, ZillionLocation) # debug_zz_loc_ids[z_loc.zz_loc.name] = id(z_loc.zz_loc) if z_loc.item is None: self.logger.warn("generate_output location has no item - is that ok?") z_loc.zz_loc.item = empty elif z_loc.item.player == self.player: - z_item = cast(ZillionItem, z_loc.item) + z_item = z_loc.item + assert isinstance(z_item, ZillionItem) z_loc.zz_loc.item = z_item.zz_item else: # another player's item # print(f"put multi item in {z_loc.zz_loc.name}") @@ -368,47 +365,32 @@ class ZillionWorld(World): f"in world {self.player} didn't get an item" ) - zz_patcher = self.zz_system.patcher - - zz_patcher.write_locations(self.zz_system.randomizer.regions, - zz_options.start_char, - self.zz_system.randomizer.loc_name_2_pretty) - self.slot_data_ready.set() - rm = self.zz_system.resource_managers - assert rm, "missing resource_managers from generate_early" - zz_patcher.all_fixes_and_options(zz_options, rm) - zz_patcher.set_external_item_interface(zz_options.start_char, zz_options.max_level) - zz_patcher.set_multiworld_items(multi_items) game_id = self.multiworld.player_name[self.player].encode() + b'\x00' + self.multiworld.seed_name[-6:].encode() - zz_patcher.set_rom_to_ram_data(game_id) + + return GenData(multi_items, self.zz_system.get_game(), game_id) def generate_output(self, output_directory: str) -> None: - """This method gets called from a threadpool, do not use world.random here. - If you need any last-second randomization, use MultiWorld.per_slot_randoms[slot] instead.""" - self.finalize_item_locations() - - assert self.zz_system.patcher, "didn't get patcher from finalize_item_locations" - # original_rom_bytes = self.zz_patcher.rom - patched_rom_bytes = self.zz_system.patcher.get_patched_bytes() + """This method gets called from a threadpool, do not use multiworld.random here. + If you need any last-second randomization, use self.random instead.""" + try: + gen_data = self.finalize_item_locations() + except BaseException: + raise + finally: + self.slot_data_ready.set() out_file_base = self.multiworld.get_out_file_name_base(self.player) - filename = os.path.join( - output_directory, - f'{out_file_base}{ZillionDeltaPatch.result_file_ending}' - ) - with open(filename, "wb") as binary_file: - binary_file.write(patched_rom_bytes) - patch = ZillionDeltaPatch( - os.path.splitext(filename)[0] + ZillionDeltaPatch.patch_file_ending, - player=self.player, - player_name=self.multiworld.player_name[self.player], - patched_path=filename - ) + patch_file_name = os.path.join(output_directory, f"{out_file_base}{ZillionPatch.patch_file_ending}") + patch = ZillionPatch(patch_file_name, + player=self.player, + player_name=self.multiworld.player_name[self.player], + gen_data_str=gen_data.to_json()) patch.write() - os.remove(filename) - def fill_slot_data(self) -> Dict[str, Any]: # json of WebHostLib.models.Slot + self.logger.debug(f"Zillion player {self.player} finished generate_output") + + def fill_slot_data(self) -> ZillionSlotInfo: # json of WebHostLib.models.Slot """Fill in the `slot_data` field in the `Connected` network package. This is a way the generator can give custom data to the client. The client will receive this as JSON in the `Connected` response.""" @@ -418,25 +400,10 @@ class ZillionWorld(World): # TODO: tell client which canisters are keywords # so it can open and get those when restoring doors - assert self.zz_system.randomizer, "didn't get randomizer from generate_early" - - rescues: Dict[str, Any] = {} self.slot_data_ready.wait() - zz_patcher = self.zz_system.patcher - assert zz_patcher, "didn't get patcher from generate_output" - for i in (0, 1): - if i in zz_patcher.rescue_locations: - ri = zz_patcher.rescue_locations[i] - rescues[str(i)] = { - "start_char": ri.start_char, - "room_code": ri.room_code, - "mask": ri.mask - } - return { - "start_char": self.zz_system.randomizer.options.start_char, - "rescues": rescues, - "loc_mem_to_id": zz_patcher.loc_memory_to_loc_id - } + assert self.zz_system.randomizer, "didn't get randomizer from generate_early" + game = self.zz_system.get_game() + return get_slot_info(game.regions, game.char_order[0], game.loc_name_2_pretty) # def modify_multidata(self, multidata: Dict[str, Any]) -> None: # """For deeper modification of server multidata.""" diff --git a/worlds/zillion/client.py b/worlds/zillion/client.py index 1a85b9df..5c2e1145 100644 --- a/worlds/zillion/client.py +++ b/worlds/zillion/client.py @@ -12,11 +12,10 @@ from Utils import async_start import colorama -from zilliandomizer.zri.memory import Memory +from zilliandomizer.zri.memory import Memory, RescueInfo from zilliandomizer.zri import events from zilliandomizer.utils.loc_name_maps import id_to_loc from zilliandomizer.options import Chars -from zilliandomizer.patch import RescueInfo from .id_maps import loc_name_to_id, make_id_to_others from .config import base_id diff --git a/worlds/zillion/gen_data.py b/worlds/zillion/gen_data.py new file mode 100644 index 00000000..aa24ff89 --- /dev/null +++ b/worlds/zillion/gen_data.py @@ -0,0 +1,35 @@ +from dataclasses import dataclass +import json +from typing import Dict, Tuple + +from zilliandomizer.game import Game as ZzGame + + +@dataclass +class GenData: + """ data passed from generation to patcher """ + + multi_items: Dict[str, Tuple[str, str]] + """ zz_loc_name to (item_name, player_name) """ + zz_game: ZzGame + game_id: bytes + """ the byte string used to detect the rom """ + + def to_json(self) -> str: + """ serialized data from generation needed to patch rom """ + jsonable = { + "multi_items": self.multi_items, + "zz_game": self.zz_game.to_jsonable(), + "game_id": list(self.game_id) + } + return json.dumps(jsonable) + + @staticmethod + def from_json(gen_data_str: str) -> "GenData": + """ the reverse of `to_json` """ + from_json = json.loads(gen_data_str) + return GenData( + from_json["multi_items"], + ZzGame.from_jsonable(from_json["zz_game"]), + bytes(from_json["game_id"]) + ) diff --git a/worlds/zillion/id_maps.py b/worlds/zillion/id_maps.py index bc9caeee..32d71fc7 100644 --- a/worlds/zillion/id_maps.py +++ b/worlds/zillion/id_maps.py @@ -1,10 +1,22 @@ -from typing import Dict, Tuple -from zilliandomizer.logic_components.items import Item as ZzItem, \ - item_name_to_id as zz_item_name_to_zz_id, items as zz_items, \ - item_name_to_item as zz_item_name_to_zz_item +from collections import defaultdict +from typing import Dict, Iterable, Mapping, Tuple, TypedDict + +from zilliandomizer.logic_components.items import ( + Item as ZzItem, + KEYWORD, + NORMAL, + RESCUE, + item_name_to_id as zz_item_name_to_zz_id, + items as zz_items, + item_name_to_item as zz_item_name_to_zz_item, +) +from zilliandomizer.logic_components.regions import RegionData +from zilliandomizer.low_resources.item_rooms import item_room_codes from zilliandomizer.options import Chars from zilliandomizer.utils.loc_name_maps import loc_to_id as pretty_loc_name_to_id -from zilliandomizer.utils import parse_reg_name +from zilliandomizer.utils import parse_loc_name, parse_reg_name +from zilliandomizer.zri.memory import RescueInfo + from .config import base_id as base_id item_name_to_id = { @@ -91,3 +103,56 @@ def zz_reg_name_to_reg_name(zz_reg_name: str) -> str: end = zz_reg_name[5:] return f"{make_room_name(row, col)} {end.upper()}" return zz_reg_name + + +class ClientRescue(TypedDict): + start_char: Chars + room_code: int + mask: int + + +class ZillionSlotInfo(TypedDict): + start_char: Chars + rescues: Dict[str, ClientRescue] + loc_mem_to_id: Dict[int, int] + """ memory location of canister to Archipelago location id number """ + + +def get_slot_info(regions: Iterable[RegionData], + start_char: Chars, + loc_name_to_pretty: Mapping[str, str]) -> ZillionSlotInfo: + items_placed_in_map_index: Dict[int, int] = defaultdict(int) + rescue_locations: Dict[int, RescueInfo] = {} + loc_memory_to_loc_id: Dict[int, int] = {} + for region in regions: + for loc in region.locations: + assert loc.item, ("There should be an item placed in every location before " + f"writing slot info. {loc.name} is missing item.") + if loc.item.code in {KEYWORD, NORMAL, RESCUE}: + row, col, _y, _x = parse_loc_name(loc.name) + map_index = row * 8 + col + item_no = items_placed_in_map_index[map_index] + room_code = item_room_codes[map_index] + + r = room_code + m = 1 << item_no + if loc.item.code == RESCUE: + rescue_locations[loc.item.id] = RescueInfo(start_char, r, m) + loc_memory = (r << 7) | m + loc_memory_to_loc_id[loc_memory] = pretty_loc_name_to_id[loc_name_to_pretty[loc.name]] + items_placed_in_map_index[map_index] += 1 + + rescues: Dict[str, ClientRescue] = {} + for i in (0, 1): + if i in rescue_locations: + ri = rescue_locations[i] + rescues[str(i)] = { + "start_char": ri.start_char, + "room_code": ri.room_code, + "mask": ri.mask + } + return { + "start_char": start_char, + "rescues": rescues, + "loc_mem_to_id": loc_memory_to_loc_id + } diff --git a/worlds/zillion/patch.py b/worlds/zillion/patch.py index 148caac9..dcbb85bc 100644 --- a/worlds/zillion/patch.py +++ b/worlds/zillion/patch.py @@ -1,22 +1,53 @@ -from typing import BinaryIO, Optional, cast -import Utils -from worlds.Files import APDeltaPatch import os +from typing import Any, BinaryIO, Optional, cast +import zipfile + +from typing_extensions import override + +import Utils +from worlds.Files import APPatch + +from zilliandomizer.patch import Patcher + +from .gen_data import GenData USHASH = 'd4bf9e7bcf9a48da53785d2ae7bc4270' -class ZillionDeltaPatch(APDeltaPatch): +class ZillionPatch(APPatch): hash = USHASH game = "Zillion" patch_file_ending = ".apzl" result_file_ending = ".sms" + gen_data_str: str + """ JSON encoded """ + + def __init__(self, *args: Any, gen_data_str: str = "", **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.gen_data_str = gen_data_str + @classmethod def get_source_data(cls) -> bytes: with open(get_base_rom_path(), "rb") as stream: return read_rom(stream) + @override + def write_contents(self, opened_zipfile: zipfile.ZipFile) -> None: + super().write_contents(opened_zipfile) + opened_zipfile.writestr("gen_data.json", + self.gen_data_str, + compress_type=zipfile.ZIP_DEFLATED) + + @override + def read_contents(self, opened_zipfile: zipfile.ZipFile) -> None: + super().read_contents(opened_zipfile) + self.gen_data_str = opened_zipfile.read("gen_data.json").decode() + + def patch(self, target: str) -> None: + self.read() + write_rom_from_gen_data(self.gen_data_str, target) + def get_base_rom_path(file_name: Optional[str] = None) -> str: options = Utils.get_options() @@ -32,3 +63,21 @@ def read_rom(stream: BinaryIO) -> bytes: data = stream.read() # I'm not aware of any sms header. return data + + +def write_rom_from_gen_data(gen_data_str: str, output_rom_file_name: str) -> None: + """ take the output of `GenData.to_json`, and create rom from it """ + gen_data = GenData.from_json(gen_data_str) + + base_rom_path = get_base_rom_path() + zz_patcher = Patcher(base_rom_path) + + zz_patcher.write_locations(gen_data.zz_game.regions, gen_data.zz_game.char_order[0]) + zz_patcher.all_fixes_and_options(gen_data.zz_game) + zz_patcher.set_external_item_interface(gen_data.zz_game.char_order[0], gen_data.zz_game.options.max_level) + zz_patcher.set_multiworld_items(gen_data.multi_items) + zz_patcher.set_rom_to_ram_data(gen_data.game_id) + + patched_rom_bytes = zz_patcher.get_patched_bytes() + with open(output_rom_file_name, "wb") as binary_file: + binary_file.write(patched_rom_bytes) diff --git a/worlds/zillion/requirements.txt b/worlds/zillion/requirements.txt index c8944925..3a784846 100644 --- a/worlds/zillion/requirements.txt +++ b/worlds/zillion/requirements.txt @@ -1,2 +1,2 @@ -zilliandomizer @ git+https://github.com/beauxq/zilliandomizer@ae00a4b186be897c7cfaf429a0e0ff83c4ecf28c#0.6.0 +zilliandomizer @ git+https://github.com/beauxq/zilliandomizer@b36a23b5a138c78732ac8efb5b5ca8b0be07dcff#0.7.0 typing-extensions>=4.7, <5