Zillion: remove rom requirement for generation (#2875)

* in the middle of work towards no rom for generation (not working)

* no rom needed for Zillion generation

* revert core changes

worlds/zillion/__init__.py

@@ -4,20 +4,22 @@ import functools
 import settings
 import threading
 import typing
-from typing import Any, Dict, List, Set, Tuple, Optional, cast
+from typing import Any, Dict, List, Set, Tuple, Optional
 import os
 import logging
 
 from BaseClasses import ItemClassification, LocationProgressType, \
     MultiWorld, Item, CollectionState, Entrance, Tutorial
+
+from .gen_data import GenData
 from .logic import cs_to_zz_locs
 from .region import ZillionLocation, ZillionRegion
 from .options import ZillionOptions, validate
-from .id_maps import item_name_to_id as _item_name_to_id, \
+from .id_maps import ZillionSlotInfo, get_slot_info, item_name_to_id as _item_name_to_id, \
     loc_name_to_id as _loc_name_to_id, make_id_to_others, \
     zz_reg_name_to_reg_name, base_id
 from .item import ZillionItem
-from .patch import ZillionDeltaPatch, get_base_rom_path
+from .patch import ZillionPatch
 
 from zilliandomizer.randomizer import Randomizer as ZzRandomizer
 from zilliandomizer.system import System
@@ -33,8 +35,8 @@ class ZillionSettings(settings.Group):
         """File name of the Zillion US rom"""
         description = "Zillion US ROM File"
         copy_to = "Zillion (UE) [!].sms"
-        assert ZillionDeltaPatch.hash
+        assert ZillionPatch.hash
-        md5s = [ZillionDeltaPatch.hash]
+        md5s = [ZillionPatch.hash]
 
     class RomStart(str):
         """
@@ -134,14 +136,6 @@ class ZillionWorld(World):
         _id_to_name, _id_to_zz_id, id_to_zz_item = make_id_to_others(start_char)
         self.id_to_zz_item = id_to_zz_item
 
-    @classmethod
-    def stage_assert_generate(cls, multiworld: MultiWorld) -> None:
-        """Checks that a game is capable of generating, usually checks for some base file like a ROM.
-        Not run for unittests since they don't produce output"""
-        rom_file = get_base_rom_path()
-        if not os.path.exists(rom_file):
-            raise FileNotFoundError(rom_file)
-
     def generate_early(self) -> None:
         if not hasattr(self.multiworld, "zillion_logic_cache"):
             setattr(self.multiworld, "zillion_logic_cache", {})
@@ -311,7 +305,9 @@ class ZillionWorld(World):
                         if sc != to_stay:
                             group_players.remove(p)
                 assert "world" in group
-                cast(ZillionWorld, group["world"])._make_item_maps(to_stay)
+                group_world = group["world"]
+                assert isinstance(group_world, ZillionWorld)
+                group_world._make_item_maps(to_stay)
 
     def post_fill(self) -> None:
         """Optional Method that is called after regular fill. Can be used to do adjustments before output generation.
@@ -319,27 +315,28 @@ class ZillionWorld(World):
 
         self.zz_system.post_fill()
 
-    def finalize_item_locations(self) -> None:
+    def finalize_item_locations(self) -> GenData:
         """
         sync zilliandomizer item locations with AP item locations
+
+        return the data needed to generate output
         """
-        rom_dir_name = os.path.dirname(get_base_rom_path())
+
-        self.zz_system.make_patcher(rom_dir_name)
+        assert self.zz_system.randomizer, "generate_early hasn't been called"
-        assert self.zz_system.randomizer and self.zz_system.patcher, "generate_early hasn't been called"
-        zz_options = self.zz_system.randomizer.options
 
         # debug_zz_loc_ids: Dict[str, int] = {}
         empty = zz_items[4]
         multi_item = empty  # a different patcher method differentiates empty from ap multi item
         multi_items: Dict[str, Tuple[str, str]] = {}  # zz_loc_name to (item_name, player_name)
-        for loc in self.multiworld.get_locations(self.player):
+        for z_loc in self.multiworld.get_locations(self.player):
-            z_loc = cast(ZillionLocation, loc)
+            assert isinstance(z_loc, ZillionLocation)
             # debug_zz_loc_ids[z_loc.zz_loc.name] = id(z_loc.zz_loc)
             if z_loc.item is None:
                 self.logger.warn("generate_output location has no item - is that ok?")
                 z_loc.zz_loc.item = empty
             elif z_loc.item.player == self.player:
-                z_item = cast(ZillionItem, z_loc.item)
+                z_item = z_loc.item
+                assert isinstance(z_item, ZillionItem)
                 z_loc.zz_loc.item = z_item.zz_item
             else:  # another player's item
                 # print(f"put multi item in {z_loc.zz_loc.name}")
@@ -368,47 +365,32 @@ class ZillionWorld(World):
                 f"in world {self.player} didn't get an item"
             )
 
-        zz_patcher = self.zz_system.patcher
-
-        zz_patcher.write_locations(self.zz_system.randomizer.regions,
-                                   zz_options.start_char,
-                                   self.zz_system.randomizer.loc_name_2_pretty)
-        self.slot_data_ready.set()
-        rm = self.zz_system.resource_managers
-        assert rm, "missing resource_managers from generate_early"
-        zz_patcher.all_fixes_and_options(zz_options, rm)
-        zz_patcher.set_external_item_interface(zz_options.start_char, zz_options.max_level)
-        zz_patcher.set_multiworld_items(multi_items)
         game_id = self.multiworld.player_name[self.player].encode() + b'\x00' + self.multiworld.seed_name[-6:].encode()
-        zz_patcher.set_rom_to_ram_data(game_id)
+
+        return GenData(multi_items, self.zz_system.get_game(), game_id)
 
     def generate_output(self, output_directory: str) -> None:
-        """This method gets called from a threadpool, do not use world.random here.
+        """This method gets called from a threadpool, do not use multiworld.random here.
-        If you need any last-second randomization, use MultiWorld.per_slot_randoms[slot] instead."""
+        If you need any last-second randomization, use self.random instead."""
-        self.finalize_item_locations()
+        try:
-
+            gen_data = self.finalize_item_locations()
-        assert self.zz_system.patcher, "didn't get patcher from finalize_item_locations"
+        except BaseException:
-        # original_rom_bytes = self.zz_patcher.rom
+            raise
-        patched_rom_bytes = self.zz_system.patcher.get_patched_bytes()
+        finally:
+            self.slot_data_ready.set()
 
         out_file_base = self.multiworld.get_out_file_name_base(self.player)
 
-        filename = os.path.join(
+        patch_file_name = os.path.join(output_directory, f"{out_file_base}{ZillionPatch.patch_file_ending}")
-            output_directory,
+        patch = ZillionPatch(patch_file_name,
-            f'{out_file_base}{ZillionDeltaPatch.result_file_ending}'
-        )
-        with open(filename, "wb") as binary_file:
-            binary_file.write(patched_rom_bytes)
-        patch = ZillionDeltaPatch(
-            os.path.splitext(filename)[0] + ZillionDeltaPatch.patch_file_ending,
                              player=self.player,
                              player_name=self.multiworld.player_name[self.player],
-            patched_path=filename
+                             gen_data_str=gen_data.to_json())
-        )
         patch.write()
-        os.remove(filename)
 
-    def fill_slot_data(self) -> Dict[str, Any]:  # json of WebHostLib.models.Slot
+        self.logger.debug(f"Zillion player {self.player} finished generate_output")
+
+    def fill_slot_data(self) -> ZillionSlotInfo:  # json of WebHostLib.models.Slot
         """Fill in the `slot_data` field in the `Connected` network package.
         This is a way the generator can give custom data to the client.
         The client will receive this as JSON in the `Connected` response."""
@@ -418,25 +400,10 @@ class ZillionWorld(World):
         # TODO: tell client which canisters are keywords
         # so it can open and get those when restoring doors
 
-        assert self.zz_system.randomizer, "didn't get randomizer from generate_early"
-
-        rescues: Dict[str, Any] = {}
         self.slot_data_ready.wait()
-        zz_patcher = self.zz_system.patcher
+        assert self.zz_system.randomizer, "didn't get randomizer from generate_early"
-        assert zz_patcher, "didn't get patcher from generate_output"
+        game = self.zz_system.get_game()
-        for i in (0, 1):
+        return get_slot_info(game.regions, game.char_order[0], game.loc_name_2_pretty)
-            if i in zz_patcher.rescue_locations:
-                ri = zz_patcher.rescue_locations[i]
-                rescues[str(i)] = {
-                    "start_char": ri.start_char,
-                    "room_code": ri.room_code,
-                    "mask": ri.mask
-                }
-        return {
-            "start_char": self.zz_system.randomizer.options.start_char,
-            "rescues": rescues,
-            "loc_mem_to_id": zz_patcher.loc_memory_to_loc_id
-        }
 
     # def modify_multidata(self, multidata: Dict[str, Any]) -> None:
     #     """For deeper modification of server multidata."""

worlds/zillion/client.py

@@ -12,11 +12,10 @@ from Utils import async_start
 
 import colorama
 
-from zilliandomizer.zri.memory import Memory
+from zilliandomizer.zri.memory import Memory, RescueInfo
 from zilliandomizer.zri import events
 from zilliandomizer.utils.loc_name_maps import id_to_loc
 from zilliandomizer.options import Chars
-from zilliandomizer.patch import RescueInfo
 
 from .id_maps import loc_name_to_id, make_id_to_others
 from .config import base_id

worlds/zillion/gen_data.py

@@ -0,0 +1,35 @@
+from dataclasses import dataclass
+import json
+from typing import Dict, Tuple
+
+from zilliandomizer.game import Game as ZzGame
+
+
+@dataclass
+class GenData:
+    """ data passed from generation to patcher """
+
+    multi_items: Dict[str, Tuple[str, str]]
+    """ zz_loc_name to (item_name, player_name) """
+    zz_game: ZzGame
+    game_id: bytes
+    """ the byte string used to detect the rom """
+
+    def to_json(self) -> str:
+        """ serialized data from generation needed to patch rom """
+        jsonable = {
+            "multi_items": self.multi_items,
+            "zz_game": self.zz_game.to_jsonable(),
+            "game_id": list(self.game_id)
+        }
+        return json.dumps(jsonable)
+
+    @staticmethod
+    def from_json(gen_data_str: str) -> "GenData":
+        """ the reverse of `to_json` """
+        from_json = json.loads(gen_data_str)
+        return GenData(
+            from_json["multi_items"],
+            ZzGame.from_jsonable(from_json["zz_game"]),
+            bytes(from_json["game_id"])
+        )

worlds/zillion/id_maps.py

@@ -1,10 +1,22 @@
-from typing import Dict, Tuple
+from collections import defaultdict
-from zilliandomizer.logic_components.items import Item as ZzItem, \
+from typing import Dict, Iterable, Mapping, Tuple, TypedDict
-    item_name_to_id as zz_item_name_to_zz_id, items as zz_items, \
+
-    item_name_to_item as zz_item_name_to_zz_item
+from zilliandomizer.logic_components.items import (
+    Item as ZzItem,
+    KEYWORD,
+    NORMAL,
+    RESCUE,
+    item_name_to_id as zz_item_name_to_zz_id,
+    items as zz_items,
+    item_name_to_item as zz_item_name_to_zz_item,
+)
+from zilliandomizer.logic_components.regions import RegionData
+from zilliandomizer.low_resources.item_rooms import item_room_codes
 from zilliandomizer.options import Chars
 from zilliandomizer.utils.loc_name_maps import loc_to_id as pretty_loc_name_to_id
-from zilliandomizer.utils import parse_reg_name
+from zilliandomizer.utils import parse_loc_name, parse_reg_name
+from zilliandomizer.zri.memory import RescueInfo
+
 from .config import base_id as base_id
 
 item_name_to_id = {
@@ -91,3 +103,56 @@ def zz_reg_name_to_reg_name(zz_reg_name: str) -> str:
         end = zz_reg_name[5:]
         return f"{make_room_name(row, col)} {end.upper()}"
     return zz_reg_name
+
+
+class ClientRescue(TypedDict):
+    start_char: Chars
+    room_code: int
+    mask: int
+
+
+class ZillionSlotInfo(TypedDict):
+    start_char: Chars
+    rescues: Dict[str, ClientRescue]
+    loc_mem_to_id: Dict[int, int]
+    """ memory location of canister to Archipelago location id number """
+
+
+def get_slot_info(regions: Iterable[RegionData],
+                  start_char: Chars,
+                  loc_name_to_pretty: Mapping[str, str]) -> ZillionSlotInfo:
+    items_placed_in_map_index: Dict[int, int] = defaultdict(int)
+    rescue_locations: Dict[int, RescueInfo] = {}
+    loc_memory_to_loc_id: Dict[int, int] = {}
+    for region in regions:
+        for loc in region.locations:
+            assert loc.item, ("There should be an item placed in every location before "
+                              f"writing slot info. {loc.name} is missing item.")
+            if loc.item.code in {KEYWORD, NORMAL, RESCUE}:
+                row, col, _y, _x = parse_loc_name(loc.name)
+                map_index = row * 8 + col
+                item_no = items_placed_in_map_index[map_index]
+                room_code = item_room_codes[map_index]
+
+                r = room_code
+                m = 1 << item_no
+                if loc.item.code == RESCUE:
+                    rescue_locations[loc.item.id] = RescueInfo(start_char, r, m)
+                loc_memory = (r << 7) | m
+                loc_memory_to_loc_id[loc_memory] = pretty_loc_name_to_id[loc_name_to_pretty[loc.name]]
+                items_placed_in_map_index[map_index] += 1
+
+    rescues: Dict[str, ClientRescue] = {}
+    for i in (0, 1):
+        if i in rescue_locations:
+            ri = rescue_locations[i]
+            rescues[str(i)] = {
+                "start_char": ri.start_char,
+                "room_code": ri.room_code,
+                "mask": ri.mask
+            }
+    return {
+        "start_char": start_char,
+        "rescues": rescues,
+        "loc_mem_to_id": loc_memory_to_loc_id
+    }

worlds/zillion/patch.py

@@ -1,22 +1,53 @@
-from typing import BinaryIO, Optional, cast
-import Utils
-from worlds.Files import APDeltaPatch
 import os
+from typing import Any, BinaryIO, Optional, cast
+import zipfile
+
+from typing_extensions import override
+
+import Utils
+from worlds.Files import APPatch
+
+from zilliandomizer.patch import Patcher
+
+from .gen_data import GenData
 
 USHASH = 'd4bf9e7bcf9a48da53785d2ae7bc4270'
 
 
-class ZillionDeltaPatch(APDeltaPatch):
+class ZillionPatch(APPatch):
     hash = USHASH
     game = "Zillion"
     patch_file_ending = ".apzl"
     result_file_ending = ".sms"
 
+    gen_data_str: str
+    """ JSON encoded """
+
+    def __init__(self, *args: Any, gen_data_str: str = "", **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self.gen_data_str = gen_data_str
+
     @classmethod
     def get_source_data(cls) -> bytes:
         with open(get_base_rom_path(), "rb") as stream:
             return read_rom(stream)
 
+    @override
+    def write_contents(self, opened_zipfile: zipfile.ZipFile) -> None:
+        super().write_contents(opened_zipfile)
+        opened_zipfile.writestr("gen_data.json",
+                                self.gen_data_str,
+                                compress_type=zipfile.ZIP_DEFLATED)
+
+    @override
+    def read_contents(self, opened_zipfile: zipfile.ZipFile) -> None:
+        super().read_contents(opened_zipfile)
+        self.gen_data_str = opened_zipfile.read("gen_data.json").decode()
+
+    def patch(self, target: str) -> None:
+        self.read()
+        write_rom_from_gen_data(self.gen_data_str, target)
+
 
 def get_base_rom_path(file_name: Optional[str] = None) -> str:
     options = Utils.get_options()
@@ -32,3 +63,21 @@ def read_rom(stream: BinaryIO) -> bytes:
     data = stream.read()
     # I'm not aware of any sms header.
     return data
+
+
+def write_rom_from_gen_data(gen_data_str: str, output_rom_file_name: str) -> None:
+    """ take the output of `GenData.to_json`, and create rom from it """
+    gen_data = GenData.from_json(gen_data_str)
+
+    base_rom_path = get_base_rom_path()
+    zz_patcher = Patcher(base_rom_path)
+
+    zz_patcher.write_locations(gen_data.zz_game.regions, gen_data.zz_game.char_order[0])
+    zz_patcher.all_fixes_and_options(gen_data.zz_game)
+    zz_patcher.set_external_item_interface(gen_data.zz_game.char_order[0], gen_data.zz_game.options.max_level)
+    zz_patcher.set_multiworld_items(gen_data.multi_items)
+    zz_patcher.set_rom_to_ram_data(gen_data.game_id)
+
+    patched_rom_bytes = zz_patcher.get_patched_bytes()
+    with open(output_rom_file_name, "wb") as binary_file:
+        binary_file.write(patched_rom_bytes)

worlds/zillion/requirements.txt

@@ -1,2 +1,2 @@
-zilliandomizer @ git+https://github.com/beauxq/zilliandomizer@ae00a4b186be897c7cfaf429a0e0ff83c4ecf28c#0.6.0
+zilliandomizer @ git+https://github.com/beauxq/zilliandomizer@b36a23b5a138c78732ac8efb5b5ca8b0be07dcff#0.7.0
 typing-extensions>=4.7, <5