Patch: update to version 4 (#312)

Patch.py

@@ -1,6 +1,7 @@
-# TODO: convert this into a system like AutoWorld
+from __future__ import annotations
 
 import shutil
+import json
 import bsdiff4
 import yaml
 import os
@@ -9,12 +10,155 @@ import threading
 import concurrent.futures
 import zipfile
 import sys
-from typing import Tuple, Optional
+from typing import Tuple, Optional, Dict, Any, Union, BinaryIO
 
 import Utils
 
-current_patch_version = 3
+current_patch_version = 4
 
+
+class AutoPatchRegister(type):
+    patch_types: Dict[str, APDeltaPatch] = {}
+    file_endings: Dict[str, APDeltaPatch] = {}
+
+    def __new__(cls, name: str, bases, dct: Dict[str, Any]):
+        # construct class
+        new_class = super().__new__(cls, name, bases, dct)
+        if "game" in dct:
+            AutoPatchRegister.patch_types[dct["game"]] = new_class
+            if not dct["patch_file_ending"]:
+                raise Exception(f"Need an expected file ending for {name}")
+            AutoPatchRegister.file_endings[dct["patch_file_ending"]] = new_class
+        return new_class
+
+    @staticmethod
+    def get_handler(file: str) -> Optional[type(APDeltaPatch)]:
+        for file_ending, handler in AutoPatchRegister.file_endings.items():
+            if file.endswith(file_ending):
+                return handler
+
+
+class APContainer:
+    """A zipfile containing at least archipelago.json"""
+    version: int = current_patch_version
+    compression_level: int = 9
+    compression_method: int = zipfile.ZIP_DEFLATED
+    game: Optional[str] = None
+
+    # instance attributes:
+    path: Optional[str]
+    player: Optional[int]
+    player_name: str
+    server: str
+
+    def __init__(self, path: Optional[str] = None, player: Optional[int] = None,
+                 player_name: str = "", server: str = ""):
+        self.path = path
+        self.player = player
+        self.player_name = player_name
+        self.server = server
+
+    def write(self, file: Optional[Union[str, BinaryIO]] = None):
+        if not self.path and not file:
+            raise FileNotFoundError(f"Cannot write {self.__class__.__name__} due to no path provided.")
+        with zipfile.ZipFile(file if file else self.path, "w", self.compression_method, True, self.compression_level) \
+                as zf:
+            if file:
+                self.path = zf.filename
+            self.write_contents(zf)
+
+    def write_contents(self, opened_zipfile: zipfile.ZipFile):
+        manifest = self.get_manifest()
+        try:
+            manifest = json.dumps(manifest)
+        except Exception as e:
+            raise Exception(f"Manifest {manifest} did not convert to json.") from e
+        else:
+            opened_zipfile.writestr("archipelago.json", manifest)
+
+    def read(self, file: Optional[Union[str, BinaryIO]] = None):
+        """Read data into patch object. file can be file-like, such as an outer zip file's stream."""
+        if not self.path and not file:
+            raise FileNotFoundError(f"Cannot read {self.__class__.__name__} due to no path provided.")
+        with zipfile.ZipFile(file if file else self.path, "r") as zf:
+            if file:
+                self.path = zf.filename
+            self.read_contents(zf)
+
+    def read_contents(self, opened_zipfile: zipfile.ZipFile):
+        with opened_zipfile.open("archipelago.json", "r") as f:
+            manifest = json.load(f)
+        if manifest["compatible_version"] > self.version:
+            raise Exception(f"File (version: {manifest['compatible_version']}) too new "
+                            f"for this handler (version: {self.version})")
+        self.player = manifest["player"]
+        self.server = manifest["server"]
+        self.player_name = manifest["player_name"]
+
+    def get_manifest(self) -> dict:
+        return {
+            "server": self.server,  # allow immediate connection to server in multiworld. Empty string otherwise
+            "player": self.player,
+            "player_name": self.player_name,
+            "game": self.game,
+            # minimum version of patch system expected for patching to be successful
+            "compatible_version": 4,
+            "version": current_patch_version,
+        }
+
+
+class APDeltaPatch(APContainer, metaclass=AutoPatchRegister):
+    """An APContainer that additionally has delta.bsdiff4
+    containing a delta patch to get the desired file, often a rom."""
+
+    hash = Optional[str]  # base checksum of source file
+    patch_file_ending: str = ""
+    delta: Optional[bytes] = None
+    result_file_ending: str = ".sfc"
+    source_data: bytes
+
+    def __init__(self, *args, patched_path: str = "", **kwargs):
+        self.patched_path = patched_path
+        super(APDeltaPatch, self).__init__(*args, **kwargs)
+
+    def get_manifest(self) -> dict:
+        manifest = super(APDeltaPatch, self).get_manifest()
+        manifest["base_checksum"] = self.hash
+        manifest["result_file_ending"] = self.result_file_ending
+        return manifest
+
+    @classmethod
+    def get_source_data(cls) -> bytes:
+        """Get Base data"""
+        raise NotImplementedError()
+
+    @classmethod
+    def get_source_data_with_cache(cls) -> bytes:
+        if not hasattr(cls, "source_data"):
+            cls.source_data = cls.get_source_data()
+        return cls.source_data
+
+    def write_contents(self, opened_zipfile: zipfile.ZipFile):
+        super(APDeltaPatch, self).write_contents(opened_zipfile)
+        # write Delta
+        opened_zipfile.writestr("delta.bsdiff4",
+                                bsdiff4.diff(self.get_source_data_with_cache(), open(self.patched_path, "rb").read()),
+                                compress_type=zipfile.ZIP_STORED)  # bsdiff4 is a format with integrated compression
+
+    def read_contents(self, opened_zipfile: zipfile.ZipFile):
+        super(APDeltaPatch, self).read_contents(opened_zipfile)
+        self.delta = opened_zipfile.read("delta.bsdiff4")
+
+    def patch(self, target: str):
+        """Base + Delta -> Patched"""
+        if not self.delta:
+            self.read()
+        result = bsdiff4.patch(self.get_source_data_with_cache(), self.delta)
+        with open(target, "wb") as f:
+            f.write(result)
+
+
+# legacy patch handling follows:
 GAME_ALTTP = "A Link to the Past"
 GAME_SM = "Super Metroid"
 GAME_SOE = "Secret of Evermore"
@@ -104,6 +248,15 @@ def get_base_rom_data(game: str):
 
 
 def create_rom_file(patch_file: str) -> Tuple[dict, str]:
+    auto_handler = AutoPatchRegister.get_handler(patch_file)
+    if auto_handler:
+        handler: APDeltaPatch = auto_handler(patch_file)
+        target = os.path.splitext(patch_file)[0]+handler.result_file_ending
+        handler.patch(target)
+        return {"server": handler.server,
+                "player": handler.player,
+                "player_name": handler.player_name}, target
+    else:
         data, target, patched_data = create_rom_bytes(patch_file)
         with open(target, "wb") as f:
             f.write(patched_data)
@@ -233,24 +386,6 @@ if __name__ == "__main__":
                     if 'server' in data:
                         Utils.persistent_store("servers", data['hash'], data['server'])
                         print(f"Host is {data['server']}")
-                elif rom.endswith(".archipelago"):
-                    import json
-                    import zlib
-
-                    with open(rom, 'rb') as fr:
-
-                        multidata = zlib.decompress(fr.read()).decode("utf-8")
-                        with open(rom + '.txt', 'w') as fw:
-                            fw.write(multidata)
-                        multidata = json.loads(multidata)
-                        for romname in multidata['roms']:
-                            Utils.persistent_store("servers", "".join(chr(byte) for byte in romname[2]), address)
-                        from Utils import get_options
-
-                        multidata["server_options"] = get_options()["server_options"]
-                        multidata = zlib.compress(json.dumps(multidata).encode("utf-8"), 9)
-                        with open(rom + "_updated.archipelago", 'wb') as f:
-                            f.write(multidata)
 
                 elif rom.endswith(".zip"):
                     print(f"Updating host in patch files contained in {rom}")

Utils.py

@@ -72,7 +72,7 @@ def is_frozen() -> bool:
     return getattr(sys, 'frozen', False)
 
 
-def local_path(*path):
+def local_path(*path: str):
     if local_path.cached_path:
         return os.path.join(local_path.cached_path, *path)
 

WebHostLib/downloads.py

@@ -1,9 +1,12 @@
+import zipfile
+import json
+from io import BytesIO
+
 from flask import send_file, Response, render_template
 from pony.orm import select
 
-from Patch import update_patch_data, preferred_endings
+from Patch import update_patch_data, preferred_endings, AutoPatchRegister
 from WebHostLib import app, Slot, Room, Seed, cache
-import zipfile
 
 
 @app.route("/dl_patch/<suuid:room_id>/<int:patch_id>")
@@ -12,13 +15,31 @@ def download_patch(room_id, patch_id):
     if not patch:
         return "Patch not found"
     else:
-        import io
-
         room = Room.get(id=room_id)
         last_port = room.last_port
+        filelike = BytesIO(patch.data)
+        greater_than_version_3 = zipfile.is_zipfile(filelike)
+        if greater_than_version_3:
+            # Python's zipfile module cannot overwrite/delete files in a zip, so we recreate the whole thing in ram
+            new_file = BytesIO()
+            with zipfile.ZipFile(filelike, "a") as zf:
+                with zf.open("archipelago.json", "r") as f:
+                    manifest = json.load(f)
+                manifest["server"] = f"{app.config['PATCH_TARGET']}:{last_port}"
+                with zipfile.ZipFile(new_file, "w") as new_zip:
+                    for file in zf.infolist():
+                        if file.filename == "archipelago.json":
+                            new_zip.writestr("archipelago.json", json.dumps(manifest))
+                        else:
+                            new_zip.writestr(file.filename, zf.read(file), file.compress_type, 9)
 
+            fname = f"P{patch.player_id}_{patch.player_name}_{app.jinja_env.filters['suuid'](room_id)}." \
+                    f"{AutoPatchRegister.patch_types[patch.game].patch_file_ending}"
+            new_file.seek(0)
+            return send_file(new_file, as_attachment=True, attachment_filename=fname)
+        else:
             patch_data = update_patch_data(patch.data, server=f"{app.config['PATCH_TARGET']}:{last_port}")
-        patch_data = io.BytesIO(patch_data)
+            patch_data = BytesIO(patch_data)
 
             fname = f"P{patch.player_id}_{patch.player_name}_{app.jinja_env.filters['suuid'](room_id)}." \
                     f"{preferred_endings[patch.game]}"

WebHostLib/upload.py

@@ -5,13 +5,14 @@ import json
 import base64
 import MultiServer
 import uuid
+from io import BytesIO
 
 from flask import request, flash, redirect, url_for, session, render_template
 from pony.orm import flush, select
 
 from WebHostLib import app, Seed, Room, Slot
 from Utils import parse_yaml, VersionException, __version__
-from Patch import preferred_endings
+from Patch import preferred_endings, AutoPatchRegister
 from NetUtils import NetworkSlot, SlotType
 
 banned_zip_contents = (".sfc",)
@@ -25,9 +26,18 @@ def upload_zip_to_db(zfile: zipfile.ZipFile, owner=None, meta={"race": False}, s
     spoiler = ""
     multidata = None
     for file in infolist:
+        handler = AutoPatchRegister.get_handler(file.filename)
         if file.filename.endswith(banned_zip_contents):
             return "Uploaded data contained a rom file, which is likely to contain copyrighted material. " \
                    "Your file was deleted."
+        elif handler:
+            raw = zfile.open(file, "r").read()
+            patch = handler(BytesIO(raw))
+            patch.read()
+            slots.add(Slot(data=raw,
+                           player_name=patch.player_name,
+                           player_id=patch.player,
+                           game=patch.game))
         elif file.filename.endswith(tuple(preferred_endings.values())):
             data = zfile.open(file, "r").read()
             yaml_data = parse_yaml(lzma.decompress(data).decode("utf-8-sig"))
@@ -43,7 +53,8 @@ def upload_zip_to_db(zfile: zipfile.ZipFile, owner=None, meta={"race": False}, s
         elif file.filename.endswith(".apmc"):
             data = zfile.open(file, "r").read()
             metadata = json.loads(base64.b64decode(data).decode("utf-8"))
-            slots.add(Slot(data=data, player_name=metadata["player_name"],
+            slots.add(Slot(data=data,
+                           player_name=metadata["player_name"],
                            player_id=metadata["player_id"],
                            game="Minecraft"))
 
@@ -51,6 +62,7 @@ def upload_zip_to_db(zfile: zipfile.ZipFile, owner=None, meta={"race": False}, s
             _, seed_name, slot_id, slot_name = file.filename.split('.')[0].split('_', 3)
             slots.add(Slot(data=zfile.open(file, "r").read(), player_name=slot_name,
                            player_id=int(slot_id[1:]), game="VVVVVV"))
+
         elif file.filename.endswith(".apsm64ex"):
             _, seed_name, slot_id, slot_name = file.filename.split('.')[0].split('_', 3)
             slots.add(Slot(data=zfile.open(file, "r").read(), player_name=slot_name,
@@ -70,6 +82,7 @@ def upload_zip_to_db(zfile: zipfile.ZipFile, owner=None, meta={"race": False}, s
 
         elif file.filename.endswith(".txt"):
             spoiler = zfile.open(file, "r").read().decode("utf-8-sig")
+
         elif file.filename.endswith(".archipelago"):
             try:
                 multidata = zfile.open(file).read()

worlds/__init__.py

@@ -36,4 +36,5 @@ network_data_package = {
 if any(not world.data_version for world in AutoWorldRegister.world_types.values()):
     network_data_package["version"] = 0
     import logging
-    logging.warning("Datapackage is in custom mode.")
+    logging.warning(f"Datapackage is in custom mode. Custom Worlds: "
+                    f"{[world for world in AutoWorldRegister.world_types.values() if not world.data_version]}")

worlds/alttp/Rom.py

@@ -2884,6 +2884,16 @@ hash_alphabet = [
 ]
 
 
+class LttPDeltaPatch(Patch.APDeltaPatch):
+    hash = JAP10HASH
+    game = "A Link to the Past"
+    patch_file_ending = ".aplttp"
+
+    @classmethod
+    def get_source_data(cls) -> bytes:
+        return get_base_rom_bytes()
+
+
 def get_base_rom_bytes(file_name: str = "") -> bytes:
     base_rom_bytes = getattr(get_base_rom_bytes, "base_rom_bytes", None)
     if not base_rom_bytes:

worlds/alttp/__init__.py

@@ -15,7 +15,7 @@ from .ItemPool import generate_itempool, difficulties
 from .Shops import create_shops, ShopSlotFill
 from .Dungeons import create_dungeons
 from .Rom import LocalRom, patch_rom, patch_race_rom, patch_enemizer, apply_rom_settings, get_hash_string, \
-    get_base_rom_path
+    get_base_rom_path, LttPDeltaPatch
 import Patch
 
 from .InvertedRegions import create_inverted_regions, mark_dark_world_regions
@@ -303,7 +303,9 @@ class ALTTPWorld(World):
 
             rompath = os.path.join(output_directory, f'AP_{world.seed_name}{outfilepname}.sfc')
             rom.write_to_file(rompath)
-            Patch.create_patch_file(rompath, player=player, player_name=world.player_name[player])
+            patch = LttPDeltaPatch(os.path.splitext(rompath)[0]+LttPDeltaPatch.patch_file_ending, player=player,
+                                   player_name=world.player_name[player], patched_path=rompath)
+            patch.write()
             os.unlink(rompath)
             self.rom_name = rom.name
         except:

worlds/factorio/Mod.py

@@ -7,12 +7,14 @@ import threading
 import json
 
 import jinja2
-import Utils
 import shutil
+
+import Utils
+import Patch
 from . import Options
-from BaseClasses import MultiWorld
+
-from .Technologies import tech_table, rocket_recipes, recipes, free_sample_blacklist, progressive_technology_table, \
+from .Technologies import tech_table, recipes, free_sample_blacklist, progressive_technology_table, \
-    base_tech_table, tech_to_progressive_lookup, progressive_tech_table, liquids
+    base_tech_table, tech_to_progressive_lookup, liquids
 
 template_env: Optional[jinja2.Environment] = None
 
@@ -54,6 +56,22 @@ recipe_time_ranges = {
 }
 
 
+class FactorioModFile(Patch.APContainer):
+    game = "Factorio"
+    compression_method = zipfile.ZIP_DEFLATED  # Factorio can't load LZMA archives
+
+    def write_contents(self, opened_zipfile: zipfile.ZipFile):
+        # directory containing Factorio mod has to come first, or Factorio won't recognize this file as a mod.
+        mod_dir = self.path[:-4]  # cut off .zip
+        for root, dirs, files in os.walk(mod_dir):
+            for file in files:
+                opened_zipfile.write(os.path.join(root, file),
+                                     os.path.relpath(os.path.join(root, file),
+                                                     os.path.join(mod_dir, '..')))
+        # now we can add extras.
+        super(FactorioModFile, self).write_contents(opened_zipfile)
+
+
 def generate_mod(world, output_directory: str):
     player = world.player
     multiworld = world.world
@@ -159,10 +177,7 @@ def generate_mod(world, output_directory: str):
 
     # zip the result
     zf_path = os.path.join(mod_dir + ".zip")
-    with zipfile.ZipFile(zf_path, compression=zipfile.ZIP_DEFLATED, mode='w') as zf:
+    mod = FactorioModFile(zf_path, player=player, player_name=multiworld.player_name[player])
-        for root, dirs, files in os.walk(mod_dir):
+    mod.write()
-            for file in files:
+
-                zf.write(os.path.join(root, file),
-                         os.path.relpath(os.path.join(root, file),
-                                         os.path.join(mod_dir, '..')))
     shutil.rmtree(mod_dir)

worlds/sm/Rom.py

@@ -1,11 +1,21 @@
+import hashlib
+import os
+
 import Utils
-from Patch import read_rom
+from Patch import read_rom, APDeltaPatch
 
 JAP10HASH = '21f3e98df4780ee1c667b84e57d88675'
 ROM_PLAYER_LIMIT = 65535
 
-import hashlib
+
-import os
+class SMDeltaPatch(APDeltaPatch):
+    hash = JAP10HASH
+    game = "Super Metroid"
+    patch_file_ending = ".apsm"
+
+    @classmethod
+    def get_source_data(cls) -> bytes:
+        return get_base_rom_bytes()
 
 
 def get_base_rom_bytes(file_name: str = "") -> bytes:
@@ -22,6 +32,7 @@ def get_base_rom_bytes(file_name: str = "") -> bytes:
         get_base_rom_bytes.base_rom_bytes = base_rom_bytes
     return base_rom_bytes
 
+
 def get_base_rom_path(file_name: str = "") -> str:
     options = Utils.get_options()
     if not file_name:

worlds/sm/__init__.py

@@ -2,7 +2,7 @@ import logging
 import copy
 import os
 import threading
-from typing import Set, List
+from typing import Set
 
 logger = logging.getLogger("Super Metroid")
 
@@ -11,12 +11,11 @@ from .Items import lookup_name_to_id as items_lookup_name_to_id
 from .Regions import create_regions
 from .Rules import set_rules, add_entrance_rule
 from .Options import sm_options
-from .Rom import get_base_rom_path, ROM_PLAYER_LIMIT
+from .Rom import get_base_rom_path, ROM_PLAYER_LIMIT, SMDeltaPatch
 import Utils
 
 from BaseClasses import Region, Entrance, Location, MultiWorld, Item, RegionType, CollectionState
 from ..AutoWorld import World, AutoLogicRegister
-import Patch
 
 from logic.smboolmanager import SMBoolManager
 from graph.vanilla.graph_locations import locationsDict
@@ -394,22 +393,24 @@ class SMWorld(World):
         romPatcher.writeRandoSettings(self.variaRando.randoExec.randoSettings, itemLocs)
 
     def generate_output(self, output_directory: str):
-        try:
         outfilebase = 'AP_' + self.world.seed_name
         outfilepname = f'_P{self.player}'
-            outfilepname += f"_{self.world.player_name[self.player].replace(' ', '_')}" \
+        outfilepname += f"_{self.world.player_name[self.player].replace(' ', '_')}"
-
         outputFilename = os.path.join(output_directory, f'{outfilebase}{outfilepname}.sfc')
+
+        try:
             self.variaRando.PatchRom(outputFilename, self.APPatchRom)
-
             self.write_crc(outputFilename)
-
-            Patch.create_patch_file(outputFilename, player=self.player, player_name=self.world.player_name[self.player], game=Patch.GAME_SM)
-            os.unlink(outputFilename)
             self.rom_name = self.romName
         except:
             raise
+        else:
+            patch = SMDeltaPatch(os.path.splitext(outputFilename)[0]+SMDeltaPatch.patch_file_ending, player=self.player,
+                                 player_name=self.world.player_name[self.player], patched_path=outputFilename)
+            patch.write()
         finally:
+            if os.path.exists(outputFilename):
+                os.unlink(outputFilename)
             self.rom_name_available_event.set()  # make sure threading continues and errors are collected
 
     def checksum_mirror_sum(self, start, length, mask = 0x800000):
@@ -427,8 +428,6 @@ class SMWorld(World):
                 next_length += next_length
                 part2 += part2
 
-            length = mask + mask
-
         return (part1 + part2) & 0xFFFF
 
     def write_bytes(self, buffer, startaddress: int, values):
@@ -453,7 +452,6 @@ class SMWorld(World):
             new_name = base64.b64encode(bytes(self.rom_name)).decode()
             multidata["connect_names"][new_name] = multidata["connect_names"][self.world.player_name[self.player]]
 
-
     def fill_slot_data(self): 
         slot_data = {}
         if not self.world.is_race:
@@ -535,10 +533,12 @@ class SMWorld(World):
                 self.world.state.smbm[self.player].onlyBossLeft = True
                 break
 
+
 def create_locations(self, player: int):
     for name, id in locations_lookup_name_to_id.items():
         self.locations[name] = SMLocation(player, name, id)
 
+
 def create_region(self, world: MultiWorld, player: int, name: str, locations=None, exits=None):
     ret = Region(name, RegionType.LightWorld, name, player)
     ret.world = world

worlds/smz3/Rom.py

@@ -1,15 +1,25 @@
+import hashlib
+import os
+
 import Utils
-from Patch import read_rom
+from Patch import read_rom, APDeltaPatch
 
 SMJAP10HASH = '21f3e98df4780ee1c667b84e57d88675'
 LTTPJAP10HASH = '03a63945398191337e896e5771f77173'
 ROM_PLAYER_LIMIT = 256
 
-import hashlib
+
-import os
+class SMZ3DeltaPatch(APDeltaPatch):
+    hash = "3a177ba9879e3dd04fb623a219d175b2"
+    game = "SMZ3"
+    patch_file_ending = ".smz3"
+
+    @classmethod
+    def get_source_data(cls) -> bytes:
+        return get_base_rom_bytes()
 
 
-def get_base_rom_bytes(file_name: str = "") -> bytes:
+def get_base_rom_bytes() -> bytes:
     base_rom_bytes = getattr(get_base_rom_bytes, "base_rom_bytes", None)
     if not base_rom_bytes:
         sm_file_name = get_sm_base_rom_path()
@@ -18,7 +28,7 @@ def get_base_rom_bytes(file_name: str = "") -> bytes:
         basemd5 = hashlib.md5()
         basemd5.update(sm_base_rom_bytes)
         if SMJAP10HASH != basemd5.hexdigest():
-            raise Exception('Supplied Base Rom does not match known MD5 for JAP(1.0) release. '
+            raise Exception('Supplied Base Rom does not match known MD5 for SM JAP(1.0) release. '
                             'Get the correct game and version, then dump it')
         lttp_file_name = get_lttp_base_rom_path()
         lttp_base_rom_bytes = bytes(read_rom(open(lttp_file_name, "rb")))
@@ -26,12 +36,13 @@ def get_base_rom_bytes(file_name: str = "") -> bytes:
         basemd5 = hashlib.md5()
         basemd5.update(lttp_base_rom_bytes)
         if LTTPJAP10HASH != basemd5.hexdigest():
-            raise Exception('Supplied Base Rom does not match known MD5 for JAP(1.0) release. '
+            raise Exception('Supplied Base Rom does not match known MD5 for LttP JAP(1.0) release. '
                             'Get the correct game and version, then dump it')
 
         get_base_rom_bytes.base_rom_bytes = bytes(combine_smz3_rom(sm_base_rom_bytes, lttp_base_rom_bytes))
     return get_base_rom_bytes.base_rom_bytes
 
+
 def get_sm_base_rom_path(file_name: str = "") -> str:
     options = Utils.get_options()
     if not file_name:
@@ -40,6 +51,7 @@ def get_sm_base_rom_path(file_name: str = "") -> str:
         file_name = Utils.local_path(file_name)
     return file_name
 
+
 def get_lttp_base_rom_path(file_name: str = "") -> str:
     options = Utils.get_options()
     if not file_name:
@@ -48,7 +60,8 @@ def get_lttp_base_rom_path(file_name: str = "") -> str:
         file_name = Utils.local_path(file_name)
     return file_name
 
-def combine_smz3_rom(sm_rom: bytes, lttp_rom: bytes):
+
+def combine_smz3_rom(sm_rom: bytes, lttp_rom: bytes) -> bytearray:
     combined = bytearray(0x600000)
     # SM hi bank
     pos = 0

worlds/smz3/__init__.py

@@ -3,25 +3,25 @@ import copy
 import os
 import random
 import threading
-import Patch
 from typing import Dict, Set, TextIO
 
 from BaseClasses import Region, Entrance, Location, MultiWorld, Item, RegionType, CollectionState
-from worlds.generic.Rules import add_rule, set_rule
+from worlds.generic.Rules import set_rule
 import worlds.smz3.TotalSMZ3.Item as TotalSMZ3Item
 from worlds.smz3.TotalSMZ3.World import World as TotalSMZ3World
 from worlds.smz3.TotalSMZ3.Config import Config, GameMode, GanonInvincible, Goal, KeyShuffle, MorphLocation, SMLogic, SwordLocation, Z3Logic
 from worlds.smz3.TotalSMZ3.Location import LocationType, locations_start_id, Location as TotalSMZ3Location
 from worlds.smz3.TotalSMZ3.Patch import Patch as TotalSMZ3Patch, getWord, getWordArray
 from ..AutoWorld import World, AutoLogicRegister
-from .Rom import get_base_rom_bytes
+from .Rom import get_base_rom_bytes, SMZ3DeltaPatch
 from .ips import IPS_Patch
 from .Options import smz3_options
 
 world_folder = os.path.dirname(__file__)
 logger = logging.getLogger("SMZ3")
 
-class SMCollectionState(metaclass=AutoLogicRegister):
+
+class SMZ3CollectionState(metaclass=AutoLogicRegister):
     def init_mixin(self, parent: MultiWorld):
         # for unit tests where MultiWorld is instantiated before worlds
         if hasattr(parent, "state"):
@@ -41,7 +41,7 @@ class SMZ3World(World):
     """
     game: str = "SMZ3"
     topology_present = False
-    data_version = 0
+    data_version = 1
     options = smz3_options
     item_names: Set[str] = frozenset(TotalSMZ3Item.lookup_name_to_id)
     location_names: Set[str]
@@ -208,7 +208,7 @@ class SMZ3World(World):
         return data
 
     def convert_to_lttp_item_name(self, itemName):
-        return bytearray(itemName[:19].center(19, " ")  , 'utf8') + bytearray(0)
+        return bytearray(itemName[:19].center(19, " "), 'utf8') + bytearray(0)
 
     def apply_item_names(self):
         patch = {}
@@ -258,7 +258,9 @@ class SMZ3World(World):
             filename = os.path.join(output_directory, f'{outfilebase}{outfilepname}.sfc')
             with open(filename, "wb") as binary_file:
                 binary_file.write(base_combined_rom)
-            Patch.create_patch_file(filename, player=self.player, player_name=self.world.player_name[self.player], game=Patch.GAME_SMZ3)
+            patch = SMZ3DeltaPatch(os.path.splitext(filename)[0]+SMZ3DeltaPatch.patch_file_ending, player=self.player,
+                                   player_name=self.world.player_name[self.player], patched_path=filename)
+            patch.write()
             os.remove(filename)
             self.rom_name = bytearray(patcher.title, 'utf8')
         except:
@@ -422,12 +424,6 @@ class SMZ3Location(Location):
     def __init__(self, player: int, name: str, address=None, parent=None):
         super(SMZ3Location, self).__init__(player, name, address, parent)
 
-    def can_fill(self, state: CollectionState, item: Item, check_access=True) -> bool:
-        oldItem = self.item
-        self.item = item
-        result = self.always_allow(state, item) or (self.item_rule(item) and (not check_access or self.can_reach(state)))
-        self.item = oldItem
-        return result
 
 class SMZ3Item(Item):
     game = "SMZ3"

worlds/soe/Patch.py

@@ -2,10 +2,25 @@ import bsdiff4
 import yaml
 from typing import Optional
 import Utils
+from Patch import APDeltaPatch
 
 
 USHASH = '6e9c94511d04fac6e0a1e582c170be3a'
-current_patch_version = 2
+
+
+class SoEDeltaPatch(APDeltaPatch):
+    hash = USHASH
+    game = "Secret of Evermore"
+    patch_file_ending = ".apsoe"
+
+    @classmethod
+    def get_source_data(cls) -> bytes:
+        with open(get_base_rom_path(), "rb") as stream:
+            return read_rom(stream)
+
+
+def get_base_rom_path() -> str:
+    return Utils.get_options()['soe_options']['rom_file']
 
 
 def read_rom(stream, strip_header=True) -> bytes:
@@ -17,17 +32,19 @@ def read_rom(stream, strip_header=True) -> bytes:
 
 
 def generate_yaml(patch: bytes, metadata: Optional[dict] = None) -> bytes:
+    """Generate old (<4) apbp format yaml"""
     patch = yaml.dump({"meta": metadata,
                        "patch": patch,
                        "game": "Secret of Evermore",
                        # minimum version of patch system expected for patching to be successful
                        "compatible_version": 1,
-                       "version": current_patch_version,
+                       "version": 2,
                        "base_checksum": USHASH})
     return patch.encode(encoding="utf-8-sig")
 
 
 def generate_patch(vanilla_file, randomized_file, metadata: Optional[dict] = None) -> bytes:
+    """Generate old (<4) apbp format patch data. Run through lzma to get a complete apbp file."""
     with open(vanilla_file, "rb") as f:
         vanilla = read_rom(f)
     with open(randomized_file, "rb") as f:
@@ -39,19 +56,6 @@ def generate_patch(vanilla_file, randomized_file, metadata: Optional[dict] = Non
 
 
 if __name__ == '__main__':
-    import argparse
+    import sys
-    import pathlib
+    print('Please use ../../Patch.py', file=sys.stderr)
-    import lzma
+    sys.exit(1)
-    parser = argparse.ArgumentParser(description='Apply patch to Secret of Evermore.')
-    parser.add_argument('patch', type=pathlib.Path, help='path to .absoe file')
-    args = parser.parse_args()
-    with open(args.patch, "rb") as f:
-        data = Utils.parse_yaml(lzma.decompress(f.read()).decode("utf-8-sig"))
-    if data['game'] != 'Secret of Evermore':
-        raise RuntimeError('Patch is not for Secret of Evermore')
-    with open(Utils.get_options()['soe_options']['rom_file'], 'rb') as f:
-        vanilla_data = read_rom(f)
-    patched_data = bsdiff4.patch(vanilla_data, data["patch"])
-    with open(args.patch.parent / (args.patch.stem + '.sfc'), 'wb') as f:
-        f.write(patched_data)
-

worlds/soe/__init__.py

@@ -3,7 +3,6 @@ from ..generic.Rules import set_rule, add_item_rule
 from BaseClasses import Region, Location, Entrance, Item
 from Utils import get_options, output_path
 import typing
-import lzma
 import os
 import os.path
 import threading
@@ -17,7 +16,7 @@ except ImportError:
 
 from . import Logic  # load logic mixin
 from .Options import soe_options
-from .Patch import generate_patch
+from .Patch import SoEDeltaPatch, get_base_rom_path
 
 """
 In evermizer:
@@ -181,7 +180,7 @@ class SoEWorld(World):
         try:
             money = self.world.money_modifier[self.player].value
             exp = self.world.exp_modifier[self.player].value
-            rom_file = get_options()['soe_options']['rom_file']
+            rom_file = get_base_rom_path()
             out_base = output_path(output_directory, f'AP_{self.world.seed_name}_P{self.player}_{player_name}')
             out_file = out_base + '.sfc'
             placement_file = out_base + '.txt'
@@ -210,13 +209,9 @@ class SoEWorld(World):
             if (pyevermizer.main(rom_file, out_file, placement_file, self.world.seed_name, self.connect_name,
                                  self.evermizer_seed, flags, money, exp)):
                 raise RuntimeError()
-            with lzma.LZMAFile(patch_file, 'wb') as f:
+            patch = SoEDeltaPatch(patch_file, player=self.player,
-                f.write(generate_patch(rom_file, out_file,
+                                  player_name=player_name, patched_path=out_file)
-                                       {
+            patch.write()
-                                           # used by WebHost
-                                           "player_name": self.world.player_name[self.player],
-                                           "player_id": self.player
-                                       }))
         except:
             raise
         finally: