From 7394598aff667bb92e4356b456fee2c68721ec67 Mon Sep 17 00:00:00 2001 From: Fabian Dill Date: Fri, 18 Mar 2022 04:53:09 +0100 Subject: [PATCH] Patch: update to version 4 (#312) --- Patch.py | 187 +++++++++++++++++++++++++++++++++------ Utils.py | 2 +- WebHostLib/downloads.py | 37 ++++++-- WebHostLib/upload.py | 17 +++- worlds/__init__.py | 3 +- worlds/alttp/Rom.py | 10 +++ worlds/alttp/__init__.py | 6 +- worlds/factorio/Mod.py | 35 +++++--- worlds/sm/Rom.py | 17 +++- worlds/sm/__init__.py | 32 +++---- worlds/smz3/Rom.py | 29 ++++-- worlds/smz3/__init__.py | 22 ++--- worlds/soe/Patch.py | 40 +++++---- worlds/soe/__init__.py | 15 ++-- 14 files changed, 334 insertions(+), 118 deletions(-) diff --git a/Patch.py b/Patch.py index 3cf7d001..0f0ddc18 100644 --- a/Patch.py +++ b/Patch.py @@ -1,6 +1,7 @@ -# TODO: convert this into a system like AutoWorld +from __future__ import annotations import shutil +import json import bsdiff4 import yaml import os @@ -9,12 +10,155 @@ import threading import concurrent.futures import zipfile import sys -from typing import Tuple, Optional +from typing import Tuple, Optional, Dict, Any, Union, BinaryIO import Utils -current_patch_version = 3 +current_patch_version = 4 + +class AutoPatchRegister(type): + patch_types: Dict[str, APDeltaPatch] = {} + file_endings: Dict[str, APDeltaPatch] = {} + + def __new__(cls, name: str, bases, dct: Dict[str, Any]): + # construct class + new_class = super().__new__(cls, name, bases, dct) + if "game" in dct: + AutoPatchRegister.patch_types[dct["game"]] = new_class + if not dct["patch_file_ending"]: + raise Exception(f"Need an expected file ending for {name}") + AutoPatchRegister.file_endings[dct["patch_file_ending"]] = new_class + return new_class + + @staticmethod + def get_handler(file: str) -> Optional[type(APDeltaPatch)]: + for file_ending, handler in AutoPatchRegister.file_endings.items(): + if file.endswith(file_ending): + return handler + + +class APContainer: + """A zipfile containing at least archipelago.json""" + version: int = current_patch_version + compression_level: int = 9 + compression_method: int = zipfile.ZIP_DEFLATED + game: Optional[str] = None + + # instance attributes: + path: Optional[str] + player: Optional[int] + player_name: str + server: str + + def __init__(self, path: Optional[str] = None, player: Optional[int] = None, + player_name: str = "", server: str = ""): + self.path = path + self.player = player + self.player_name = player_name + self.server = server + + def write(self, file: Optional[Union[str, BinaryIO]] = None): + if not self.path and not file: + raise FileNotFoundError(f"Cannot write {self.__class__.__name__} due to no path provided.") + with zipfile.ZipFile(file if file else self.path, "w", self.compression_method, True, self.compression_level) \ + as zf: + if file: + self.path = zf.filename + self.write_contents(zf) + + def write_contents(self, opened_zipfile: zipfile.ZipFile): + manifest = self.get_manifest() + try: + manifest = json.dumps(manifest) + except Exception as e: + raise Exception(f"Manifest {manifest} did not convert to json.") from e + else: + opened_zipfile.writestr("archipelago.json", manifest) + + def read(self, file: Optional[Union[str, BinaryIO]] = None): + """Read data into patch object. file can be file-like, such as an outer zip file's stream.""" + if not self.path and not file: + raise FileNotFoundError(f"Cannot read {self.__class__.__name__} due to no path provided.") + with zipfile.ZipFile(file if file else self.path, "r") as zf: + if file: + self.path = zf.filename + self.read_contents(zf) + + def read_contents(self, opened_zipfile: zipfile.ZipFile): + with opened_zipfile.open("archipelago.json", "r") as f: + manifest = json.load(f) + if manifest["compatible_version"] > self.version: + raise Exception(f"File (version: {manifest['compatible_version']}) too new " + f"for this handler (version: {self.version})") + self.player = manifest["player"] + self.server = manifest["server"] + self.player_name = manifest["player_name"] + + def get_manifest(self) -> dict: + return { + "server": self.server, # allow immediate connection to server in multiworld. Empty string otherwise + "player": self.player, + "player_name": self.player_name, + "game": self.game, + # minimum version of patch system expected for patching to be successful + "compatible_version": 4, + "version": current_patch_version, + } + + +class APDeltaPatch(APContainer, metaclass=AutoPatchRegister): + """An APContainer that additionally has delta.bsdiff4 + containing a delta patch to get the desired file, often a rom.""" + + hash = Optional[str] # base checksum of source file + patch_file_ending: str = "" + delta: Optional[bytes] = None + result_file_ending: str = ".sfc" + source_data: bytes + + def __init__(self, *args, patched_path: str = "", **kwargs): + self.patched_path = patched_path + super(APDeltaPatch, self).__init__(*args, **kwargs) + + def get_manifest(self) -> dict: + manifest = super(APDeltaPatch, self).get_manifest() + manifest["base_checksum"] = self.hash + manifest["result_file_ending"] = self.result_file_ending + return manifest + + @classmethod + def get_source_data(cls) -> bytes: + """Get Base data""" + raise NotImplementedError() + + @classmethod + def get_source_data_with_cache(cls) -> bytes: + if not hasattr(cls, "source_data"): + cls.source_data = cls.get_source_data() + return cls.source_data + + def write_contents(self, opened_zipfile: zipfile.ZipFile): + super(APDeltaPatch, self).write_contents(opened_zipfile) + # write Delta + opened_zipfile.writestr("delta.bsdiff4", + bsdiff4.diff(self.get_source_data_with_cache(), open(self.patched_path, "rb").read()), + compress_type=zipfile.ZIP_STORED) # bsdiff4 is a format with integrated compression + + def read_contents(self, opened_zipfile: zipfile.ZipFile): + super(APDeltaPatch, self).read_contents(opened_zipfile) + self.delta = opened_zipfile.read("delta.bsdiff4") + + def patch(self, target: str): + """Base + Delta -> Patched""" + if not self.delta: + self.read() + result = bsdiff4.patch(self.get_source_data_with_cache(), self.delta) + with open(target, "wb") as f: + f.write(result) + + +# legacy patch handling follows: GAME_ALTTP = "A Link to the Past" GAME_SM = "Super Metroid" GAME_SOE = "Secret of Evermore" @@ -104,10 +248,19 @@ def get_base_rom_data(game: str): def create_rom_file(patch_file: str) -> Tuple[dict, str]: - data, target, patched_data = create_rom_bytes(patch_file) - with open(target, "wb") as f: - f.write(patched_data) - return data, target + auto_handler = AutoPatchRegister.get_handler(patch_file) + if auto_handler: + handler: APDeltaPatch = auto_handler(patch_file) + target = os.path.splitext(patch_file)[0]+handler.result_file_ending + handler.patch(target) + return {"server": handler.server, + "player": handler.player, + "player_name": handler.player_name}, target + else: + data, target, patched_data = create_rom_bytes(patch_file) + with open(target, "wb") as f: + f.write(patched_data) + return data, target def update_patch_data(patch_data: bytes, server: str = "") -> bytes: @@ -233,24 +386,6 @@ if __name__ == "__main__": if 'server' in data: Utils.persistent_store("servers", data['hash'], data['server']) print(f"Host is {data['server']}") - elif rom.endswith(".archipelago"): - import json - import zlib - - with open(rom, 'rb') as fr: - - multidata = zlib.decompress(fr.read()).decode("utf-8") - with open(rom + '.txt', 'w') as fw: - fw.write(multidata) - multidata = json.loads(multidata) - for romname in multidata['roms']: - Utils.persistent_store("servers", "".join(chr(byte) for byte in romname[2]), address) - from Utils import get_options - - multidata["server_options"] = get_options()["server_options"] - multidata = zlib.compress(json.dumps(multidata).encode("utf-8"), 9) - with open(rom + "_updated.archipelago", 'wb') as f: - f.write(multidata) elif rom.endswith(".zip"): print(f"Updating host in patch files contained in {rom}") @@ -279,4 +414,4 @@ if __name__ == "__main__": import traceback traceback.print_exc() - input("Press enter to close.") \ No newline at end of file + input("Press enter to close.") diff --git a/Utils.py b/Utils.py index 05b55e48..63bc7ac0 100644 --- a/Utils.py +++ b/Utils.py @@ -72,7 +72,7 @@ def is_frozen() -> bool: return getattr(sys, 'frozen', False) -def local_path(*path): +def local_path(*path: str): if local_path.cached_path: return os.path.join(local_path.cached_path, *path) diff --git a/WebHostLib/downloads.py b/WebHostLib/downloads.py index c81c32e3..0df536a4 100644 --- a/WebHostLib/downloads.py +++ b/WebHostLib/downloads.py @@ -1,9 +1,12 @@ +import zipfile +import json +from io import BytesIO + from flask import send_file, Response, render_template from pony.orm import select -from Patch import update_patch_data, preferred_endings +from Patch import update_patch_data, preferred_endings, AutoPatchRegister from WebHostLib import app, Slot, Room, Seed, cache -import zipfile @app.route("/dl_patch//") @@ -12,16 +15,34 @@ def download_patch(room_id, patch_id): if not patch: return "Patch not found" else: - import io - room = Room.get(id=room_id) last_port = room.last_port + filelike = BytesIO(patch.data) + greater_than_version_3 = zipfile.is_zipfile(filelike) + if greater_than_version_3: + # Python's zipfile module cannot overwrite/delete files in a zip, so we recreate the whole thing in ram + new_file = BytesIO() + with zipfile.ZipFile(filelike, "a") as zf: + with zf.open("archipelago.json", "r") as f: + manifest = json.load(f) + manifest["server"] = f"{app.config['PATCH_TARGET']}:{last_port}" + with zipfile.ZipFile(new_file, "w") as new_zip: + for file in zf.infolist(): + if file.filename == "archipelago.json": + new_zip.writestr("archipelago.json", json.dumps(manifest)) + else: + new_zip.writestr(file.filename, zf.read(file), file.compress_type, 9) - patch_data = update_patch_data(patch.data, server=f"{app.config['PATCH_TARGET']}:{last_port}") - patch_data = io.BytesIO(patch_data) + fname = f"P{patch.player_id}_{patch.player_name}_{app.jinja_env.filters['suuid'](room_id)}." \ + f"{AutoPatchRegister.patch_types[patch.game].patch_file_ending}" + new_file.seek(0) + return send_file(new_file, as_attachment=True, attachment_filename=fname) + else: + patch_data = update_patch_data(patch.data, server=f"{app.config['PATCH_TARGET']}:{last_port}") + patch_data = BytesIO(patch_data) - fname = f"P{patch.player_id}_{patch.player_name}_{app.jinja_env.filters['suuid'](room_id)}." \ - f"{preferred_endings[patch.game]}" + fname = f"P{patch.player_id}_{patch.player_name}_{app.jinja_env.filters['suuid'](room_id)}." \ + f"{preferred_endings[patch.game]}" return send_file(patch_data, as_attachment=True, attachment_filename=fname) diff --git a/WebHostLib/upload.py b/WebHostLib/upload.py index 4e3095e1..e6b2c7de 100644 --- a/WebHostLib/upload.py +++ b/WebHostLib/upload.py @@ -5,13 +5,14 @@ import json import base64 import MultiServer import uuid +from io import BytesIO from flask import request, flash, redirect, url_for, session, render_template from pony.orm import flush, select from WebHostLib import app, Seed, Room, Slot from Utils import parse_yaml, VersionException, __version__ -from Patch import preferred_endings +from Patch import preferred_endings, AutoPatchRegister from NetUtils import NetworkSlot, SlotType banned_zip_contents = (".sfc",) @@ -25,9 +26,18 @@ def upload_zip_to_db(zfile: zipfile.ZipFile, owner=None, meta={"race": False}, s spoiler = "" multidata = None for file in infolist: + handler = AutoPatchRegister.get_handler(file.filename) if file.filename.endswith(banned_zip_contents): return "Uploaded data contained a rom file, which is likely to contain copyrighted material. " \ "Your file was deleted." + elif handler: + raw = zfile.open(file, "r").read() + patch = handler(BytesIO(raw)) + patch.read() + slots.add(Slot(data=raw, + player_name=patch.player_name, + player_id=patch.player, + game=patch.game)) elif file.filename.endswith(tuple(preferred_endings.values())): data = zfile.open(file, "r").read() yaml_data = parse_yaml(lzma.decompress(data).decode("utf-8-sig")) @@ -43,7 +53,8 @@ def upload_zip_to_db(zfile: zipfile.ZipFile, owner=None, meta={"race": False}, s elif file.filename.endswith(".apmc"): data = zfile.open(file, "r").read() metadata = json.loads(base64.b64decode(data).decode("utf-8")) - slots.add(Slot(data=data, player_name=metadata["player_name"], + slots.add(Slot(data=data, + player_name=metadata["player_name"], player_id=metadata["player_id"], game="Minecraft")) @@ -51,6 +62,7 @@ def upload_zip_to_db(zfile: zipfile.ZipFile, owner=None, meta={"race": False}, s _, seed_name, slot_id, slot_name = file.filename.split('.')[0].split('_', 3) slots.add(Slot(data=zfile.open(file, "r").read(), player_name=slot_name, player_id=int(slot_id[1:]), game="VVVVVV")) + elif file.filename.endswith(".apsm64ex"): _, seed_name, slot_id, slot_name = file.filename.split('.')[0].split('_', 3) slots.add(Slot(data=zfile.open(file, "r").read(), player_name=slot_name, @@ -70,6 +82,7 @@ def upload_zip_to_db(zfile: zipfile.ZipFile, owner=None, meta={"race": False}, s elif file.filename.endswith(".txt"): spoiler = zfile.open(file, "r").read().decode("utf-8-sig") + elif file.filename.endswith(".archipelago"): try: multidata = zfile.open(file).read() diff --git a/worlds/__init__.py b/worlds/__init__.py index 4ab5b473..d85c67c3 100644 --- a/worlds/__init__.py +++ b/worlds/__init__.py @@ -36,4 +36,5 @@ network_data_package = { if any(not world.data_version for world in AutoWorldRegister.world_types.values()): network_data_package["version"] = 0 import logging - logging.warning("Datapackage is in custom mode.") + logging.warning(f"Datapackage is in custom mode. Custom Worlds: " + f"{[world for world in AutoWorldRegister.world_types.values() if not world.data_version]}") diff --git a/worlds/alttp/Rom.py b/worlds/alttp/Rom.py index bc3183ee..bcde3473 100644 --- a/worlds/alttp/Rom.py +++ b/worlds/alttp/Rom.py @@ -2884,6 +2884,16 @@ hash_alphabet = [ ] +class LttPDeltaPatch(Patch.APDeltaPatch): + hash = JAP10HASH + game = "A Link to the Past" + patch_file_ending = ".aplttp" + + @classmethod + def get_source_data(cls) -> bytes: + return get_base_rom_bytes() + + def get_base_rom_bytes(file_name: str = "") -> bytes: base_rom_bytes = getattr(get_base_rom_bytes, "base_rom_bytes", None) if not base_rom_bytes: diff --git a/worlds/alttp/__init__.py b/worlds/alttp/__init__.py index 290710fd..512e0c12 100644 --- a/worlds/alttp/__init__.py +++ b/worlds/alttp/__init__.py @@ -15,7 +15,7 @@ from .ItemPool import generate_itempool, difficulties from .Shops import create_shops, ShopSlotFill from .Dungeons import create_dungeons from .Rom import LocalRom, patch_rom, patch_race_rom, patch_enemizer, apply_rom_settings, get_hash_string, \ - get_base_rom_path + get_base_rom_path, LttPDeltaPatch import Patch from .InvertedRegions import create_inverted_regions, mark_dark_world_regions @@ -303,7 +303,9 @@ class ALTTPWorld(World): rompath = os.path.join(output_directory, f'AP_{world.seed_name}{outfilepname}.sfc') rom.write_to_file(rompath) - Patch.create_patch_file(rompath, player=player, player_name=world.player_name[player]) + patch = LttPDeltaPatch(os.path.splitext(rompath)[0]+LttPDeltaPatch.patch_file_ending, player=player, + player_name=world.player_name[player], patched_path=rompath) + patch.write() os.unlink(rompath) self.rom_name = rom.name except: diff --git a/worlds/factorio/Mod.py b/worlds/factorio/Mod.py index be983701..446bdab1 100644 --- a/worlds/factorio/Mod.py +++ b/worlds/factorio/Mod.py @@ -7,12 +7,14 @@ import threading import json import jinja2 -import Utils import shutil + +import Utils +import Patch from . import Options -from BaseClasses import MultiWorld -from .Technologies import tech_table, rocket_recipes, recipes, free_sample_blacklist, progressive_technology_table, \ - base_tech_table, tech_to_progressive_lookup, progressive_tech_table, liquids + +from .Technologies import tech_table, recipes, free_sample_blacklist, progressive_technology_table, \ + base_tech_table, tech_to_progressive_lookup, liquids template_env: Optional[jinja2.Environment] = None @@ -54,6 +56,22 @@ recipe_time_ranges = { } +class FactorioModFile(Patch.APContainer): + game = "Factorio" + compression_method = zipfile.ZIP_DEFLATED # Factorio can't load LZMA archives + + def write_contents(self, opened_zipfile: zipfile.ZipFile): + # directory containing Factorio mod has to come first, or Factorio won't recognize this file as a mod. + mod_dir = self.path[:-4] # cut off .zip + for root, dirs, files in os.walk(mod_dir): + for file in files: + opened_zipfile.write(os.path.join(root, file), + os.path.relpath(os.path.join(root, file), + os.path.join(mod_dir, '..'))) + # now we can add extras. + super(FactorioModFile, self).write_contents(opened_zipfile) + + def generate_mod(world, output_directory: str): player = world.player multiworld = world.world @@ -159,10 +177,7 @@ def generate_mod(world, output_directory: str): # zip the result zf_path = os.path.join(mod_dir + ".zip") - with zipfile.ZipFile(zf_path, compression=zipfile.ZIP_DEFLATED, mode='w') as zf: - for root, dirs, files in os.walk(mod_dir): - for file in files: - zf.write(os.path.join(root, file), - os.path.relpath(os.path.join(root, file), - os.path.join(mod_dir, '..'))) + mod = FactorioModFile(zf_path, player=player, player_name=multiworld.player_name[player]) + mod.write() + shutil.rmtree(mod_dir) diff --git a/worlds/sm/Rom.py b/worlds/sm/Rom.py index 3e191c2e..e982258d 100644 --- a/worlds/sm/Rom.py +++ b/worlds/sm/Rom.py @@ -1,11 +1,21 @@ +import hashlib +import os + import Utils -from Patch import read_rom +from Patch import read_rom, APDeltaPatch JAP10HASH = '21f3e98df4780ee1c667b84e57d88675' ROM_PLAYER_LIMIT = 65535 -import hashlib -import os + +class SMDeltaPatch(APDeltaPatch): + hash = JAP10HASH + game = "Super Metroid" + patch_file_ending = ".apsm" + + @classmethod + def get_source_data(cls) -> bytes: + return get_base_rom_bytes() def get_base_rom_bytes(file_name: str = "") -> bytes: @@ -22,6 +32,7 @@ def get_base_rom_bytes(file_name: str = "") -> bytes: get_base_rom_bytes.base_rom_bytes = base_rom_bytes return base_rom_bytes + def get_base_rom_path(file_name: str = "") -> str: options = Utils.get_options() if not file_name: diff --git a/worlds/sm/__init__.py b/worlds/sm/__init__.py index faccbe1f..59d6d780 100644 --- a/worlds/sm/__init__.py +++ b/worlds/sm/__init__.py @@ -2,7 +2,7 @@ import logging import copy import os import threading -from typing import Set, List +from typing import Set logger = logging.getLogger("Super Metroid") @@ -11,12 +11,11 @@ from .Items import lookup_name_to_id as items_lookup_name_to_id from .Regions import create_regions from .Rules import set_rules, add_entrance_rule from .Options import sm_options -from .Rom import get_base_rom_path, ROM_PLAYER_LIMIT +from .Rom import get_base_rom_path, ROM_PLAYER_LIMIT, SMDeltaPatch import Utils from BaseClasses import Region, Entrance, Location, MultiWorld, Item, RegionType, CollectionState from ..AutoWorld import World, AutoLogicRegister -import Patch from logic.smboolmanager import SMBoolManager from graph.vanilla.graph_locations import locationsDict @@ -394,23 +393,25 @@ class SMWorld(World): romPatcher.writeRandoSettings(self.variaRando.randoExec.randoSettings, itemLocs) def generate_output(self, output_directory: str): + outfilebase = 'AP_' + self.world.seed_name + outfilepname = f'_P{self.player}' + outfilepname += f"_{self.world.player_name[self.player].replace(' ', '_')}" + outputFilename = os.path.join(output_directory, f'{outfilebase}{outfilepname}.sfc') + try: - outfilebase = 'AP_' + self.world.seed_name - outfilepname = f'_P{self.player}' - outfilepname += f"_{self.world.player_name[self.player].replace(' ', '_')}" \ - - outputFilename = os.path.join(output_directory, f'{outfilebase}{outfilepname}.sfc') self.variaRando.PatchRom(outputFilename, self.APPatchRom) - self.write_crc(outputFilename) - - Patch.create_patch_file(outputFilename, player=self.player, player_name=self.world.player_name[self.player], game=Patch.GAME_SM) - os.unlink(outputFilename) self.rom_name = self.romName except: raise + else: + patch = SMDeltaPatch(os.path.splitext(outputFilename)[0]+SMDeltaPatch.patch_file_ending, player=self.player, + player_name=self.world.player_name[self.player], patched_path=outputFilename) + patch.write() finally: - self.rom_name_available_event.set() # make sure threading continues and errors are collected + if os.path.exists(outputFilename): + os.unlink(outputFilename) + self.rom_name_available_event.set() # make sure threading continues and errors are collected def checksum_mirror_sum(self, start, length, mask = 0x800000): while (not(length & mask) and mask): @@ -427,8 +428,6 @@ class SMWorld(World): next_length += next_length part2 += part2 - length = mask + mask - return (part1 + part2) & 0xFFFF def write_bytes(self, buffer, startaddress: int, values): @@ -453,7 +452,6 @@ class SMWorld(World): new_name = base64.b64encode(bytes(self.rom_name)).decode() multidata["connect_names"][new_name] = multidata["connect_names"][self.world.player_name[self.player]] - def fill_slot_data(self): slot_data = {} if not self.world.is_race: @@ -535,10 +533,12 @@ class SMWorld(World): self.world.state.smbm[self.player].onlyBossLeft = True break + def create_locations(self, player: int): for name, id in locations_lookup_name_to_id.items(): self.locations[name] = SMLocation(player, name, id) + def create_region(self, world: MultiWorld, player: int, name: str, locations=None, exits=None): ret = Region(name, RegionType.LightWorld, name, player) ret.world = world diff --git a/worlds/smz3/Rom.py b/worlds/smz3/Rom.py index eb78697b..83f4d396 100644 --- a/worlds/smz3/Rom.py +++ b/worlds/smz3/Rom.py @@ -1,15 +1,25 @@ +import hashlib +import os + import Utils -from Patch import read_rom +from Patch import read_rom, APDeltaPatch SMJAP10HASH = '21f3e98df4780ee1c667b84e57d88675' LTTPJAP10HASH = '03a63945398191337e896e5771f77173' ROM_PLAYER_LIMIT = 256 -import hashlib -import os + +class SMZ3DeltaPatch(APDeltaPatch): + hash = "3a177ba9879e3dd04fb623a219d175b2" + game = "SMZ3" + patch_file_ending = ".smz3" + + @classmethod + def get_source_data(cls) -> bytes: + return get_base_rom_bytes() -def get_base_rom_bytes(file_name: str = "") -> bytes: +def get_base_rom_bytes() -> bytes: base_rom_bytes = getattr(get_base_rom_bytes, "base_rom_bytes", None) if not base_rom_bytes: sm_file_name = get_sm_base_rom_path() @@ -18,7 +28,7 @@ def get_base_rom_bytes(file_name: str = "") -> bytes: basemd5 = hashlib.md5() basemd5.update(sm_base_rom_bytes) if SMJAP10HASH != basemd5.hexdigest(): - raise Exception('Supplied Base Rom does not match known MD5 for JAP(1.0) release. ' + raise Exception('Supplied Base Rom does not match known MD5 for SM JAP(1.0) release. ' 'Get the correct game and version, then dump it') lttp_file_name = get_lttp_base_rom_path() lttp_base_rom_bytes = bytes(read_rom(open(lttp_file_name, "rb"))) @@ -26,12 +36,13 @@ def get_base_rom_bytes(file_name: str = "") -> bytes: basemd5 = hashlib.md5() basemd5.update(lttp_base_rom_bytes) if LTTPJAP10HASH != basemd5.hexdigest(): - raise Exception('Supplied Base Rom does not match known MD5 for JAP(1.0) release. ' + raise Exception('Supplied Base Rom does not match known MD5 for LttP JAP(1.0) release. ' 'Get the correct game and version, then dump it') - + get_base_rom_bytes.base_rom_bytes = bytes(combine_smz3_rom(sm_base_rom_bytes, lttp_base_rom_bytes)) return get_base_rom_bytes.base_rom_bytes + def get_sm_base_rom_path(file_name: str = "") -> str: options = Utils.get_options() if not file_name: @@ -40,6 +51,7 @@ def get_sm_base_rom_path(file_name: str = "") -> str: file_name = Utils.local_path(file_name) return file_name + def get_lttp_base_rom_path(file_name: str = "") -> str: options = Utils.get_options() if not file_name: @@ -48,7 +60,8 @@ def get_lttp_base_rom_path(file_name: str = "") -> str: file_name = Utils.local_path(file_name) return file_name -def combine_smz3_rom(sm_rom: bytes, lttp_rom: bytes): + +def combine_smz3_rom(sm_rom: bytes, lttp_rom: bytes) -> bytearray: combined = bytearray(0x600000) # SM hi bank pos = 0 diff --git a/worlds/smz3/__init__.py b/worlds/smz3/__init__.py index adb79d11..b0a5993b 100644 --- a/worlds/smz3/__init__.py +++ b/worlds/smz3/__init__.py @@ -3,25 +3,25 @@ import copy import os import random import threading -import Patch from typing import Dict, Set, TextIO from BaseClasses import Region, Entrance, Location, MultiWorld, Item, RegionType, CollectionState -from worlds.generic.Rules import add_rule, set_rule +from worlds.generic.Rules import set_rule import worlds.smz3.TotalSMZ3.Item as TotalSMZ3Item from worlds.smz3.TotalSMZ3.World import World as TotalSMZ3World from worlds.smz3.TotalSMZ3.Config import Config, GameMode, GanonInvincible, Goal, KeyShuffle, MorphLocation, SMLogic, SwordLocation, Z3Logic from worlds.smz3.TotalSMZ3.Location import LocationType, locations_start_id, Location as TotalSMZ3Location from worlds.smz3.TotalSMZ3.Patch import Patch as TotalSMZ3Patch, getWord, getWordArray from ..AutoWorld import World, AutoLogicRegister -from .Rom import get_base_rom_bytes +from .Rom import get_base_rom_bytes, SMZ3DeltaPatch from .ips import IPS_Patch from .Options import smz3_options world_folder = os.path.dirname(__file__) logger = logging.getLogger("SMZ3") -class SMCollectionState(metaclass=AutoLogicRegister): + +class SMZ3CollectionState(metaclass=AutoLogicRegister): def init_mixin(self, parent: MultiWorld): # for unit tests where MultiWorld is instantiated before worlds if hasattr(parent, "state"): @@ -41,7 +41,7 @@ class SMZ3World(World): """ game: str = "SMZ3" topology_present = False - data_version = 0 + data_version = 1 options = smz3_options item_names: Set[str] = frozenset(TotalSMZ3Item.lookup_name_to_id) location_names: Set[str] @@ -208,7 +208,7 @@ class SMZ3World(World): return data def convert_to_lttp_item_name(self, itemName): - return bytearray(itemName[:19].center(19, " ") , 'utf8') + bytearray(0) + return bytearray(itemName[:19].center(19, " "), 'utf8') + bytearray(0) def apply_item_names(self): patch = {} @@ -258,7 +258,9 @@ class SMZ3World(World): filename = os.path.join(output_directory, f'{outfilebase}{outfilepname}.sfc') with open(filename, "wb") as binary_file: binary_file.write(base_combined_rom) - Patch.create_patch_file(filename, player=self.player, player_name=self.world.player_name[self.player], game=Patch.GAME_SMZ3) + patch = SMZ3DeltaPatch(os.path.splitext(filename)[0]+SMZ3DeltaPatch.patch_file_ending, player=self.player, + player_name=self.world.player_name[self.player], patched_path=filename) + patch.write() os.remove(filename) self.rom_name = bytearray(patcher.title, 'utf8') except: @@ -422,12 +424,6 @@ class SMZ3Location(Location): def __init__(self, player: int, name: str, address=None, parent=None): super(SMZ3Location, self).__init__(player, name, address, parent) - def can_fill(self, state: CollectionState, item: Item, check_access=True) -> bool: - oldItem = self.item - self.item = item - result = self.always_allow(state, item) or (self.item_rule(item) and (not check_access or self.can_reach(state))) - self.item = oldItem - return result class SMZ3Item(Item): game = "SMZ3" diff --git a/worlds/soe/Patch.py b/worlds/soe/Patch.py index 0812c3f1..5478e935 100644 --- a/worlds/soe/Patch.py +++ b/worlds/soe/Patch.py @@ -2,10 +2,25 @@ import bsdiff4 import yaml from typing import Optional import Utils +from Patch import APDeltaPatch USHASH = '6e9c94511d04fac6e0a1e582c170be3a' -current_patch_version = 2 + + +class SoEDeltaPatch(APDeltaPatch): + hash = USHASH + game = "Secret of Evermore" + patch_file_ending = ".apsoe" + + @classmethod + def get_source_data(cls) -> bytes: + with open(get_base_rom_path(), "rb") as stream: + return read_rom(stream) + + +def get_base_rom_path() -> str: + return Utils.get_options()['soe_options']['rom_file'] def read_rom(stream, strip_header=True) -> bytes: @@ -17,17 +32,19 @@ def read_rom(stream, strip_header=True) -> bytes: def generate_yaml(patch: bytes, metadata: Optional[dict] = None) -> bytes: + """Generate old (<4) apbp format yaml""" patch = yaml.dump({"meta": metadata, "patch": patch, "game": "Secret of Evermore", # minimum version of patch system expected for patching to be successful "compatible_version": 1, - "version": current_patch_version, + "version": 2, "base_checksum": USHASH}) return patch.encode(encoding="utf-8-sig") def generate_patch(vanilla_file, randomized_file, metadata: Optional[dict] = None) -> bytes: + """Generate old (<4) apbp format patch data. Run through lzma to get a complete apbp file.""" with open(vanilla_file, "rb") as f: vanilla = read_rom(f) with open(randomized_file, "rb") as f: @@ -39,19 +56,6 @@ def generate_patch(vanilla_file, randomized_file, metadata: Optional[dict] = Non if __name__ == '__main__': - import argparse - import pathlib - import lzma - parser = argparse.ArgumentParser(description='Apply patch to Secret of Evermore.') - parser.add_argument('patch', type=pathlib.Path, help='path to .absoe file') - args = parser.parse_args() - with open(args.patch, "rb") as f: - data = Utils.parse_yaml(lzma.decompress(f.read()).decode("utf-8-sig")) - if data['game'] != 'Secret of Evermore': - raise RuntimeError('Patch is not for Secret of Evermore') - with open(Utils.get_options()['soe_options']['rom_file'], 'rb') as f: - vanilla_data = read_rom(f) - patched_data = bsdiff4.patch(vanilla_data, data["patch"]) - with open(args.patch.parent / (args.patch.stem + '.sfc'), 'wb') as f: - f.write(patched_data) - + import sys + print('Please use ../../Patch.py', file=sys.stderr) + sys.exit(1) diff --git a/worlds/soe/__init__.py b/worlds/soe/__init__.py index 2f977d83..b1fa4d80 100644 --- a/worlds/soe/__init__.py +++ b/worlds/soe/__init__.py @@ -3,7 +3,6 @@ from ..generic.Rules import set_rule, add_item_rule from BaseClasses import Region, Location, Entrance, Item from Utils import get_options, output_path import typing -import lzma import os import os.path import threading @@ -17,7 +16,7 @@ except ImportError: from . import Logic # load logic mixin from .Options import soe_options -from .Patch import generate_patch +from .Patch import SoEDeltaPatch, get_base_rom_path """ In evermizer: @@ -181,7 +180,7 @@ class SoEWorld(World): try: money = self.world.money_modifier[self.player].value exp = self.world.exp_modifier[self.player].value - rom_file = get_options()['soe_options']['rom_file'] + rom_file = get_base_rom_path() out_base = output_path(output_directory, f'AP_{self.world.seed_name}_P{self.player}_{player_name}') out_file = out_base + '.sfc' placement_file = out_base + '.txt' @@ -210,13 +209,9 @@ class SoEWorld(World): if (pyevermizer.main(rom_file, out_file, placement_file, self.world.seed_name, self.connect_name, self.evermizer_seed, flags, money, exp)): raise RuntimeError() - with lzma.LZMAFile(patch_file, 'wb') as f: - f.write(generate_patch(rom_file, out_file, - { - # used by WebHost - "player_name": self.world.player_name[self.player], - "player_id": self.player - })) + patch = SoEDeltaPatch(patch_file, player=self.player, + player_name=player_name, patched_path=out_file) + patch.write() except: raise finally: