implement binary patching for multimystery (for now no gui/cli support)

MultiClient.py

@@ -934,12 +934,16 @@ async def game_watcher(ctx : Context):
 
 async def main():
     parser = argparse.ArgumentParser()
+    parser.add_argument('diff_file', default="", type=str, nargs="?",
+                        help='Path to a Berserker Multiworld Binary Patch file')
     parser.add_argument('--snes', default='localhost:8080', help='Address of the QUsb2snes server.')
     parser.add_argument('--connect', default=None, help='Address of the multiworld host.')
     parser.add_argument('--password', default=None, help='Password of the multiworld host.')
     parser.add_argument('--loglevel', default='info', choices=['debug', 'info', 'warning', 'error', 'critical'])
     args = parser.parse_args()
-
+    if args.diff_file:
+        import Patch
+        args.connect = Patch.create_rom_file(args.diff_file)["server"]
     logging.basicConfig(format='%(message)s', level=getattr(logging, args.loglevel.upper(), logging.INFO))
 
     ctx = Context(args.snes, args.connect, args.password)

MultiMystery.py

@@ -26,17 +26,22 @@ if __name__ == "__main__":
     try:
         print(f"{__author__}'s MultiMystery Launcher V{__version__}")
         import ModuleUpdate
+
         ModuleUpdate.update()
 
-        from Utils import parse_yaml
+        from Utils import parse_yaml, get_public_ipv4
+        from Patch import create_patch_file
 
-        multi_mystery_options = parse_yaml(open("host.yaml").read())["multi_mystery_options"]
+        options = parse_yaml(open("host.yaml").read())
+
+        multi_mystery_options = options["multi_mystery_options"]
         output_path = multi_mystery_options["output_path"]
         enemizer_path = multi_mystery_options["enemizer_path"]
         player_files_path = multi_mystery_options["player_files_path"]
         race = multi_mystery_options["race"]
         create_spoiler = multi_mystery_options["create_spoiler"]
         zip_roms = multi_mystery_options["zip_roms"]
+        zip_diffs = multi_mystery_options["zip_diffs"]
         zip_spoiler = multi_mystery_options["zip_spoiler"]
         zip_multidata = multi_mystery_options["zip_multidata"]
         zip_format = multi_mystery_options["zip_format"]
@@ -44,7 +49,7 @@ if __name__ == "__main__":
         player_name = multi_mystery_options["player_name"]
         meta_file_path = multi_mystery_options["meta_file_path"]
         teams = multi_mystery_options["teams"]
-        rom_file = multi_mystery_options["rom_file"]
+        rom_file = options["general_options"]["rom_file"]
 
 
         py_version = f"{sys.version_info.major}.{sys.version_info.minor}"
@@ -136,13 +141,19 @@ if __name__ == "__main__":
             zipname = os.path.join(output_path, f"ER_{seedname}.{typical_zip_ending}")
 
             print(f"Creating zipfile {zipname}")
-
+            ipv4 = get_public_ipv4()
             with zipfile.ZipFile(zipname, "w", compression=compression, compresslevel=9) as zf:
                 for file in os.listdir(output_path):
-                    if zip_roms and file.endswith(".sfc") and seedname in file:
-                        pack_file(file)
-                        if zip_roms == 2 and player_name.lower() not in file.lower():
-                            remove_zipped_file(file)
+                    if file.endswith(".sfc") and seedname in file:
+                        if zip_diffs:
+                            diff = os.path.split(create_patch_file(os.path.join(output_path, file), ipv4))[1]
+                            pack_file(diff)
+                            if zip_diffs == 2:
+                                remove_zipped_file(diff)
+                        if zip_roms:
+                            pack_file(file)
+                            if zip_roms == 2 and player_name.lower() not in file.lower():
+                                remove_zipped_file(file)
                 if zip_multidata and os.path.exists(os.path.join(output_path, multidataname)):
                     pack_file(multidataname)
                     if zip_multidata == 2:

MultiServer.py

@@ -3,7 +3,6 @@ import asyncio
 import functools
 import json
 import logging
-import urllib.request
 import zlib
 import collections
 import typing
@@ -613,23 +612,17 @@ async def main():
     except Exception as e:
         logging.error('Failed to read multiworld data (%s)' % e)
         return
-    import socket
-    ip = socket.gethostbyname(socket.gethostname())
-    try:
-        ip = urllib.request.urlopen('https://checkip.amazonaws.com/').read().decode('utf8').strip()
-    except Exception as e:
-        try:
-            ip = urllib.request.urlopen('https://v4.ident.me').read().decode('utf8').strip()
-        except:
-            logging.exception(e)
-            pass # we could be offline, in a local game, so no point in erroring out
 
-    logging.info('Hosting game at %s:%d (%s)' % (ip, ctx.port, 'No password' if not ctx.password else 'Password: %s' % ctx.password))
+    ip = Utils.get_public_ipv4()
+
+    logging.info('Hosting game at %s:%d (%s)' % (
+    ip, ctx.port, 'No password' if not ctx.password else 'Password: %s' % ctx.password))
 
     ctx.disable_save = args.disable_save
     if not ctx.disable_save:
         if not ctx.save_filename:
-            ctx.save_filename = (ctx.data_filename[:-9] if ctx.data_filename[-9:] == 'multidata' else (ctx.data_filename + '_')) + 'multisave'
+            ctx.save_filename = (ctx.data_filename[:-9] if ctx.data_filename[-9:] == 'multidata' else (
+                        ctx.data_filename + '_')) + 'multisave'
         try:
             with open(ctx.save_filename, 'rb') as f:
                 jsonobj = json.loads(zlib.decompress(f.read()).decode("utf-8"))

Patch.py

@@ -0,0 +1,54 @@
+import bsdiff4
+import yaml
+import os
+import lzma
+
+import Utils
+
+base_rom_bytes = None
+
+
+def get_base_rom_bytes() -> bytes:
+    global base_rom_bytes
+    if not base_rom_bytes:
+        with open("host.yaml") as f:
+            options = Utils.parse_yaml(f.read())
+        file_name = options["general_options"]["rom_file"]
+        base_rom_bytes = load_bytes(file_name)
+    return base_rom_bytes
+
+
+def generate_patch(rom: bytes, metadata=None) -> bytes:
+    if metadata is None:
+        metadata = {}
+    patch = bsdiff4.diff(get_base_rom_bytes(), rom)
+    patch = yaml.dump({"meta": metadata,
+                       "patch": patch})
+    return patch.encode()
+
+
+def create_patch_file(rom_file_to_patch: str, server: str = "") -> str:
+    bytes = generate_patch(load_bytes(rom_file_to_patch),
+                           {
+                               "server": server})  # allow immediate connection to server in multiworld. Empty string otherwise
+    target = os.path.splitext(rom_file_to_patch)[0] + ".bbp"
+    write_lzma(bytes, target)
+    return target
+
+
+def create_rom_file(patch_file) -> dict:
+    data = Utils.parse_yaml(lzma.decompress(load_bytes(patch_file)).decode("utf-8-sig"))
+    patched_data = bsdiff4.patch(get_base_rom_bytes(), data["patch"])
+    with open(os.path.splitext(patch_file)[0] + ".sfc", "wb") as f:
+        f.write(patched_data)
+    return data["meta"]
+
+
+def load_bytes(path: str):
+    with open(path, "rb") as f:
+        return f.read()
+
+
+def write_lzma(data: bytes, path: str):
+    with lzma.LZMAFile(path, 'wb') as f:
+        f.write(data)

Utils.py

@@ -151,3 +151,18 @@ class Hint(typing.NamedTuple):
     location: int
     item: int
     found: bool
+
+def get_public_ipv4() -> str:
+    import socket
+    import urllib.request
+    import logging
+    ip = socket.gethostbyname(socket.gethostname())
+    try:
+        ip = urllib.request.urlopen('https://checkip.amazonaws.com/').read().decode('utf8').strip()
+    except Exception as e:
+        try:
+            ip = urllib.request.urlopen('https://v4.ident.me').read().decode('utf8').strip()
+        except:
+            logging.exception(e)
+            pass  # we could be offline, in a local game, so no point in erroring out
+    return ip

host.yaml

@@ -1,6 +1,9 @@
 #options for MultiServer
 #null means nothing, for the server this means to default the value
 #these overwrite command line arguments!
+general_options:
+  #File name of the v1.0 J rom
+  rom_file: "Zelda no Densetsu - Kamigami no Triforce (Japan).sfc"
 server_options:
   host: null
   port: null
@@ -15,11 +18,9 @@ server_options:
   #points given to player for each acquired item
   location_check_points: 1
   #point cost to receive a hint via !hint for players
-  hint_cost: 1000 #set to 0 if you want free hints
+  hint_cost: 50 #set to 0 if you want free hints
 #options for MultiMystery.py
 multi_mystery_options:
-#File name of the v1.0 J rom
-  rom_file: "Zelda no Densetsu - Kamigami no Triforce (Japan).sfc"
 #teams, however, note that there is currently no way to supply names for teams 2+ through MultiMystery
   teams: 1
 #Where to place the resulting files
@@ -31,21 +32,23 @@ multi_mystery_options:
 #meta file name, within players folder
   meta_file_path: "meta.yaml"
 #automatically launches {player_name}.yaml's ROM file using the OS's default program once generation completes. (likely your emulator)
-#does nothing if the name is not found
-#example: player_name = "Berserker"
+  #does nothing if the name is not found
+  #example: player_name = "Berserker"
   player_name: "" # the hosts name
-#create a spoiler file
+  #create a spoiler file
   create_spoiler: 1
-#Zip the resulting roms
-#0 -> Don't
-#1 -> Create a zip
-#2 -> Create a zip and delete the ROMs that will be in it, except the hosts (requires player_name to be set correctly)
+  #Zip the resulting roms
+  #0 -> Don't
+  #1 -> Create a zip
+  #2 -> Create a zip and delete the ROMs that will be in it, except the hosts (requires player_name to be set correctly)
   zip_roms: 1
-#include the spoiler log in the zip, 2 -> delete the non-zipped one
+  # zip diff files
+  zip_diffs: 2
+  #include the spoiler log in the zip, 2 -> delete the non-zipped one
   zip_spoiler: 0
-#include the multidata file in the zip, 2 -> delete the non-zipped one, which also means the server won't autostart
+  #include the multidata file in the zip, 2 -> delete the non-zipped one, which also means the server won't autostart
   zip_multidata: 0
-#zip algorithm to use
+  #zip algorithm to use
   zip_format: 2 # 1 -> zip, 2 -> 7z, 3->bz2
-#create roms flagged as race roms
+  #create roms flagged as race roms
   race: 0

requirements.txt

@@ -3,4 +3,5 @@ colorama>=0.4.3
 websockets>=8.1
 PyYAML>=5.3
 collections_extended>=1.0.3
-fuzzywuzzy>=0.18.0
+fuzzywuzzy>=0.18.0
+bsdiff4>=1.1.9