Merge branch 'bsdiff4_baserom'

# Conflicts:
#	Main.py

AdjusterMain.py

@@ -13,9 +13,6 @@ def adjust(args):
 
     if os.stat(args.rom).st_size in (0x200000, 0x400000) and os.path.splitext(args.rom)[-1].lower() == '.sfc':
         rom = LocalRom(args.rom, patch=False)
-        if os.path.isfile(args.baserom):
-            baserom = LocalRom(args.baserom, patch=True)
-            rom.orig_buffer = baserom.orig_buffer
     else:
         raise RuntimeError(
             'Provided Rom is not a valid Link to the Past Randomizer Rom. Please provide one for adjusting.')

EntranceRandomizer.py

@@ -280,10 +280,6 @@ def parse_arguments(argv, no_defaults=False):
                              ''')
     parser.add_argument('--suppress_rom', help='Do not create an output rom file.', action='store_true')
     parser.add_argument('--gui', help='Launch the GUI', action='store_true')
-    parser.add_argument('--jsonout', action='store_true', help='''\
-                            Output .json patch to stdout instead of a patched rom. Used
-                            for VT site integration, do not use otherwise.
-                            ''')
     parser.add_argument('--skip_progression_balancing', action='store_true', default=defval(False),
                         help="Skip Multiworld Progression balancing.")
     parser.add_argument('--skip_playthrough', action='store_true', default=defval(False))
@@ -302,8 +298,7 @@ def parse_arguments(argv, no_defaults=False):
     parser.add_argument('--race', default=defval(False), action='store_true')
     parser.add_argument('--outputname')
     parser.add_argument('--create_diff', default=defval(False), action='store_true', help='''\
-    create a binary patch file from which the randomized rom can be recreated using MultiClient.
+    create a binary patch file from which the randomized rom can be recreated using MultiClient.''')
-    Does not work with jsonout.''')
     parser.add_argument('--disable_glitch_boots', default=defval(False), action='store_true', help='''\
     turns off starting with Pegasus Boots in glitched modes.''')
 
@@ -357,15 +352,14 @@ def start():
         sys.exit(0)
 
     # ToDo: Validate files further than mere existance
-    if not args.jsonout and not os.path.isfile(args.rom):
+    if not os.path.isfile(args.rom):
-        input('Could not find valid base rom for patching at expected path %s. Please run with -h to see help for further information. \nPress Enter to exit.' % args.rom)
+        input(
+            'Could not find valid base rom for patching at expected path %s. Please run with -h to see help for further information. \nPress Enter to exit.' % args.rom)
+        sys.exit(1)
+    if any([sprite is not None and not os.path.isfile(sprite) and not get_sprite_from_name(sprite) for sprite in
+            args.sprite.values()]):
+        input('Could not find link sprite sheet at given location. \nPress Enter to exit.')
         sys.exit(1)
-    if any([sprite is not None and not os.path.isfile(sprite) and not get_sprite_from_name(sprite) for sprite in args.sprite.values()]):
-        if not args.jsonout:
-            input('Could not find link sprite sheet at given location. \nPress Enter to exit.')
-            sys.exit(1)
-        else:
-            raise IOError('Cannot find sprite file at %s' % args.sprite)
 
     # set up logger
     loglevel = {'error': logging.ERROR, 'info': logging.INFO, 'warning': logging.WARNING, 'debug': logging.DEBUG}[args.loglevel]

Main.py

@@ -13,7 +13,7 @@ from Items import ItemFactory
 from Regions import create_regions, create_shops, mark_light_world_regions
 from InvertedRegions import create_inverted_regions, mark_dark_world_regions
 from EntranceShuffle import link_entrances, link_inverted_entrances
-from Rom import patch_rom, patch_race_rom, patch_enemizer, apply_rom_settings, LocalRom, JsonRom, get_hash_string
+from Rom import patch_rom, patch_race_rom, patch_enemizer, apply_rom_settings, LocalRom, get_hash_string
 from Rules import set_rules
 from Dungeons import create_dungeons, fill_dungeons, fill_dungeons_restrictive
 from Fill import distribute_items_cutoff, distribute_items_staleness, distribute_items_restrictive, flood_items, balance_multiworld_progression
@@ -155,7 +155,6 @@ def main(args, seed=None):
     outfilebase = 'BM_%s' % (args.outputname if args.outputname else world.seed)
 
     rom_names = []
-    jsonout = {}
 
     def _gen_rom(team: int, player: int):
         sprite_random_on_hit = type(args.sprite[player]) is str and args.sprite[player].lower() == 'randomonhit'
@@ -163,15 +162,13 @@ def main(args, seed=None):
                         or world.enemy_health[player] != 'default' or world.enemy_damage[player] != 'default'
                         or args.shufflepots[player] or sprite_random_on_hit)
 
-        rom = JsonRom() if args.jsonout or use_enemizer else LocalRom(args.rom)
+        rom = LocalRom(args.rom)
 
         patch_rom(world, rom, player, team, use_enemizer)
 
-        if use_enemizer and (args.enemizercli or not args.jsonout):
+        if use_enemizer:
             patch_enemizer(world, player, rom, args.rom, args.enemizercli, args.shufflepots[player],
                            sprite_random_on_hit)
-            if not args.jsonout:
-                rom = LocalRom.fromJsonRom(rom, args.rom, 0x400000)
 
         if args.race:
             patch_race_rom(rom)
@@ -182,49 +179,46 @@ def main(args, seed=None):
                            args.fastmenu[player], args.disablemusic[player], args.sprite[player],
                            args.ow_palettes[player], args.uw_palettes[player])
 
-        if args.jsonout:
+        mcsb_name = ''
-            jsonout[f'patch_t{team}_p{player}'] = rom.patches
+        if all([world.mapshuffle[player], world.compassshuffle[player], world.keyshuffle[player],
-        else:
+                world.bigkeyshuffle[player]]):
-            mcsb_name = ''
+            mcsb_name = '-keysanity'
-            if all([world.mapshuffle[player], world.compassshuffle[player], world.keyshuffle[player],
+        elif [world.mapshuffle[player], world.compassshuffle[player], world.keyshuffle[player],
-                    world.bigkeyshuffle[player]]):
+              world.bigkeyshuffle[player]].count(True) == 1:
-                mcsb_name = '-keysanity'
+            mcsb_name = '-mapshuffle' if world.mapshuffle[player] else '-compassshuffle' if world.compassshuffle[
-            elif [world.mapshuffle[player], world.compassshuffle[player], world.keyshuffle[player],
+                player] else '-keyshuffle' if world.keyshuffle[player] else '-bigkeyshuffle'
-                  world.bigkeyshuffle[player]].count(True) == 1:
+        elif any([world.mapshuffle[player], world.compassshuffle[player], world.keyshuffle[player],
-                mcsb_name = '-mapshuffle' if world.mapshuffle[player] else '-compassshuffle' if world.compassshuffle[
+                  world.bigkeyshuffle[player]]):
-                    player] else '-keyshuffle' if world.keyshuffle[player] else '-bigkeyshuffle'
+            mcsb_name = '-%s%s%s%sshuffle' % (
-            elif any([world.mapshuffle[player], world.compassshuffle[player], world.keyshuffle[player],
+                'M' if world.mapshuffle[player] else '', 'C' if world.compassshuffle[player] else '',
-                      world.bigkeyshuffle[player]]):
+                'S' if world.keyshuffle[player] else '', 'B' if world.bigkeyshuffle[player] else '')
-                mcsb_name = '-%s%s%s%sshuffle' % (
-                    'M' if world.mapshuffle[player] else '', 'C' if world.compassshuffle[player] else '',
-                    'S' if world.keyshuffle[player] else '', 'B' if world.bigkeyshuffle[player] else '')
 
-            outfilepname = f'_T{team + 1}' if world.teams > 1 else ''
+        outfilepname = f'_T{team + 1}' if world.teams > 1 else ''
-            if world.players > 1:
+        if world.players > 1:
-                outfilepname += f'_P{player}'
+            outfilepname += f'_P{player}'
-            if world.players > 1 or world.teams > 1:
+        if world.players > 1 or world.teams > 1:
-                outfilepname += f"_{world.player_names[player][team].replace(' ', '_')}" if world.player_names[player][
+            outfilepname += f"_{world.player_names[player][team].replace(' ', '_')}" if world.player_names[player][
-                                                                                                team] != 'Player%d' % player else ''
+                                                                                            team] != 'Player%d' % player else ''
-            outfilesuffix = ('_%s_%s-%s-%s-%s%s_%s-%s%s%s%s%s' % (world.logic[player], world.difficulty[player],
+        outfilesuffix = ('_%s_%s-%s-%s-%s%s_%s-%s%s%s%s%s' % (world.logic[player], world.difficulty[player],
-                                                                  world.difficulty_adjustments[player],
+                                                              world.difficulty_adjustments[player],
-                                                                  world.mode[player], world.goal[player],
+                                                              world.mode[player], world.goal[player],
-                                                                  "" if world.timer[player] in [False,
+                                                              "" if world.timer[player] in [False,
-                                                                                                'display'] else "-" +
+                                                                                            'display'] else "-" +
-                                                                                                                world.timer[
+                                                                                                            world.timer[
-                                                                                                                    player],
+                                                                                                                player],
-                                                                  world.shuffle[player], world.algorithm,
+                                                              world.shuffle[player], world.algorithm,
-                                                                  mcsb_name,
+                                                              mcsb_name,
-                                                                  "-retro" if world.retro[player] else "",
+                                                              "-retro" if world.retro[player] else "",
-                                                                  "-prog_" + world.progressive[player] if
+                                                              "-prog_" + world.progressive[player] if
-                                                                  world.progressive[player] in ['off',
+                                                              world.progressive[player] in ['off',
-                                                                                                'random'] else "",
+                                                                                            'random'] else "",
-                                                                  "-nohints" if not world.hints[
+                                                              "-nohints" if not world.hints[
-                                                                      player] else "")) if not args.outputname else ''
+                                                                  player] else "")) if not args.outputname else ''
-            rompath = output_path(f'{outfilebase}{outfilepname}{outfilesuffix}.sfc')
+        rompath = output_path(f'{outfilebase}{outfilepname}{outfilesuffix}.sfc')
-            rom.write_to_file(rompath)
+        rom.write_to_file(rompath)
-            if args.create_diff:
+        if args.create_diff:
-                import Patch
+            import Patch
-                Patch.create_patch_file(rompath)
+            Patch.create_patch_file(rompath)
         return (player, team, list(rom.name))
 
     if not args.suppress_rom:
@@ -266,19 +260,15 @@ def main(args, seed=None):
                                               "server_options": get_options()["server_options"],
                                               "er_hint_data": er_hint_data,
                                               }).encode("utf-8"), 9)
-        if args.jsonout:
+
-            jsonout["multidata"] = list(multidata)
+        with open(output_path('%s.multidata' % outfilebase), 'wb') as f:
-        else:
+            f.write(multidata)
-            with open(output_path('%s.multidata' % outfilebase), 'wb') as f:
-                f.write(multidata)
 
     if not args.skip_playthrough:
         logger.info('Calculating playthrough.')
         create_playthrough(world)
 
-    if args.jsonout:
+    if args.create_spoiler:
-        print(json.dumps({**jsonout, 'spoiler': world.spoiler.to_json()}))
-    elif args.create_spoiler:
         world.spoiler.to_file(output_path('%s_Spoiler.txt' % outfilebase))
 
     logger.info('Done. Enjoy.')

Patch.py

@@ -57,16 +57,19 @@ def create_patch_file(rom_file_to_patch: str, server: str = "") -> str:
     write_lzma(bytes, target)
     return target
 
-
+def create_rom_bytes(patch_file: str) -> Tuple[dict, str, bytearray]:
-def create_rom_file(patch_file: str) -> Tuple[dict, str]:
     data = Utils.parse_yaml(lzma.decompress(load_bytes(patch_file)).decode("utf-8-sig"))
     patched_data = bsdiff4.patch(get_base_rom_bytes(), data["patch"])
     rom_hash = patched_data[int(0x7FC0):int(0x7FD5)]
     data["meta"]["hash"] = "".join(chr(x) for x in rom_hash)
     target = os.path.splitext(patch_file)[0] + ".sfc"
+    return data["meta"], target, patched_data
+
+def create_rom_file(patch_file: str) -> Tuple[dict, str]:
+    data, target, patched_data = create_rom_bytes(patch_file)
     with open(target, "wb") as f:
         f.write(patched_data)
-    return data["meta"], target
+    return data, target
 
 
 def update_patch_data(patch_data: bytes, server: str = "") -> bytes:

Rom.py

@@ -24,53 +24,6 @@ from EntranceShuffle import door_addresses
 JAP10HASH = '03a63945398191337e896e5771f77173'
 RANDOMIZERBASEHASH = 'aec17dd8b3c76c16d0b0311c36eb1c00'
 
-
-class JsonRom(object):
-
-    def __init__(self, name=None, hash=None):
-        self.name = name
-        self.hash = hash
-        self.orig_buffer = None
-        self.patches = {}
-        self.addresses = []
-
-    def write_byte(self, address, value):
-        self.write_bytes(address, [value])
-
-    def write_bytes(self, startaddress, values):
-        if not values:
-            return
-        values = list(values)
-
-        pos = bisect.bisect_right(self.addresses, startaddress)
-        intervalstart = self.addresses[pos-1] if pos else None
-        intervalpatch = self.patches[str(intervalstart)] if pos else None
-
-        if pos and startaddress <= intervalstart + len(intervalpatch): # merge with previous segment
-            offset = startaddress - intervalstart
-            intervalpatch[offset:offset+len(values)] = values
-            startaddress = intervalstart
-            values = intervalpatch
-        else: # new segment
-            self.addresses.insert(pos, startaddress)
-            self.patches[str(startaddress)] = values
-            pos = pos + 1
-
-        while pos < len(self.addresses) and self.addresses[pos] <= startaddress + len(values): # merge the next segment into this one
-            intervalstart = self.addresses[pos]
-            values.extend(self.patches[str(intervalstart)][startaddress+len(values)-intervalstart:])
-            del self.patches[str(intervalstart)]
-            del self.addresses[pos]
-
-    def write_to_file(self, file):
-        with open(file, 'w') as stream:
-            json.dump([self.patches], stream)
-
-    def get_hash(self):
-        h = hashlib.md5()
-        h.update(json.dumps([self.patches]).encode('utf-8'))
-        return h.hexdigest()
-
 class LocalRom(object):
 
     def __init__(self, file, patch=True, name=None, hash=None):
@@ -93,6 +46,10 @@ class LocalRom(object):
         with open(file, 'wb') as outfile:
             outfile.write(self.buffer)
 
+    def read_from_file(self, file):
+        with open(file, 'rb') as stream:
+            self.buffer = bytearray(stream.read())
+
     @staticmethod
     def fromJsonRom(rom, file, rom_size=0x200000):
         ret = LocalRom(file, True, rom.name, rom.hash)
@@ -101,12 +58,37 @@ class LocalRom(object):
             ret.write_bytes(int(address), values)
         return ret
 
+    @staticmethod
+    def verify(buffer, expected=RANDOMIZERBASEHASH):
+        buffermd5 = hashlib.md5()
+        buffermd5.update(buffer)
+        return expected == buffermd5.hexdigest()
+
     def patch_base_rom(self):
+        from Patch import create_patch_file, create_rom_bytes
+
+        if os.path.isfile(local_path('data/baserom.sfc')):
+            with open(local_path('data/baserom.sfc'), 'rb') as stream:
+                buffer = bytearray(stream.read())
+
+            if self.verify(buffer):
+                self.buffer = buffer
+                if not os.path.exists(local_path('data/baserom.bmbp')):
+                    create_patch_file(local_path('data/baserom.sfc'))
+                return
+
+        if os.path.isfile(local_path('data/baserom.bmbp')):
+            _, target, buffer = create_rom_bytes(local_path('data/baserom.bmbp'))
+            if self.verify(buffer):
+                self.buffer = bytearray(buffer)
+                with open(local_path('data/baserom.sfc'), 'wb') as stream:
+                    stream.write(buffer)
+                return
+
         # verify correct checksum of baserom
-        basemd5 = hashlib.md5()
+        if not self.verify(self.buffer, JAP10HASH):
-        basemd5.update(self.buffer)
+            logging.getLogger('').warning(
-        if JAP10HASH != basemd5.hexdigest():
+                'Supplied Base Rom does not match known MD5 for JAP(1.0) release. Will try to patch anyway.')
-            logging.getLogger('').warning('Supplied Base Rom does not match known MD5 for JAP(1.0) release. Will try to patch anyway.')
 
         # extend to 2MB
         self.buffer.extend(bytearray([0x00]) * (0x200000 - len(self.buffer)))
@@ -120,10 +102,14 @@ class LocalRom(object):
                     self.write_bytes(int(baseaddress), values)
 
         # verify md5
-        patchedmd5 = hashlib.md5()
+        if self.verify(self.buffer):
-        patchedmd5.update(self.buffer)
+            with open(local_path('data/baserom.sfc'), 'wb') as stream:
-        if patchedmd5.hexdigest() not in [RANDOMIZERBASEHASH]:
+                stream.write(self.buffer)
-            raise RuntimeError('Provided Base Rom unsuitable for patching. Please provide a JAP(1.0) "Zelda no Densetsu - Kamigami no Triforce (Japan).sfc" rom to use as a base.')
+            create_patch_file(local_path('data/baserom.sfc'))
+            os.remove(local_path('data/base2current.json'))
+        else:
+            raise RuntimeError(
+                'Provided Base Rom unsuitable for patching. Please provide a JAP(1.0) "Zelda no Densetsu - Kamigami no Triforce (Japan).sfc" rom to use as a base.')
 
     def write_crc(self):
         crc = (sum(self.buffer[:0x7FDC] + self.buffer[0x7FE0:]) + 0x01FE) & 0xFFFF
@@ -158,12 +144,9 @@ def read_rom(stream) -> bytearray:
     return buffer
 
 def patch_enemizer(world, player, rom, baserom_path, enemizercli, shufflepots, random_sprite_on_hit):
-    baserom_path = os.path.abspath(baserom_path)
+    randopatch_path = os.path.abspath(output_path(f'enemizer_randopatch_{player}.sfc'))
-    basepatch_path = os.path.abspath(local_path('data/base2current.json'))
-    enemizer_basepatch_path = os.path.join(os.path.dirname(enemizercli), "enemizerBasePatch.json")
-    randopatch_path = os.path.abspath(output_path(f'enemizer_randopatch_{player}.json'))
     options_path = os.path.abspath(output_path(f'enemizer_options_{player}.json'))
-    enemizer_output_path = os.path.abspath(output_path(f'enemizer_output_{player}.json'))
+    enemizer_output_path = os.path.abspath(output_path(f'enemizer_output_{player}.sfc'))
 
     # write options file for enemizer
     options = {
@@ -255,21 +238,14 @@ def patch_enemizer(world, player, rom, baserom_path, enemizercli, shufflepots, r
         json.dump(options, f)
 
     subprocess.check_call([os.path.abspath(enemizercli),
-                           '--rom', baserom_path,
+                           '--rom', randopatch_path,
                            '--seed', str(world.rom_seeds[player]),
-                           '--base', basepatch_path,
+                           '--binary',
-                           '--randomizer', randopatch_path,
                            '--enemizer', options_path,
                            '--output', enemizer_output_path],
                           cwd=os.path.dirname(enemizercli), stdout=subprocess.DEVNULL)
-
+    rom.read_from_file(enemizer_output_path)
-    with open(enemizer_basepatch_path, 'r') as f:
+    os.remove(enemizer_output_path)
-        for patch in json.load(f):
-            rom.write_bytes(patch["address"], patch["patchData"])
-
-    with open(enemizer_output_path, 'r') as f:
-        for patch in json.load(f):
-            rom.write_bytes(patch["address"], patch["patchData"])
 
     if random_sprite_on_hit:
         _populate_sprite_table()
@@ -285,7 +261,7 @@ def patch_enemizer(world, player, rom, baserom_path, enemizercli, shufflepots, r
                 rom.write_bytes(0x307000 + (i * 0x8000), sprite.palette)
                 rom.write_bytes(0x307078 + (i * 0x8000), sprite.glove_palette)
 
-    for used in (randopatch_path, options_path, enemizer_output_path):
+    for used in (randopatch_path, options_path):
         try:
             os.remove(used)
         except OSError: