diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..10c13a6b --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,60 @@ +# This workflow will build a release-like distribution when manually dispatched + +name: Build + +on: workflow_dispatch + +jobs: + build-ubuntu1804: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + - name: Install base dependencies + run: | + sudo apt update + sudo apt -y install build-essential p7zip xz-utils wget libglib2.0-0 + sudo apt -y install python3-gi libgirepository1.0-dev # should pull dependencies for gi installation below + - name: Get a recent python + uses: actions/setup-python@v3 + with: + python-version: '3.9' + - name: Install build-time dependencies + run: | + echo "PYTHON=python3.9" >> $GITHUB_ENV + wget -nv https://github.com/AppImage/AppImageKit/releases/download/13/appimagetool-x86_64.AppImage + chmod a+rx appimagetool-x86_64.AppImage + ./appimagetool-x86_64.AppImage --appimage-extract + echo -e '#/bin/sh\n./squashfs-root/AppRun "$@"' > appimagetool + chmod a+rx appimagetool + - name: Download run-time dependencies + run: | + wget -nv https://github.com/black-sliver/sni/releases/download/v0.0.78-2/sni-v0.0.78-2-manylinux2014-amd64.tar.xz + tar xf sni-*.tar.xz + rm sni-*.tar.xz + mv sni-* SNI + wget -nv https://github.com/Ijwu/Enemizer/releases/download/6.4/ubuntu.16.04-x64.7z + 7za x -oEnemizerCLI/ ubuntu.16.04-x64.7z + - name: Build + run: | + "${{ env.PYTHON }}" -m pip install --upgrade pip setuptools virtualenv PyGObject # pygobject should probably move to requirements + "${{ env.PYTHON }}" -m venv venv + source venv/bin/activate + pip install -r requirements.txt + python setup.py build --yes bdist_appimage --yes + echo -e "setup.py build output:\n `ls build`" + echo -e "setup.py dist output:\n `ls dist`" + cd dist && export APPIMAGE_NAME="`ls *.AppImage`" && cd .. + export TAR_NAME="${APPIMAGE_NAME%.AppImage}.tar.gz" + (cd build && DIR_NAME="`ls | grep exe`" && mv "$DIR_NAME" Archipelago && tar -czvf ../dist/$TAR_NAME Archipelago && mv Archipelago "$DIR_NAME") + echo "APPIMAGE_NAME=$APPIMAGE_NAME" >> $GITHUB_ENV + echo "TAR_NAME=$TAR_NAME" >> $GITHUB_ENV + - name: Store AppImage + uses: actions/upload-artifact@v2 + with: + name: ${{ env.APPIMAGE_NAME }} + path: dist/${{ env.APPIMAGE_NAME }} + - name: Store .tar.gz + uses: actions/upload-artifact@v2 + with: + name: ${{ env.TAR_NAME }} + path: dist/${{ env.TAR_NAME }} diff --git a/Generate.py b/Generate.py index 4a1e3dd6..9acd4356 100644 --- a/Generate.py +++ b/Generate.py @@ -15,7 +15,7 @@ ModuleUpdate.update() import Utils from worlds.alttp import Options as LttPOptions from worlds.generic import PlandoConnection -from Utils import parse_yaml, version_tuple, __version__, tuplize_version, get_options +from Utils import parse_yaml, version_tuple, __version__, tuplize_version, get_options, local_path, user_path from worlds.alttp.EntranceRandomizer import parse_arguments from Main import main as ERmain from BaseClasses import seeddigits, get_seed @@ -27,24 +27,31 @@ import copy categories = set(AutoWorldRegister.world_types) + def mystery_argparse(): options = get_options() defaults = options["generator"] + def resolve_path(path: str, resolver: typing.Callable[[str], str]) -> str: + return path if os.path.isabs(path) else resolver(path) + parser = argparse.ArgumentParser(description="CMD Generation Interface, defaults come from host.yaml.") - parser.add_argument('--weights_file_path', default = defaults["weights_file_path"], + parser.add_argument('--weights_file_path', default=defaults["weights_file_path"], help='Path to the weights file to use for rolling game settings, urls are also valid') parser.add_argument('--samesettings', help='Rolls settings per weights file rather than per player', action='store_true') - parser.add_argument('--player_files_path', default=defaults["player_files_path"], + parser.add_argument('--player_files_path', default=resolve_path(defaults["player_files_path"], user_path), help="Input directory for player files.") parser.add_argument('--seed', help='Define seed number to generate.', type=int) parser.add_argument('--multi', default=defaults["players"], type=lambda value: max(int(value), 1)) parser.add_argument('--spoiler', type=int, default=defaults["spoiler"]) - parser.add_argument('--lttp_rom', default=options["lttp_options"]["rom_file"], help="Path to the 1.0 JP LttP Baserom.") - parser.add_argument('--sm_rom', default=options["sm_options"]["rom_file"], help="Path to the 1.0 JP SM Baserom.") - parser.add_argument('--enemizercli', default=defaults["enemizer_path"]) - parser.add_argument('--outputpath', default=options["general_options"]["output_path"]) + parser.add_argument('--lttp_rom', default=options["lttp_options"]["rom_file"], + help="Path to the 1.0 JP LttP Baserom.") # absolute, relative to cwd or relative to app path + parser.add_argument('--sm_rom', default=options["sm_options"]["rom_file"], + help="Path to the 1.0 JP SM Baserom.") + parser.add_argument('--enemizercli', default=resolve_path(defaults["enemizer_path"], local_path)) + parser.add_argument('--outputpath', default=resolve_path(options["general_options"]["output_path"], user_path), + help="Path to output folder. Absolute or relative to cwd.") # absolute or relative to cwd parser.add_argument('--race', action='store_true', default=defaults["race"]) parser.add_argument('--meta_file_path', default=defaults["meta_file_path"]) parser.add_argument('--log_level', default='info', help='Sets log level') diff --git a/Launcher.py b/Launcher.py new file mode 100644 index 00000000..996a4e52 --- /dev/null +++ b/Launcher.py @@ -0,0 +1,307 @@ +""" +Archipelago launcher for bundled app. + +* if run with APBP as argument, launch corresponding client. +* if run with executable as argument, run it passing argv[2:] as arguments +* if run without arguments, open launcher GUI + +Scroll down to components= to add components to the launcher as well as setup.py +""" + + +import argparse +from os.path import isfile +import sys +from typing import Iterable, Sequence, Callable, Union, Optional +import subprocess +import itertools +from Utils import is_frozen, user_path, local_path, init_logging +from shutil import which +import shlex +from enum import Enum, auto +import logging + + +is_linux = sys.platform.startswith('linux') +is_macos = sys.platform == 'darwin' +is_windows = sys.platform in ("win32", "cygwin", "msys") + + +def open_host_yaml(): + file = user_path('host.yaml') + if is_linux: + exe = which('sensible-editor') or which('gedit') or \ + which('xdg-open') or which('gnome-open') or which('kde-open') + subprocess.Popen([exe, file]) + elif is_macos: + exe = which("open") + subprocess.Popen([exe, file]) + else: + import webbrowser + webbrowser.open(file) + + +def open_patch(): + try: + import tkinter + import tkinter.filedialog + except Exception as e: + logging.error("Could not load tkinter, which is likely not installed. " + "This attempt was made because Launcher.open_patch was used.") + raise e + else: + root = tkinter.Tk() + root.withdraw() + suffixes = [] + for c in components: + if isfile(get_exe(c)[-1]): + suffixes += c.file_identifier.suffixes if c.type == Type.CLIENT and \ + isinstance(c.file_identifier, SuffixIdentifier) else [] + filename = tkinter.filedialog.askopenfilename(filetypes=(('Patches', ' '.join(suffixes)),)) + file, component = identify(filename) + if file and component: + subprocess.Popen([*get_exe(component), file]) + + +def browse_files(): + file = user_path() + if is_linux: + exe = which('xdg-open') or which('gnome-open') or which('kde-open') + subprocess.Popen([exe, file]) + elif is_macos: + exe = which("open") + subprocess.Popen([exe, file]) + else: + import webbrowser + webbrowser.open(file) + + +class Type(Enum): + TOOL = auto() + FUNC = auto() # not a real component + CLIENT = auto() + ADJUSTER = auto() + + +class SuffixIdentifier: + suffixes: Iterable[str] + + def __init__(self, *args: str): + self.suffixes = args + + def __call__(self, path: str): + if isinstance(path, str): + for suffix in self.suffixes: + if path.endswith(suffix): + return True + return False + + +class Component: + display_name: str + type: Optional[Type] + script_name: Optional[str] + frozen_name: Optional[str] + icon: str # just the name, no suffix + cli: bool + func: Optional[Callable] + file_identifier: Optional[Callable[[str], bool]] + + def __init__(self, display_name: str, script_name: Optional[str] = None, frozen_name: Optional[str] = None, + cli: bool = False, icon: str = 'icon', component_type: Type = None, func: Optional[Callable] = None, + file_identifier: Optional[Callable[[str], bool]] = None): + self.display_name = display_name + self.script_name = script_name + self.frozen_name = frozen_name or f'Archipelago{script_name}' if script_name else None + self.icon = icon + self.cli = cli + self.type = component_type or \ + None if not display_name else \ + Type.FUNC if func else \ + Type.CLIENT if 'Client' in display_name else \ + Type.ADJUSTER if 'Adjuster' in display_name else Type.TOOL + self.func = func + self.file_identifier = file_identifier + + def handles_file(self, path: str): + return self.file_identifier(path) if self.file_identifier else False + + +components: Iterable[Component] = ( + # Launcher + Component('', 'Launcher'), + # Core + Component('Host', 'MultiServer', 'ArchipelagoServer', cli=True, + file_identifier=SuffixIdentifier('.archipelago', '.zip')), + Component('Generate', 'Generate', cli=True), + Component('Text Client', 'CommonClient', 'ArchipelagoTextClient'), + # SNI + Component('SNI Client', 'SNIClient', + file_identifier=SuffixIdentifier('.apz3', '.apm3', '.apsoe', '.aplttp', '.apsm', '.apsmz3')), + Component('LttP Adjuster', 'LttPAdjuster'), + # Factorio + Component('Factorio Client', 'FactorioClient'), + # Minecraft + Component('Minecraft Client', 'MinecraftClient', icon='mcicon', + file_identifier=SuffixIdentifier('.apmc')), + # Ocarina of Time + Component('OoT Client', 'OoTClient', + file_identifier=SuffixIdentifier('.apz5')), + Component('OoT Adjuster', 'OoTAdjuster'), + # FF1 + Component('FF1 Client', 'FF1Client'), + # ChecksFinder + Component('ChecksFinder Client', 'ChecksFinderClient'), + # Functions + Component('Open host.yaml', func=open_host_yaml), + Component('Open Patch', func=open_patch), + Component('Browse Files', func=browse_files), +) +icon_paths = { + 'icon': local_path('data', 'icon.ico' if is_windows else 'icon.png'), + 'mcicon': local_path('data', 'mcicon.ico') +} + + +def identify(path: Union[None, str]): + if path is None: + return None, None + for component in components: + if component.handles_file(path): + return path, component.script_name + return (None, None) if '/' in path or '\\' in path else (None, path) + + +def get_exe(component: Union[str, Component]) -> Optional[Sequence[str]]: + if isinstance(component, str): + name = component + component = None + if name.startswith('Archipelago'): + name = name[11:] + if name.endswith('.exe'): + name = name[:-4] + if name.endswith('.py'): + name = name[:-3] + if not name: + return None + for c in components: + if c.script_name == name or c.frozen_name == f'Archipelago{name}': + component = c + break + if not component: + return None + if is_frozen(): + suffix = '.exe' if is_windows else '' + return [local_path(f'{component.frozen_name}{suffix}')] + else: + return [sys.executable, local_path(f'{component.script_name}.py')] + + +def launch(exe, in_terminal=False): + if in_terminal: + if is_windows: + subprocess.Popen(['start', *exe], shell=True) + return + elif is_linux: + terminal = which('x-terminal-emulator') or which('gnome-terminal') or which('xterm') + if terminal: + subprocess.Popen([terminal, '-e', shlex.join(exe)]) + return + elif is_macos: + terminal = [which('open'), '-W', '-a', 'Terminal.app'] + subprocess.Popen([*terminal, *exe]) + return + subprocess.Popen(exe) + + +def run_gui(): + if not sys.stdout: + from kvui import App, ContainerLayout, GridLayout, Button, Label # this kills stdout + else: + from kivy.app import App + from kivy.uix.button import Button + from kivy.uix.floatlayout import FloatLayout as ContainerLayout + from kivy.uix.gridlayout import GridLayout + from kivy.uix.label import Label + + class Launcher(App): + base_title: str = "Archipelago Launcher" + container: ContainerLayout + grid: GridLayout + + _tools = {c.display_name: c for c in components if c.type == Type.TOOL and isfile(get_exe(c)[-1])} + _clients = {c.display_name: c for c in components if c.type == Type.CLIENT and isfile(get_exe(c)[-1])} + _adjusters = {c.display_name: c for c in components if c.type == Type.ADJUSTER and isfile(get_exe(c)[-1])} + _funcs = {c.display_name: c for c in components if c.type == Type.FUNC} + + def __init__(self, ctx=None): + self.title = self.base_title + self.ctx = ctx + self.icon = r"data/icon.png" + super().__init__() + + def build(self): + self.container = ContainerLayout() + self.grid = GridLayout(cols=2) + self.container.add_widget(self.grid) + + button_layout = self.grid # make buttons fill the window + for (tool, client) in itertools.zip_longest(itertools.chain( + self._tools.items(), self._funcs.items(), self._adjusters.items()), self._clients.items()): + # column 1 + if tool: + button = Button(text=tool[0]) + button.component = tool[1] + button.bind(on_release=self.component_action) + button_layout.add_widget(button) + else: + button_layout.add_widget(Label()) + # column 2 + if client: + button = Button(text=client[0]) + button.component = client[1] + button.bind(on_press=self.component_action) + button_layout.add_widget(button) + else: + button_layout.add_widget(Label()) + + return self.container + + @staticmethod + def component_action(button): + if button.component.type == Type.FUNC: + button.component.func() + else: + launch(get_exe(button.component), button.component.cli) + + Launcher().run() + + +def main(args: Optional[Union[argparse.Namespace, dict]] = None): + if isinstance(args, argparse.Namespace): + args = {k: v for k, v in args._get_kwargs()} + elif not args: + args = {} + + if "Patch|Game|Component" in args: + file, component = identify(args["Patch|Game|Component"]) + if file: + args['file'] = file + if component: + args['component'] = component + + if 'file' in args: + subprocess.run([*get_exe(args['component']), args['file'], *args['args']]) + elif 'component' in args: + subprocess.run([*get_exe(args['component']), *args['args']]) + else: + run_gui() + + +if __name__ == '__main__': + init_logging('Launcher') + parser = argparse.ArgumentParser(description='Archipelago Launcher') + parser.add_argument('Patch|Game|Component', type=str, nargs='?', + help="Pass either a patch file, a generated game or the name of a component to run.") + parser.add_argument('args', nargs="*", help="Arguments to pass to component.") + main(parser.parse_args()) diff --git a/LttPAdjuster.py b/LttPAdjuster.py index 452aaf91..ad419883 100644 --- a/LttPAdjuster.py +++ b/LttPAdjuster.py @@ -21,7 +21,7 @@ from urllib.parse import urlparse from urllib.request import urlopen from worlds.alttp.Rom import Sprite, LocalRom, apply_rom_settings, get_base_rom_bytes -from Utils import output_path, local_path, open_file, get_cert_none_ssl_context, persistent_store, get_adjuster_settings, tkinter_center_window +from Utils import output_path, local_path, user_path, open_file, get_cert_none_ssl_context, persistent_store, get_adjuster_settings, tkinter_center_window from Patch import GAME_ALTTP class AdjusterWorld(object): @@ -286,7 +286,7 @@ def run_sprite_update(): def update_sprites(task, on_finish=None): resultmessage = "" successful = True - sprite_dir = local_path("data", "sprites", "alttpr") + sprite_dir = user_path("data", "sprites", "alttpr") os.makedirs(sprite_dir, exist_ok=True) ctx = get_cert_none_ssl_context() def finished(): @@ -1013,11 +1013,11 @@ class SpriteSelector(): @property def alttpr_sprite_dir(self): - return local_path("data", "sprites", "alttpr") + return user_path("data", "sprites", "alttpr") @property def custom_sprite_dir(self): - return local_path("data", "sprites", "custom") + return user_path("data", "sprites", "custom") def get_image_for_sprite(sprite, gif_only: bool = False): diff --git a/Patch.py b/Patch.py index 0f0ddc18..da7da26a 100644 --- a/Patch.py +++ b/Patch.py @@ -238,8 +238,8 @@ def get_base_rom_data(game: str): elif game == GAME_SM: from worlds.sm.Rom import get_base_rom_bytes elif game == GAME_SOE: - file_name = Utils.get_options()["soe_options"]["rom_file"] - get_base_rom_bytes = lambda: bytes(read_rom(open(file_name, "rb"))) + from worlds.soe.Patch import get_base_rom_path + get_base_rom_bytes = lambda: bytes(read_rom(open(get_base_rom_path(), "rb"))) elif game == GAME_SMZ3: from worlds.smz3.Rom import get_base_rom_bytes else: diff --git a/SNIClient.py b/SNIClient.py index 843672aa..5a5bae66 100644 --- a/SNIClient.py +++ b/SNIClient.py @@ -572,8 +572,14 @@ def launch_sni(ctx: Context): if not sys.stdout: # if it spawns a visible console, may as well populate it subprocess.Popen(os.path.abspath(sni_path), cwd=os.path.dirname(sni_path)) else: - subprocess.Popen(os.path.abspath(sni_path), cwd=os.path.dirname(sni_path), stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL) + proc = subprocess.Popen(os.path.abspath(sni_path), cwd=os.path.dirname(sni_path), + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + try: + proc.wait(.1) # wait a bit to see if startup fails (missing dependencies) + snes_logger.info('Failed to start SNI. Try running it externally for error output.') + except subprocess.TimeoutExpired: + pass # seems to be running + else: snes_logger.info( f"Attempt to start SNI was aborted as path {sni_path} was not found, " diff --git a/Utils.py b/Utils.py index 63bc7ac0..027ba73f 100644 --- a/Utils.py +++ b/Utils.py @@ -1,5 +1,6 @@ from __future__ import annotations +import shutil import typing import builtins import os @@ -72,10 +73,10 @@ def is_frozen() -> bool: return getattr(sys, 'frozen', False) -def local_path(*path: str): - if local_path.cached_path: - return os.path.join(local_path.cached_path, *path) - +def local_path(*path: str) -> str: + """Returns path to a file in the local Archipelago installation or source.""" + if hasattr(local_path, 'cached_path'): + pass elif is_frozen(): if hasattr(sys, "_MEIPASS"): # we are running in a PyInstaller bundle @@ -95,21 +96,47 @@ def local_path(*path: str): return os.path.join(local_path.cached_path, *path) -local_path.cached_path = None +def home_path(*path: str) -> str: + """Returns path to a file in the user home's Archipelago directory.""" + if hasattr(home_path, 'cached_path'): + pass + elif sys.platform.startswith('linux'): + home_path.cached_path = os.path.expanduser('~/Archipelago') + os.makedirs(home_path.cached_path, 0o700, exist_ok=True) + else: + # not implemented + home_path.cached_path = local_path() # this will generate the same exceptions we got previously + + return os.path.join(home_path.cached_path, *path) -def output_path(*path): - if output_path.cached_path: +def user_path(*path: str) -> str: + """Returns either local_path or home_path based on write permissions.""" + if hasattr(user_path, 'cached_path'): + pass + elif os.access(local_path(), os.W_OK): + user_path.cached_path = local_path() + else: + user_path.cached_path = home_path() + # populate home from local - TODO: upgrade feature + if user_path.cached_path != local_path() and not os.path.exists(user_path('host.yaml')): + for dn in ('Players', 'data/sprites'): + shutil.copytree(local_path(dn), user_path(dn), dirs_exist_ok=True) + for fn in ('manifest.json', 'host.yaml'): + shutil.copy2(local_path(fn), user_path(fn)) + + return os.path.join(user_path.cached_path, *path) + + +def output_path(*path: str): + if hasattr(output_path, 'cached_path'): return os.path.join(output_path.cached_path, *path) - output_path.cached_path = local_path(get_options()["general_options"]["output_path"]) + output_path.cached_path = user_path(get_options()["general_options"]["output_path"]) path = os.path.join(output_path.cached_path, *path) os.makedirs(os.path.dirname(path), exist_ok=True) return path -output_path.cached_path = None - - def open_file(filename): if sys.platform == 'win32': os.startfile(filename) @@ -263,8 +290,11 @@ def update_options(src: dict, dest: dict, filename: str, keys: list) -> dict: @cache_argsless def get_options() -> dict: if not hasattr(get_options, "options"): - locations = ("options.yaml", "host.yaml", - local_path("options.yaml"), local_path("host.yaml")) + filenames = ("options.yaml", "host.yaml") + locations = [] + if os.path.join(os.getcwd()) != local_path(): + locations += filenames # use files from cwd only if it's not the local_path + locations += [user_path(filename) for filename in filenames] for location in locations: if os.path.exists(location): @@ -274,7 +304,7 @@ def get_options() -> dict: get_options.options = update_options(get_default_options(), options, location, list()) break else: - raise FileNotFoundError(f"Could not find {locations[1]} to load options.") + raise FileNotFoundError(f"Could not find {filenames[1]} to load options.") return get_options.options @@ -289,7 +319,7 @@ def get_location_name_from_id(code: int) -> str: def persistent_store(category: str, key: typing.Any, value: typing.Any): - path = local_path("_persistent_storage.yaml") + path = user_path("_persistent_storage.yaml") storage: dict = persistent_load() category = storage.setdefault(category, {}) category[key] = value @@ -301,7 +331,7 @@ def persistent_load() -> typing.Dict[dict]: storage = getattr(persistent_load, "storage", None) if storage: return storage - path = local_path("_persistent_storage.yaml") + path = user_path("_persistent_storage.yaml") storage: dict = {} if os.path.exists(path): try: @@ -388,7 +418,7 @@ loglevel_mapping = {'error': logging.ERROR, 'info': logging.INFO, 'warning': log def init_logging(name: str, loglevel: typing.Union[str, int] = logging.INFO, write_mode: str = "w", log_format: str = "[%(name)s]: %(message)s", exception_logger: str = ""): loglevel: int = loglevel_mapping.get(loglevel, loglevel) - log_folder = local_path("logs") + log_folder = user_path("logs") os.makedirs(log_folder, exist_ok=True) root_logger = logging.getLogger() for handler in root_logger.handlers[:]: diff --git a/WebHost.py b/WebHost.py index 0d557b61..773120b7 100644 --- a/WebHost.py +++ b/WebHost.py @@ -21,6 +21,8 @@ from WebHostLib.lttpsprites import update_sprites_lttp from WebHostLib.options import create as create_options_files configpath = os.path.abspath("config.yaml") +if not os.path.exists(configpath): # fall back to config.yaml in home + configpath = os.path.abspath(Utils.user_path('config.yaml')) def get_app(): diff --git a/WebHostLib/lttpsprites.py b/WebHostLib/lttpsprites.py index ead68f86..6f5a6b98 100644 --- a/WebHostLib/lttpsprites.py +++ b/WebHostLib/lttpsprites.py @@ -2,7 +2,7 @@ import os import threading import json -from Utils import local_path +from Utils import local_path, user_path from worlds.alttp.Rom import Sprite @@ -14,8 +14,8 @@ def update_sprites_lttp(): from LttPAdjuster import update_sprites # Target directories - input_dir = local_path("data", "sprites", "alttpr") - output_dir = local_path("WebHostLib", "static", "generated") + input_dir = user_path("data", "sprites", "alttpr") + output_dir = local_path("WebHostLib", "static", "generated") # TODO: move to user_path os.makedirs(os.path.join(output_dir, "sprites"), exist_ok=True) # update sprites through gui.py's functions diff --git a/setup.py b/setup.py index 84afb5ed..9bea0d72 100644 --- a/setup.py +++ b/setup.py @@ -3,30 +3,29 @@ import shutil import sys import sysconfig from pathlib import Path - -import ModuleUpdate -# I don't really want to have another root directory file for a single requirement, but this special case is also jank. -# Might move this into a cleaner solution when I think of one. -with open("freeze_requirements.txt", "w") as f: - f.write("cx-Freeze>=6.9\n") -ModuleUpdate.requirements_files.add("freeze_requirements.txt") -ModuleUpdate.requirements_files.add(os.path.join("WebHostLib", "requirements.txt")) -ModuleUpdate.update() - -import cx_Freeze -from kivy_deps import sdl2, glew +from hashlib import sha3_512 +import base64 +import datetime from Utils import version_tuple +from collections.abc import Iterable +import typing +import setuptools +from Launcher import components, icon_paths -arch_folder = "exe.{platform}-{version}".format(platform=sysconfig.get_platform(), - version=sysconfig.get_python_version()) -buildfolder = Path("build", arch_folder) -sbuildfolder = str(buildfolder) -libfolder = Path(buildfolder, "lib") -library = Path(libfolder, "library.zip") -print("Outputting to: " + sbuildfolder) -icon = os.path.join("data", "icon.ico") -mcicon = os.path.join("data", "mcicon.ico") +# This is a bit jank. We need cx-Freeze to be able to run anything from this script, so install it +import subprocess +import pkg_resources +requirement = 'cx-Freeze>=6.9' +try: + pkg_resources.require(requirement) + import cx_Freeze +except pkg_resources.ResolutionError: + if '--yes' not in sys.argv and '-y' not in sys.argv: + input(f'Requirement {requirement} is not satisfied, press enter to install it') + subprocess.call([sys.executable, '-m', 'pip', 'install', requirement, '--upgrade']) + import cx_Freeze + if os.path.exists("X:/pw.txt"): print("Using signtool") @@ -36,40 +35,24 @@ if os.path.exists("X:/pw.txt"): else: signtool = None -from hashlib import sha3_512 -import base64 + +arch_folder = "exe.{platform}-{version}".format(platform=sysconfig.get_platform(), + version=sysconfig.get_python_version()) +buildfolder = Path("build", arch_folder) +is_windows = sys.platform in ("win32", "cygwin", "msys") -def _threaded_hash(filepath): - hasher = sha3_512() - hasher.update(open(filepath, "rb").read()) - return base64.b85encode(hasher.digest()).decode() +# see Launcher.py on how to add scripts to setup.py +exes = [ + cx_Freeze.Executable( + script=f'{c.script_name}.py', + target_name=c.frozen_name + (".exe" if is_windows else ""), + icon=icon_paths[c.icon], + base="Win32GUI" if is_windows and not c.cli else None + ) for c in components if c.script_name +] - -os.makedirs(buildfolder, exist_ok=True) - - -def manifest_creation(folder, create_hashes=False): - # Since the setup is now split into components and the manifest is not, - # it makes most sense to just remove the hashes for now. Not aware of anyone using them. - hashes = {} - manifestpath = os.path.join(folder, "manifest.json") - if create_hashes: - from concurrent.futures import ThreadPoolExecutor - pool = ThreadPoolExecutor() - for dirpath, dirnames, filenames in os.walk(folder): - for filename in filenames: - path = os.path.join(dirpath, filename) - hashes[os.path.relpath(path, start=folder)] = pool.submit(_threaded_hash, path) - - import json - manifest = { - "buildtime": buildtime.isoformat(sep=" ", timespec="seconds"), - "hashes": {path: hash.result() for path, hash in hashes.items()}, - "version": version_tuple} - - json.dump(manifest, open(manifestpath, "wt"), indent=4) - print("Created Manifest") +extra_data = ["LICENSE", "data", "EnemizerCLI", "host.yaml", "SNI"] def remove_sprites_from_folder(folder): @@ -78,40 +61,279 @@ def remove_sprites_from_folder(folder): os.remove(folder / file) -scripts = { - # Core - "MultiServer.py": ("ArchipelagoServer", False, icon), - "Generate.py": ("ArchipelagoGenerate", False, icon), - "CommonClient.py": ("ArchipelagoTextClient", True, icon), - # SNI - "SNIClient.py": ("ArchipelagoSNIClient", True, icon), - "LttPAdjuster.py": ("ArchipelagoLttPAdjuster", True, icon), - # Factorio - "FactorioClient.py": ("ArchipelagoFactorioClient", True, icon), - # Minecraft - "MinecraftClient.py": ("ArchipelagoMinecraftClient", False, mcicon), - # Ocarina of Time - "OoTClient.py": ("ArchipelagoOoTClient", True, icon), - "OoTAdjuster.py": ("ArchipelagoOoTAdjuster", True, icon), - # FF1 - "FF1Client.py": ("ArchipelagoFF1Client", True, icon), - # ChecksFinder - "ChecksFinderClient.py": ("ArchipelagoChecksFinderClient", True, icon), -} +def _threaded_hash(filepath): + hasher = sha3_512() + hasher.update(open(filepath, "rb").read()) + return base64.b85encode(hasher.digest()).decode() -exes = [] -for script, (scriptname, gui, icon) in scripts.items(): - exes.append(cx_Freeze.Executable( - script=script, - target_name=scriptname + ("" if sys.platform == "linux" else ".exe"), - icon=icon, - base="Win32GUI" if sys.platform == "win32" and gui else None - )) +# cx_Freeze's build command runs other commands. Override to accept --yes and store that. +class BuildCommand(cx_Freeze.dist.build): + user_options = [ + ('yes', 'y', 'Answer "yes" to all questions.'), + ] + yes: bool + last_yes: bool = False # used by sub commands of build -import datetime + def initialize_options(self): + super().initialize_options() + type(self).last_yes = self.yes = False + + def finalize_options(self): + super().finalize_options() + type(self).last_yes = self.yes + + +# Override cx_Freeze's build_exe command for pre and post build steps +class BuildExeCommand(cx_Freeze.dist.build_exe): + user_options = cx_Freeze.dist.build_exe.user_options + [ + ('yes', 'y', 'Answer "yes" to all questions.'), + ('extra-data=', None, 'Additional files to add.'), + ] + yes: bool + extra_data: Iterable # [any] not available in 3.8 + + buildfolder: Path + libfolder: Path + library: Path + buildtime: datetime.datetime + + def initialize_options(self): + super().initialize_options() + self.yes = BuildCommand.last_yes + self.extra_data = [] + + def finalize_options(self): + super().finalize_options() + self.buildfolder = self.build_exe + self.libfolder = Path(self.buildfolder, "lib") + self.library = Path(self.libfolder, "library.zip") + + def installfile(self, path, keep_content=False): + folder = self.buildfolder + print('copying', path, '->', folder) + if path.is_dir(): + folder /= path.name + if folder.is_dir() and not keep_content: + shutil.rmtree(folder) + shutil.copytree(path, folder, dirs_exist_ok=True) + elif path.is_file(): + shutil.copy(path, folder) + else: + print('Warning,', path, 'not found') + + def create_manifest(self, create_hashes=False): + # Since the setup is now split into components and the manifest is not, + # it makes most sense to just remove the hashes for now. Not aware of anyone using them. + hashes = {} + manifestpath = os.path.join(self.buildfolder, "manifest.json") + if create_hashes: + from concurrent.futures import ThreadPoolExecutor + pool = ThreadPoolExecutor() + for dirpath, dirnames, filenames in os.walk(self.buildfolder): + for filename in filenames: + path = os.path.join(dirpath, filename) + hashes[os.path.relpath(path, start=self.buildfolder)] = pool.submit(_threaded_hash, path) + + import json + manifest = { + "buildtime": self.buildtime.isoformat(sep=" ", timespec="seconds"), + "hashes": {path: hash.result() for path, hash in hashes.items()}, + "version": version_tuple} + + json.dump(manifest, open(manifestpath, "wt"), indent=4) + print("Created Manifest") + + def run(self): + # pre build steps + print(f"Outputting to: {self.buildfolder}") + os.makedirs(self.buildfolder, exist_ok=True) + import ModuleUpdate + ModuleUpdate.requirements_files.add(os.path.join("WebHostLib", "requirements.txt")) + ModuleUpdate.update(yes=self.yes) + + # regular cx build + self.buildtime = datetime.datetime.utcnow() + super().run() + + # post build steps + if sys.platform == "win32": # kivy_deps is win32 only, linux picks them up automatically + from kivy_deps import sdl2, glew + for folder in sdl2.dep_bins + glew.dep_bins: + shutil.copytree(folder, self.libfolder, dirs_exist_ok=True) + print('copying', folder, '->', self.libfolder) + + for data in self.extra_data: + self.installfile(Path(data)) + + os.makedirs(self.buildfolder / "Players" / "Templates", exist_ok=True) + from WebHostLib.options import create + create() + from worlds.AutoWorld import AutoWorldRegister + for worldname, worldtype in AutoWorldRegister.world_types.items(): + if not worldtype.hidden: + file_name = worldname+".yaml" + shutil.copyfile(os.path.join("WebHostLib", "static", "generated", "configs", file_name), + self.buildfolder / "Players" / "Templates" / file_name) + shutil.copyfile("meta.yaml", self.buildfolder / "Players" / "Templates" / "meta.yaml") + + try: + from maseya import z3pr + except ImportError: + print("Maseya Palette Shuffle not found, skipping data files.") + z3pr = None + else: + # maseya Palette Shuffle exists and needs its data files + print("Maseya Palette Shuffle found, including data files...") + file = z3pr.__file__ + self.installfile(Path(os.path.dirname(file)) / "data", keep_content=True) + + if signtool: + for exe in self.distribution.executables: + print(f"Signing {exe.target_name}") + os.system(signtool + os.path.join(self.buildfolder, exe.target_name)) + print(f"Signing SNI") + os.system(signtool + os.path.join(self.buildfolder, "SNI", "SNI.exe")) + print(f"Signing OoT Utils") + for exe_path in (("Compress", "Compress.exe"), ("Decompress", "Decompress.exe")): + os.system(signtool + os.path.join(self.buildfolder, "lib", "worlds", "oot", "data", *exe_path)) + + remove_sprites_from_folder(self.buildfolder / "data" / "sprites" / "alttpr") + + self.create_manifest() + + if is_windows: + with open("setup.ini", "w") as f: + min_supported_windows = "6.2.9200" if sys.version_info > (3, 9) else "6.0.6000" + f.write(f"[Data]\nsource_path={self.buildfolder}\nmin_windows={min_supported_windows}\n") + else: + # make sure extra programs are executable + enemizer_exe = self.buildfolder / 'EnemizerCLI/EnemizerCLI.Core' + sni_exe = self.buildfolder / 'SNI/sni' + extra_exes = (enemizer_exe, sni_exe) + for extra_exe in extra_exes: + if extra_exe.is_file(): + extra_exe.chmod(0o755) + # rewrite windows-specific things in host.yaml + host_yaml = self.buildfolder / 'host.yaml' + with host_yaml.open('r+b') as f: + data = f.read() + data = data.replace(b'EnemizerCLI.Core.exe', b'EnemizerCLI.Core') + data = data.replace(b'factorio\\\\bin\\\\x64\\\\factorio', b'factorio/bin/x64/factorio') + f.seek(0, os.SEEK_SET) + f.write(data) + f.truncate() + + +class AppImageCommand(setuptools.Command): + description = "build an app image from build output" + user_options = [ + ("build-folder=", None, "Folder to convert to AppImage."), + ("dist-file=", None, "AppImage output file."), + ("app-dir=", None, "Folder to use for packaging."), + ("app-icon=", None, "The icon to use for the AppImage."), + ("app-exec=", None, "The application to run inside the image."), + ("yes", "y", 'Answer "yes" to all questions.'), + ] + build_folder: typing.Optional[Path] + dist_file: typing.Optional[Path] + app_dir: typing.Optional[Path] + app_name: str + app_exec: typing.Optional[Path] + app_icon: typing.Optional[Path] # source file + app_id: str # lower case name, used for icon and .desktop + yes: bool + + def write_desktop(self): + desktop_filename = self.app_dir / f'{self.app_id}.desktop' + with open(desktop_filename, 'w', encoding="utf-8") as f: + f.write("\n".join(( + "[Desktop Entry]", + f'Name={self.app_name}', + f'Exec={self.app_exec}', + "Type=Application", + "Categories=Game", + f'Icon={self.app_id}', + '' + ))) + desktop_filename.chmod(0o755) + + def write_launcher(self, default_exe: Path): + launcher_filename = self.app_dir / f'AppRun' + with open(launcher_filename, 'w', encoding="utf-8") as f: + f.write(f"""#!/bin/sh +exe="{default_exe}" +match="${{1#--executable=}}" +if [ "${{#match}}" -lt "${{#1}}" ]; then + exe="$match" + shift +elif [ "$1" == "-executable" ] || [ "$1" == "--executable" ]; then + exe="$2" + shift; shift +fi +tmp="${{exe#*/}}" +if [ ! "${{#tmp}}" -lt "${{#exe}}" ]; then + exe="{default_exe.parent}/$exe" +fi +$APPDIR/$exe "$@" +""") + launcher_filename.chmod(0o755) + + def install_icon(self, src: Path, name: typing.Optional[str] = None, symlink: typing.Optional[Path] = None): + try: + from PIL import Image + except ModuleNotFoundError: + if not self.yes: + input(f'Requirement PIL is not satisfied, press enter to install it') + subprocess.call([sys.executable, '-m', 'pip', 'install', 'Pillow', '--upgrade']) + from PIL import Image + im = Image.open(src) + res, _ = im.size + + if not name: + name = src.stem + ext = src.suffix + dest_dir = Path(self.app_dir / f'usr/share/icons/hicolor/{res}x{res}/apps') + dest_dir.mkdir(parents=True, exist_ok=True) + dest_file = dest_dir / f'{name}{ext}' + shutil.copy(src, dest_file) + if symlink: + symlink.symlink_to(dest_file.relative_to(symlink.parent)) + + def initialize_options(self): + self.build_folder = None + self.app_dir = None + self.app_name = self.distribution.metadata.name + self.app_icon = self.distribution.executables[0].icon + self.app_exec = Path('opt/{app_name}/{exe}'.format( + app_name=self.distribution.metadata.name, exe=self.distribution.executables[0].target_name + )) + self.dist_file = Path("dist", "{app_name}_{app_version}_{platform}.AppImage".format( + app_name=self.distribution.metadata.name, app_version=self.distribution.metadata.version, + platform=sysconfig.get_platform() + )) + self.yes = False + + def finalize_options(self): + if not self.app_dir: + self.app_dir = self.build_folder.parent / "AppDir" + self.app_id = self.app_name.lower() + + def run(self): + self.dist_file.parent.mkdir(parents=True, exist_ok=True) + if self.app_dir.is_dir(): + shutil.rmtree(self.app_dir) + self.app_dir.mkdir(parents=True) + opt_dir = self.app_dir / "opt" / self.distribution.metadata.name + shutil.copytree(self.build_folder, opt_dir) + root_icon = self.app_dir / f'{self.app_id}{self.app_icon.suffix}' + self.install_icon(self.app_icon, self.app_id, symlink=root_icon) + shutil.copy(root_icon, self.app_dir / '.DirIcon') + self.write_desktop() + self.write_launcher(self.app_exec) + print(f'{self.app_dir} -> {self.dist_file}') + subprocess.call(f'./appimagetool -n "{self.app_dir}" "{self.dist_file}"', shell=True) -buildtime = datetime.datetime.utcnow() cx_Freeze.setup( name="Archipelago", @@ -130,71 +352,18 @@ cx_Freeze.setup( "include_msvcr": False, "replace_paths": [("*", "")], "optimize": 1, - "build_exe": buildfolder + "build_exe": buildfolder, + "extra_data": extra_data, + "bin_includes": [] if is_windows else ["libffi.so"] + }, + "bdist_appimage": { + "build_folder": buildfolder, }, }, + # override commands to get custom stuff in + cmdclass={ + "build": BuildCommand, + "build_exe": BuildExeCommand, + "bdist_appimage": AppImageCommand, + }, ) - - -def installfile(path, keep_content=False): - lbuildfolder = buildfolder - print('copying', path, '->', lbuildfolder) - if path.is_dir(): - lbuildfolder /= path.name - if lbuildfolder.is_dir() and not keep_content: - shutil.rmtree(lbuildfolder) - shutil.copytree(path, lbuildfolder, dirs_exist_ok=True) - elif path.is_file(): - shutil.copy(path, lbuildfolder) - else: - print('Warning,', path, 'not found') - - -for folder in sdl2.dep_bins + glew.dep_bins: - shutil.copytree(folder, libfolder, dirs_exist_ok=True) - print('copying', folder, '->', libfolder) - -extra_data = ["LICENSE", "data", "EnemizerCLI", "host.yaml", "SNI"] - -for data in extra_data: - installfile(Path(data)) - -os.makedirs(buildfolder / "Players" / "Templates", exist_ok=True) -from WebHostLib.options import create -create() -from worlds.AutoWorld import AutoWorldRegister -for worldname, worldtype in AutoWorldRegister.world_types.items(): - if not worldtype.hidden: - file_name = worldname+".yaml" - shutil.copyfile(os.path.join("WebHostLib", "static", "generated", "configs", file_name), - buildfolder / "Players" / "Templates" / file_name) -shutil.copyfile("meta.yaml", buildfolder / "Players" / "Templates" / "meta.yaml") - -try: - from maseya import z3pr -except ImportError: - print("Maseya Palette Shuffle not found, skipping data files.") -else: - # maseya Palette Shuffle exists and needs its data files - print("Maseya Palette Shuffle found, including data files...") - file = z3pr.__file__ - installfile(Path(os.path.dirname(file)) / "data", keep_content=True) - -if signtool: - for exe in exes: - print(f"Signing {exe.target_name}") - os.system(signtool + os.path.join(buildfolder, exe.target_name)) - print(f"Signing SNI") - os.system(signtool + os.path.join(buildfolder, "SNI", "SNI.exe")) - print(f"Signing OoT Utils") - for exe_path in (("Compress", "Compress.exe"), ("Decompress", "Decompress.exe")): - os.system(signtool + os.path.join(buildfolder, "lib", "worlds", "oot", "data", *exe_path)) - -remove_sprites_from_folder(buildfolder / "data" / "sprites" / "alttpr") - -manifest_creation(buildfolder) - -if sys.platform == "win32": - with open("setup.ini", "w") as f: - min_supported_windows = "6.2.9200" if sys.version_info > (3, 9) else "6.0.6000" - f.write(f"[Data]\nsource_path={buildfolder}\nmin_windows={min_supported_windows}\n") diff --git a/worlds/alttp/Rom.py b/worlds/alttp/Rom.py index bcde3473..9a602ae0 100644 --- a/worlds/alttp/Rom.py +++ b/worlds/alttp/Rom.py @@ -34,7 +34,7 @@ from worlds.alttp.Text import KingsReturn_texts, Sanctuary_texts, Kakariko_texts DeathMountain_texts, \ LostWoods_texts, WishingWell_texts, DesertPalace_texts, MountainTower_texts, LinksHouse_texts, Lumberjacks_texts, \ SickKid_texts, FluteBoy_texts, Zora_texts, MagicShop_texts, Sahasrahla_names -from Utils import local_path, int16_as_bytes, int32_as_bytes, snes_to_pc, is_frozen +from Utils import local_path, user_path, int16_as_bytes, int32_as_bytes, snes_to_pc, is_frozen from worlds.alttp.Items import ItemFactory, item_table, item_name_groups, progression_items from worlds.alttp.EntranceShuffle import door_addresses from worlds.alttp.Options import smallkey_shuffle @@ -140,7 +140,7 @@ class LocalRom(object): _, target, buffer = Patch.create_rom_bytes(local_path('data', 'basepatch.apbp'), ignore_version=True) if self.verify(buffer): self.buffer = bytearray(buffer) - with open(local_path('basepatch.sfc'), 'wb') as stream: + with open(user_path('basepatch.sfc'), 'wb') as stream: stream.write(buffer) return raise RuntimeError('Base patch unverified. Unable to continue.') @@ -497,7 +497,7 @@ def _populate_sprite_table(): logging.debug(f"Spritefile {file} could not be loaded as a valid sprite.") with concurrent.futures.ThreadPoolExecutor() as pool: - for dir in [local_path('data', 'sprites', 'alttpr'), local_path('data', 'sprites', 'custom')]: + for dir in [user_path('data', 'sprites', 'alttpr'), user_path('data', 'sprites', 'custom')]: for file in os.listdir(dir): pool.submit(load_sprite_from_file, os.path.join(dir, file)) @@ -2914,5 +2914,5 @@ def get_base_rom_path(file_name: str = "") -> str: if not file_name: file_name = options["lttp_options"]["rom_file"] if not os.path.exists(file_name): - file_name = Utils.local_path(file_name) + file_name = Utils.user_path(file_name) return file_name diff --git a/worlds/oot/Rom.py b/worlds/oot/Rom.py index 8c4129e2..939b4e87 100644 --- a/worlds/oot/Rom.py +++ b/worlds/oot/Rom.py @@ -6,7 +6,7 @@ import subprocess import copy import threading from .Utils import subprocess_args, data_path, get_version_bytes, __version__ -from Utils import local_path +from Utils import user_path from .ntype import BigStream from .crc import calculate_crc @@ -27,7 +27,7 @@ class Rom(BigStream): if file is None: return - decomp_file = local_path('ZOOTDEC.z64') + decomp_file = user_path('ZOOTDEC.z64') with open(data_path('generated/symbols.json'), 'r') as stream: symbols = json.load(stream) diff --git a/worlds/sm/Rom.py b/worlds/sm/Rom.py index e982258d..84ed131f 100644 --- a/worlds/sm/Rom.py +++ b/worlds/sm/Rom.py @@ -38,5 +38,5 @@ def get_base_rom_path(file_name: str = "") -> str: if not file_name: file_name = options["sm_options"]["rom_file"] if not os.path.exists(file_name): - file_name = Utils.local_path(file_name) + file_name = Utils.user_path(file_name) return file_name diff --git a/worlds/smz3/Rom.py b/worlds/smz3/Rom.py index 83f4d396..9c468fc5 100644 --- a/worlds/smz3/Rom.py +++ b/worlds/smz3/Rom.py @@ -48,7 +48,7 @@ def get_sm_base_rom_path(file_name: str = "") -> str: if not file_name: file_name = options["sm_options"]["rom_file"] if not os.path.exists(file_name): - file_name = Utils.local_path(file_name) + file_name = Utils.user_path(file_name) return file_name @@ -57,7 +57,7 @@ def get_lttp_base_rom_path(file_name: str = "") -> str: if not file_name: file_name = options["lttp_options"]["rom_file"] if not os.path.exists(file_name): - file_name = Utils.local_path(file_name) + file_name = Utils.user_path(file_name) return file_name diff --git a/worlds/soe/Patch.py b/worlds/soe/Patch.py index 5478e935..21bdd942 100644 --- a/worlds/soe/Patch.py +++ b/worlds/soe/Patch.py @@ -3,6 +3,7 @@ import yaml from typing import Optional import Utils from Patch import APDeltaPatch +import os USHASH = '6e9c94511d04fac6e0a1e582c170be3a' @@ -19,8 +20,13 @@ class SoEDeltaPatch(APDeltaPatch): return read_rom(stream) -def get_base_rom_path() -> str: - return Utils.get_options()['soe_options']['rom_file'] +def get_base_rom_path(file_name: Optional[str] = None) -> str: + options = Utils.get_options() + if not file_name: + file_name = options["soe_options"]["rom_file"] + if not os.path.exists(file_name): + file_name = Utils.user_path(file_name) + return file_name def read_rom(stream, strip_header=True) -> bytes: