From ffd7d5da747deff4dd969adc6b2b3bce190b2969 Mon Sep 17 00:00:00 2001 From: black-sliver <59490463+black-sliver@users.noreply.github.com> Date: Sat, 25 Mar 2023 19:54:42 +0100 Subject: [PATCH] ModuleUpdate/Setup: install pkg_resources, check pip, typing and cleanup (#1593) * ModuleUpdater/setup: install pkg_resources and check for pip plus minor cleanup in the github actions * ModuleUpdate/setup: make flake8 happy * ModuleUpdate/setup: make mypy happier --- .github/workflows/build.yml | 6 ++--- .github/workflows/lint.yml | 4 +-- .github/workflows/release.yml | 5 ++-- .github/workflows/unittests.yml | 4 +-- ModuleUpdate.py | 43 ++++++++++++++++++++++++------ setup.py | 47 ++++++++++++++++++++++++--------- 6 files changed, 77 insertions(+), 32 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d4e1efd4..5ab2d18e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -36,8 +36,7 @@ jobs: Expand-Archive -Path enemizer.zip -DestinationPath EnemizerCLI -Force - name: Build run: | - python -m pip install --upgrade pip setuptools - pip install -r requirements.txt + python -m pip install --upgrade pip python setup.py build_exe --yes $NAME="$(ls build)".Split('.',2)[1] $ZIP_NAME="Archipelago_$NAME.7z" @@ -85,8 +84,7 @@ jobs: # charset-normalizer was somehow incomplete in the github runner "${{ env.PYTHON }}" -m venv venv source venv/bin/activate - "${{ env.PYTHON }}" -m pip install --upgrade pip PyGObject setuptools charset-normalizer - pip install -r requirements.txt + "${{ env.PYTHON }}" -m pip install --upgrade pip PyGObject charset-normalizer python setup.py build_exe --yes bdist_appimage --yes echo -e "setup.py build output:\n `ls build`" echo -e "setup.py dist output:\n `ls dist`" diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 7ecda45e..c20d244a 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -12,7 +12,7 @@ on: - '**.py' jobs: - build: + flake8: runs-on: ubuntu-latest @@ -25,7 +25,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip wheel - pip install flake8 pytest pytest-subtests + pip install flake8 if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Lint with flake8 run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fa3dd321..4f1a8eec 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -63,9 +63,8 @@ jobs: # charset-normalizer was somehow incomplete in the github runner "${{ env.PYTHON }}" -m venv venv source venv/bin/activate - "${{ env.PYTHON }}" -m pip install --upgrade pip PyGObject setuptools charset-normalizer - pip install -r requirements.txt - python setup.py build --yes bdist_appimage --yes + "${{ env.PYTHON }}" -m pip install --upgrade pip PyGObject charset-normalizer + python setup.py build_exe --yes bdist_appimage --yes echo -e "setup.py build output:\n `ls build`" echo -e "setup.py dist output:\n `ls dist`" cd dist && export APPIMAGE_NAME="`ls *.AppImage`" && cd .. diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index 93be745a..254d92dd 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -52,8 +52,8 @@ jobs: python-version: ${{ matrix.python.version }} - name: Install dependencies run: | - python -m pip install --upgrade pip wheel - pip install flake8 pytest pytest-subtests + python -m pip install --upgrade pip + pip install pytest pytest-subtests python ModuleUpdate.py --yes --force --append "WebHostLib/requirements.txt" - name: Unittests run: | diff --git a/ModuleUpdate.py b/ModuleUpdate.py index a8258ce1..ac40dbd6 100644 --- a/ModuleUpdate.py +++ b/ModuleUpdate.py @@ -1,7 +1,6 @@ import os import sys import subprocess -import pkg_resources import warnings local_dir = os.path.dirname(__file__) @@ -22,18 +21,50 @@ if not update_ran: requirements_files.add(req_file) +def check_pip(): + # detect if pip is available + try: + import pip # noqa: F401 + except ImportError: + raise RuntimeError("pip not available. Please install pip.") + + +def confirm(msg: str): + try: + input(f"\n{msg}") + except KeyboardInterrupt: + print("\nAborting") + sys.exit(1) + + def update_command(): + check_pip() for file in requirements_files: - subprocess.call([sys.executable, '-m', 'pip', 'install', '-r', file, '--upgrade']) + subprocess.call([sys.executable, "-m", "pip", "install", "-r", file, "--upgrade"]) + + +def install_pkg_resources(yes=False): + try: + import pkg_resources # noqa: F401 + except ImportError: + check_pip() + if not yes: + confirm("pkg_resources not found, press enter to install it") + subprocess.call([sys.executable, "-m", "pip", "install", "--upgrade", "setuptools"]) def update(yes=False, force=False): global update_ran if not update_ran: update_ran = True + if force: update_command() return + + install_pkg_resources(yes=yes) + import pkg_resources + for req_file in requirements_files: path = os.path.join(os.path.dirname(sys.argv[0]), req_file) if not os.path.exists(path): @@ -52,7 +83,7 @@ def update(yes=False, force=False): egg = egg.split(";", 1)[0].rstrip() if any(compare in egg for compare in ("==", ">=", ">", "<", "<=", "!=")): warnings.warn(f"Specifying version as #egg={egg} will become unavailable in pip 25.0. " - "Use name @ url#version instead.", DeprecationWarning) + "Use name @ url#version instead.", DeprecationWarning) line = egg else: egg = "" @@ -79,11 +110,7 @@ def update(yes=False, force=False): if not yes: import traceback traceback.print_exc() - try: - input(f"\nRequirement {requirement} is not satisfied, press enter to install it") - except KeyboardInterrupt: - print("\nAborting") - sys.exit(1) + confirm(f"Requirement {requirement} is not satisfied, press enter to install it") update_command() return diff --git a/setup.py b/setup.py index 66cbb722..e693f849 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,6 @@ import io import json import threading import subprocess -import pkg_resources from collections.abc import Iterable from hashlib import sha3_512 @@ -22,12 +21,27 @@ from pathlib import Path # This is a bit jank. We need cx-Freeze to be able to run anything from this script, so install it try: requirement = 'cx-Freeze>=6.14.7' - pkg_resources.require(requirement) - import cx_Freeze -except pkg_resources.ResolutionError: + import pkg_resources + try: + pkg_resources.require(requirement) + install_cx_freeze = False + except pkg_resources.ResolutionError: + install_cx_freeze = True +except ImportError: + install_cx_freeze = True + pkg_resources = None # type: ignore [assignment] + +if install_cx_freeze: + # check if pip is available + try: + import pip # noqa: F401 + except ImportError: + raise RuntimeError("pip not available. Please install pip.") + # install and import cx_freeze if '--yes' not in sys.argv and '-y' not in sys.argv: input(f'Requirement {requirement} is not satisfied, press enter to install it') subprocess.call([sys.executable, '-m', 'pip', 'install', requirement, '--upgrade']) + import pkg_resources import cx_Freeze # .build only exists if cx-Freeze is the right version, so we have to update/install that first before this line @@ -120,6 +134,7 @@ def download_SNI(): print(f"No SNI found for system spec {platform_name} {machine_name}") +signtool: typing.Optional[str] if os.path.exists("X:/pw.txt"): print("Using signtool") with open("X:/pw.txt", encoding="utf-8-sig") as f: @@ -144,7 +159,7 @@ exes = [ target_name=c.frozen_name + (".exe" if is_windows else ""), icon=icon_paths[c.icon], base="Win32GUI" if is_windows and not c.cli else None - ) for c in components if c.script_name + ) for c in components if c.script_name and c.frozen_name ] extra_data = ["LICENSE", "data", "EnemizerCLI", "host.yaml", "SNI"] @@ -307,7 +322,6 @@ class BuildExeCommand(cx_Freeze.command.build_exe.BuildEXE): # which should be ok with zipfile.ZipFile(self.libfolder / "worlds" / (file_name + ".apworld"), "x", zipfile.ZIP_DEFLATED, compresslevel=9) as zf: - entry: os.DirEntry for path in world_directory.rglob("*.*"): relative_path = os.path.join(*path.parts[path.parts.index("worlds")+1:]) zf.write(path, relative_path) @@ -330,9 +344,9 @@ class BuildExeCommand(cx_Freeze.command.build_exe.BuildEXE): for exe in self.distribution.executables: print(f"Signing {exe.target_name}") os.system(signtool + os.path.join(self.buildfolder, exe.target_name)) - print(f"Signing SNI") + print("Signing SNI") os.system(signtool + os.path.join(self.buildfolder, "SNI", "SNI.exe")) - print(f"Signing OoT Utils") + print("Signing OoT Utils") for exe_path in (("Compress", "Compress.exe"), ("Decompress", "Decompress.exe")): os.system(signtool + os.path.join(self.buildfolder, "lib", "worlds", "oot", "data", *exe_path)) @@ -386,7 +400,8 @@ class AppImageCommand(setuptools.Command): yes: bool def write_desktop(self): - desktop_filename = self.app_dir / f'{self.app_id}.desktop' + assert self.app_dir, "Invalid app_dir" + desktop_filename = self.app_dir / f"{self.app_id}.desktop" with open(desktop_filename, 'w', encoding="utf-8") as f: f.write("\n".join(( "[Desktop Entry]", @@ -400,7 +415,8 @@ class AppImageCommand(setuptools.Command): desktop_filename.chmod(0o755) def write_launcher(self, default_exe: Path): - launcher_filename = self.app_dir / f'AppRun' + assert self.app_dir, "Invalid app_dir" + launcher_filename = self.app_dir / "AppRun" with open(launcher_filename, 'w', encoding="utf-8") as f: f.write(f"""#!/bin/sh exe="{default_exe}" @@ -422,11 +438,12 @@ $APPDIR/$exe "$@" launcher_filename.chmod(0o755) def install_icon(self, src: Path, name: typing.Optional[str] = None, symlink: typing.Optional[Path] = None): + assert self.app_dir, "Invalid app_dir" try: from PIL import Image except ModuleNotFoundError: if not self.yes: - input(f'Requirement PIL is not satisfied, press enter to install it') + input("Requirement PIL is not satisfied, press enter to install it") subprocess.call([sys.executable, '-m', 'pip', 'install', 'Pillow', '--upgrade']) from PIL import Image im = Image.open(src) @@ -503,8 +520,12 @@ def find_libs(*args: str) -> typing.Sequence[typing.Tuple[str, str]]: return (lib, lib_arch, lib_libc), path if not hasattr(find_libs, "cache"): - data = subprocess.run([shutil.which('ldconfig'), '-p'], capture_output=True, text=True).stdout.split('\n')[1:] - find_libs.cache = {k: v for k, v in (parse(line) for line in data if '=>' in line)} + ldconfig = shutil.which("ldconfig") + assert ldconfig, "Make sure ldconfig is in PATH" + data = subprocess.run([ldconfig, "-p"], capture_output=True, text=True).stdout.split("\n")[1:] + find_libs.cache = { # type: ignore [attr-defined] + k: v for k, v in (parse(line) for line in data if "=>" in line) + } def find_lib(lib, arch, libc): for k, v in find_libs.cache.items():