ModuleUpdate/Setup: install pkg_resources, check pip, typing and cleanup (#1593)

* ModuleUpdater/setup: install pkg_resources and check for pip

plus minor cleanup in the github actions

* ModuleUpdate/setup: make flake8 happy

* ModuleUpdate/setup: make mypy happier

.github/workflows/build.yml

@@ -36,8 +36,7 @@ jobs:
           Expand-Archive -Path enemizer.zip -DestinationPath EnemizerCLI -Force
       - name: Build
         run: |
-          python -m pip install --upgrade pip setuptools
+          python -m pip install --upgrade pip
-          pip install -r requirements.txt
           python setup.py build_exe --yes
           $NAME="$(ls build)".Split('.',2)[1]
           $ZIP_NAME="Archipelago_$NAME.7z"
@@ -85,8 +84,7 @@ jobs:
           # charset-normalizer was somehow incomplete in the github runner
           "${{ env.PYTHON }}" -m venv venv
           source venv/bin/activate
-          "${{ env.PYTHON }}" -m pip install --upgrade pip PyGObject setuptools charset-normalizer
+          "${{ env.PYTHON }}" -m pip install --upgrade pip PyGObject charset-normalizer
-          pip install -r requirements.txt
           python setup.py build_exe --yes bdist_appimage --yes
           echo -e "setup.py build output:\n `ls build`"
           echo -e "setup.py dist output:\n `ls dist`"

.github/workflows/lint.yml

@@ -12,7 +12,7 @@ on:
       - '**.py'
 
 jobs:
-  build:
+  flake8:
 
     runs-on: ubuntu-latest
 
@@ -25,7 +25,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip wheel
-        pip install flake8 pytest pytest-subtests
+        pip install flake8
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
     - name: Lint with flake8
       run: |

.github/workflows/release.yml

@@ -63,9 +63,8 @@ jobs:
           # charset-normalizer was somehow incomplete in the github runner
           "${{ env.PYTHON }}" -m venv venv
           source venv/bin/activate
-          "${{ env.PYTHON }}" -m pip install --upgrade pip PyGObject setuptools charset-normalizer
+          "${{ env.PYTHON }}" -m pip install --upgrade pip PyGObject charset-normalizer
-          pip install -r requirements.txt
+          python setup.py build_exe --yes bdist_appimage --yes
-          python setup.py build --yes bdist_appimage --yes
           echo -e "setup.py build output:\n `ls build`"
           echo -e "setup.py dist output:\n `ls dist`"
           cd dist && export APPIMAGE_NAME="`ls *.AppImage`" && cd ..

.github/workflows/unittests.yml

@@ -52,8 +52,8 @@ jobs:
         python-version: ${{ matrix.python.version }}
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip wheel
+        python -m pip install --upgrade pip
-        pip install flake8 pytest pytest-subtests
+        pip install pytest pytest-subtests
         python ModuleUpdate.py --yes --force --append "WebHostLib/requirements.txt"
     - name: Unittests
       run: |

ModuleUpdate.py

@@ -1,7 +1,6 @@
 import os
 import sys
 import subprocess
-import pkg_resources
 import warnings
 
 local_dir = os.path.dirname(__file__)
@@ -22,18 +21,50 @@ if not update_ran:
                     requirements_files.add(req_file)
 
 
+def check_pip():
+    # detect if pip is available
+    try:
+        import pip  # noqa: F401
+    except ImportError:
+        raise RuntimeError("pip not available. Please install pip.")
+
+
+def confirm(msg: str):
+    try:
+        input(f"\n{msg}")
+    except KeyboardInterrupt:
+        print("\nAborting")
+        sys.exit(1)
+
+
 def update_command():
+    check_pip()
     for file in requirements_files:
-        subprocess.call([sys.executable, '-m', 'pip', 'install', '-r', file, '--upgrade'])
+        subprocess.call([sys.executable, "-m", "pip", "install", "-r", file, "--upgrade"])
+
+
+def install_pkg_resources(yes=False):
+    try:
+        import pkg_resources  # noqa: F401
+    except ImportError:
+        check_pip()
+        if not yes:
+            confirm("pkg_resources not found, press enter to install it")
+        subprocess.call([sys.executable, "-m", "pip", "install", "--upgrade", "setuptools"])
 
 
 def update(yes=False, force=False):
     global update_ran
     if not update_ran:
         update_ran = True
+
         if force:
             update_command()
             return
+
+        install_pkg_resources(yes=yes)
+        import pkg_resources
+
         for req_file in requirements_files:
             path = os.path.join(os.path.dirname(sys.argv[0]), req_file)
             if not os.path.exists(path):
@@ -52,7 +83,7 @@ def update(yes=False, force=False):
                             egg = egg.split(";", 1)[0].rstrip()
                             if any(compare in egg for compare in ("==", ">=", ">", "<", "<=", "!=")):
                                 warnings.warn(f"Specifying version as #egg={egg} will become unavailable in pip 25.0. "
-                                               "Use name @ url#version instead.", DeprecationWarning)
+                                              "Use name @ url#version instead.", DeprecationWarning)
                                 line = egg
                         else:
                             egg = ""
@@ -79,11 +110,7 @@ def update(yes=False, force=False):
                             if not yes:
                                 import traceback
                                 traceback.print_exc()
-                                try:
+                                confirm(f"Requirement {requirement} is not satisfied, press enter to install it")
-                                    input(f"\nRequirement {requirement} is not satisfied, press enter to install it")
-                                except KeyboardInterrupt:
-                                    print("\nAborting")
-                                    sys.exit(1)
                             update_command()
                             return
 

setup.py

@@ -12,7 +12,6 @@ import io
 import json
 import threading
 import subprocess
-import pkg_resources
 
 from collections.abc import Iterable
 from hashlib import sha3_512
@@ -22,12 +21,27 @@ from pathlib import Path
 # This is a bit jank. We need cx-Freeze to be able to run anything from this script, so install it
 try:
     requirement = 'cx-Freeze>=6.14.7'
-    pkg_resources.require(requirement)
+    import pkg_resources
-    import cx_Freeze
+    try:
-except pkg_resources.ResolutionError:
+        pkg_resources.require(requirement)
+        install_cx_freeze = False
+    except pkg_resources.ResolutionError:
+        install_cx_freeze = True
+except ImportError:
+    install_cx_freeze = True
+    pkg_resources = None  # type: ignore [assignment]
+
+if install_cx_freeze:
+    # check if pip is available
+    try:
+        import pip  # noqa: F401
+    except ImportError:
+        raise RuntimeError("pip not available. Please install pip.")
+    # install and import cx_freeze
     if '--yes' not in sys.argv and '-y' not in sys.argv:
         input(f'Requirement {requirement} is not satisfied, press enter to install it')
     subprocess.call([sys.executable, '-m', 'pip', 'install', requirement, '--upgrade'])
+    import pkg_resources
     import cx_Freeze
 
 # .build only exists if cx-Freeze is the right version, so we have to update/install that first before this line
@@ -120,6 +134,7 @@ def download_SNI():
         print(f"No SNI found for system spec {platform_name} {machine_name}")
 
 
+signtool: typing.Optional[str]
 if os.path.exists("X:/pw.txt"):
     print("Using signtool")
     with open("X:/pw.txt", encoding="utf-8-sig") as f:
@@ -144,7 +159,7 @@ exes = [
         target_name=c.frozen_name + (".exe" if is_windows else ""),
         icon=icon_paths[c.icon],
         base="Win32GUI" if is_windows and not c.cli else None
-    ) for c in components if c.script_name
+    ) for c in components if c.script_name and c.frozen_name
 ]
 
 extra_data = ["LICENSE", "data", "EnemizerCLI", "host.yaml", "SNI"]
@@ -307,7 +322,6 @@ class BuildExeCommand(cx_Freeze.command.build_exe.BuildEXE):
                 # which should be ok
                 with zipfile.ZipFile(self.libfolder / "worlds" / (file_name + ".apworld"), "x", zipfile.ZIP_DEFLATED,
                                      compresslevel=9) as zf:
-                    entry: os.DirEntry
                     for path in world_directory.rglob("*.*"):
                         relative_path = os.path.join(*path.parts[path.parts.index("worlds")+1:])
                         zf.write(path, relative_path)
@@ -330,9 +344,9 @@ class BuildExeCommand(cx_Freeze.command.build_exe.BuildEXE):
             for exe in self.distribution.executables:
                 print(f"Signing {exe.target_name}")
                 os.system(signtool + os.path.join(self.buildfolder, exe.target_name))
-            print(f"Signing SNI")
+            print("Signing SNI")
             os.system(signtool + os.path.join(self.buildfolder, "SNI", "SNI.exe"))
-            print(f"Signing OoT Utils")
+            print("Signing OoT Utils")
             for exe_path in (("Compress", "Compress.exe"), ("Decompress", "Decompress.exe")):
                 os.system(signtool + os.path.join(self.buildfolder, "lib", "worlds", "oot", "data", *exe_path))
 
@@ -386,7 +400,8 @@ class AppImageCommand(setuptools.Command):
     yes: bool
 
     def write_desktop(self):
-        desktop_filename = self.app_dir / f'{self.app_id}.desktop'
+        assert self.app_dir, "Invalid app_dir"
+        desktop_filename = self.app_dir / f"{self.app_id}.desktop"
         with open(desktop_filename, 'w', encoding="utf-8") as f:
             f.write("\n".join((
                 "[Desktop Entry]",
@@ -400,7 +415,8 @@ class AppImageCommand(setuptools.Command):
         desktop_filename.chmod(0o755)
 
     def write_launcher(self, default_exe: Path):
-        launcher_filename = self.app_dir / f'AppRun'
+        assert self.app_dir, "Invalid app_dir"
+        launcher_filename = self.app_dir / "AppRun"
         with open(launcher_filename, 'w', encoding="utf-8") as f:
             f.write(f"""#!/bin/sh
 exe="{default_exe}"
@@ -422,11 +438,12 @@ $APPDIR/$exe "$@"
         launcher_filename.chmod(0o755)
 
     def install_icon(self, src: Path, name: typing.Optional[str] = None, symlink: typing.Optional[Path] = None):
+        assert self.app_dir, "Invalid app_dir"
         try:
             from PIL import Image
         except ModuleNotFoundError:
             if not self.yes:
-                input(f'Requirement PIL is not satisfied, press enter to install it')
+                input("Requirement PIL is not satisfied, press enter to install it")
             subprocess.call([sys.executable, '-m', 'pip', 'install', 'Pillow', '--upgrade'])
             from PIL import Image
         im = Image.open(src)
@@ -503,8 +520,12 @@ def find_libs(*args: str) -> typing.Sequence[typing.Tuple[str, str]]:
         return (lib, lib_arch, lib_libc), path
 
     if not hasattr(find_libs, "cache"):
-        data = subprocess.run([shutil.which('ldconfig'), '-p'], capture_output=True, text=True).stdout.split('\n')[1:]
+        ldconfig = shutil.which("ldconfig")
-        find_libs.cache = {k: v for k, v in (parse(line) for line in data if '=>' in line)}
+        assert ldconfig, "Make sure ldconfig is in PATH"
+        data = subprocess.run([ldconfig, "-p"], capture_output=True, text=True).stdout.split("\n")[1:]
+        find_libs.cache = {  # type: ignore [attr-defined]
+            k: v for k, v in (parse(line) for line in data if "=>" in line)
+        }
 
     def find_lib(lib, arch, libc):
         for k, v in find_libs.cache.items():