[feature] Download/use standalone python build when chose --python ve…

…rsion doesn't exist (#1243) * set up cache dir for standalone pythons to live in * adapt yen's downloader / unpacker implementation to work well in pipx's codebase * idempotent downloader function to download any desired major/minor python version available * update `find_python_interpreter` to download the correct python version if it's not available on user's system * add `--fetch-missing-python` flag to relevant commands * add `PIPX_FETCH_MISSING_PYTHON` environment variable * refactor `download_python_build_standalone` to automatically cleanup temp files in case of errors * add news update for new feature * add "fetch missing" unit tests for unix and (hopefully) windows * improve error messaging for python-standalone errors * remove hard-coded version numbers from interpreter unit tests * Apply suggestions from code review, fixup unit tests Co-authored-by: chrysle <[email protected]> * implement changes requested by chrysle * implement changes requested by gitznik, fixup unit tests * group and de-duplicate python cli args * refactor unit tests, mock out github API response * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * add unit test for python_standalone failure mode * refactor unit tests, remove need to derive target python version * de-duplicate unit test Co-authored-by: chrysle <[email protected]> --------- Co-authored-by: chrysle <[email protected]> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: chrysle <[email protected]>
pypa · Feb 7, 2024 · 407b797 · 407b797
1 parent c2df428
commit 407b797
Show file tree

Hide file tree

Showing 8 changed files with 326 additions and 54 deletions.
diff --git a/changelog.d/1242.feature.md b/changelog.d/1242.feature.md
@@ -0,0 +1,3 @@
+Add a `--fetch-missing-python` flag to all commands that accept a `--python` flag.
+
+When combined, this will automatically download a standalone copy of the requested python version if it's not already available on the user's system.
diff --git a/src/pipx/commands/environment.py b/src/pipx/commands/environment.py
@@ -8,6 +8,7 @@
     PIPX_LOCAL_VENVS,
     PIPX_LOG_DIR,
     PIPX_SHARED_LIBS,
+    PIPX_STANDALONE_PYTHON_CACHEDIR,
     PIPX_TRASH_DIR,
     PIPX_VENV_CACHEDIR,
     ExitCode,
@@ -25,6 +26,7 @@ def environment(value: str) -> ExitCode:
         "PIPX_MAN_DIR",
         "PIPX_SHARED_LIBS",
         "PIPX_DEFAULT_PYTHON",
+        "PIPX_FETCH_MISSING_PYTHON",
         "USE_EMOJI",
     ]
     derived_values = {
@@ -36,6 +38,7 @@ def environment(value: str) -> ExitCode:
         "PIPX_LOG_DIR": PIPX_LOG_DIR,
         "PIPX_TRASH_DIR": PIPX_TRASH_DIR,
         "PIPX_VENV_CACHEDIR": PIPX_VENV_CACHEDIR,
+        "PIPX_STANDALONE_PYTHON_CACHEDIR": PIPX_STANDALONE_PYTHON_CACHEDIR,
         "PIPX_DEFAULT_PYTHON": DEFAULT_PYTHON,
         "USE_EMOJI": str(EMOJI_SUPPORT).lower(),
     }

diff --git a/src/pipx/constants.py b/src/pipx/constants.py
@@ -16,13 +16,15 @@
 if FALLBACK_PIPX_HOME.exists() or os.environ.get("PIPX_HOME") is not None:
     PIPX_HOME = Path(os.environ.get("PIPX_HOME", FALLBACK_PIPX_HOME)).resolve()
     PIPX_LOCAL_VENVS = PIPX_HOME / "venvs"
+    PIPX_STANDALONE_PYTHON_CACHEDIR = PIPX_HOME / "py"
     PIPX_LOG_DIR = PIPX_HOME / "logs"
     DEFAULT_PIPX_SHARED_LIBS = PIPX_HOME / "shared"
     PIPX_TRASH_DIR = PIPX_HOME / ".trash"
     PIPX_VENV_CACHEDIR = PIPX_HOME / ".cache"
 else:
     PIPX_HOME = DEFAULT_PIPX_HOME
     PIPX_LOCAL_VENVS = PIPX_HOME / "venvs"
+    PIPX_STANDALONE_PYTHON_CACHEDIR = PIPX_HOME / "py"
     PIPX_LOG_DIR = user_log_path("pipx")
     DEFAULT_PIPX_SHARED_LIBS = PIPX_HOME / "shared"
     PIPX_TRASH_DIR = PIPX_HOME / "trash"
@@ -32,6 +34,7 @@
 PIPX_SHARED_PTH = "pipx_shared.pth"
 LOCAL_BIN_DIR = Path(os.environ.get("PIPX_BIN_DIR", DEFAULT_PIPX_BIN_DIR)).resolve()
 LOCAL_MAN_DIR = Path(os.environ.get("PIPX_MAN_DIR", DEFAULT_PIPX_MAN_DIR)).resolve()
+FETCH_MISSING_PYTHON = os.environ.get("PIPX_FETCH_MISSING_PYTHON", False)
 TEMP_VENV_EXPIRATION_THRESHOLD_DAYS = 14
 MINIMUM_PYTHON_VERSION = "3.8"
 

diff --git a/src/pipx/interpreter.py b/src/pipx/interpreter.py
@@ -6,7 +6,8 @@
 from pathlib import Path
 from typing import Optional
 
-from pipx.constants import WINDOWS
+from pipx.constants import FETCH_MISSING_PYTHON, WINDOWS
+from pipx.standalone_python import download_python_build_standalone
 from pipx.util import PipxError
 
 logger = logging.getLogger(__name__)
@@ -42,10 +43,12 @@ def __init__(self, source: str, version: str, wrap_message: bool = True):
                 message += (
                     "The provided version looks like a version for Python Launcher, " "but `py` was not found on PATH."
                 )
+        if source == "the python-build-standalone project":
+            message += "listed in https://github.com/indygreg/python-build-standalone/releases/latest."
         super().__init__(message, wrap_message)
 
 
-def find_python_interpreter(python_version: str) -> str:
+def find_python_interpreter(python_version: str, fetch_missing_python: bool = False) -> str:
     if Path(python_version).is_file():
         return python_version
 
@@ -58,6 +61,14 @@ def find_python_interpreter(python_version: str) -> str:
 
     if shutil.which(python_version):
         return python_version
+
+    if fetch_missing_python or FETCH_MISSING_PYTHON:
+        try:
+            standalone_executable = download_python_build_standalone(python_version)
+            return standalone_executable
+        except PipxError as e:
+            raise InterpreterResolutionError(source="the python-build-standalone project", version=python_version) from e
+
     raise InterpreterResolutionError(source="PATH", version=python_version)
 
 

diff --git a/src/pipx/main.py b/src/pipx/main.py
@@ -23,9 +23,18 @@
 from pipx import commands, constants
 from pipx.animate import hide_cursor, show_cursor
 from pipx.colors import bold, green
-from pipx.constants import EXIT_CODE_SPECIFIED_PYTHON_EXECUTABLE_NOT_FOUND, MINIMUM_PYTHON_VERSION, WINDOWS, ExitCode
+from pipx.constants import (
+    EXIT_CODE_SPECIFIED_PYTHON_EXECUTABLE_NOT_FOUND,
+    MINIMUM_PYTHON_VERSION,
+    WINDOWS,
+    ExitCode,
+)
 from pipx.emojis import hazard
-from pipx.interpreter import DEFAULT_PYTHON, InterpreterResolutionError, find_python_interpreter
+from pipx.interpreter import (
+    DEFAULT_PYTHON,
+    InterpreterResolutionError,
+    find_python_interpreter,
+)
 from pipx.util import PipxError, mkdir, pipx_wrap, rmdir
 from pipx.venv import VenvContainer
 from pipx.version import version as __version__
@@ -188,8 +197,9 @@ def run_pipx_command(args: argparse.Namespace) -> ExitCode:  # noqa: C901
         skip_list = [canonicalize_name(x) for x in args.skip]
 
     if "python" in args and args.python is not None:
+        fetch_missing_python = args.fetch_missing_python
         try:
-            interpreter = find_python_interpreter(args.python)
+            interpreter = find_python_interpreter(args.python, fetch_missing_python=fetch_missing_python)
             args.python = interpreter
         except InterpreterResolutionError as e:
             print(
@@ -271,7 +281,11 @@ def run_pipx_command(args: argparse.Namespace) -> ExitCode:  # noqa: C901
         )
     elif args.command == "list":
         return commands.list_packages(
-            venv_container, args.include_injected, args.json, args.short, args.skip_maintenance
+            venv_container,
+            args.include_injected,
+            args.json,
+            args.short,
+            args.skip_maintenance,
         )
     elif args.command == "uninstall":
         return commands.uninstall(venv_dir, constants.LOCAL_BIN_DIR, constants.LOCAL_MAN_DIR, verbose)
@@ -336,6 +350,25 @@ def add_include_dependencies(parser: argparse.ArgumentParser) -> None:
     parser.add_argument("--include-deps", help="Include apps of dependent packages", action="store_true")
 
 
+def add_python_options(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        "--python",
+        default=DEFAULT_PYTHON,
+        help=(
+            "Python to install with. Possible values can be the executable name (python3.11), "
+            "the version to pass to py launcher (3.11), or the full path to the executable."
+            f"Requires Python {MINIMUM_PYTHON_VERSION} or above."
+        ),
+    )
+    parser.add_argument(
+        "--fetch-missing-python",
+        action="store_true",
+        help=(
+            "Whether to fetch a standalone python build from GitHub if the specified python version is not found locally on the system."
+        ),
+    )
+
+
 def _add_install(subparsers: argparse._SubParsersAction, shared_parser: argparse.ArgumentParser) -> None:
     p = subparsers.add_parser(
         "install",
@@ -360,15 +393,7 @@ def _add_install(subparsers: argparse._SubParsersAction, shared_parser: argparse
             "NOTE: The suffix feature is experimental and subject to change."
         ),
     )
-    p.add_argument(
-        "--python",
-        # Don't pass a default Python here so we know whether --python flag was passed
-        help=(
-            "Python to install with. Possible values can be the executable name (python3.11), "
-            "the version to pass to py launcher (3.11), or the full path to the executable."
-            f"Requires Python {MINIMUM_PYTHON_VERSION} or above."
-        ),
-    )
+    add_python_options(p)
     p.add_argument(
         "--preinstall",
         action="append",
@@ -519,15 +544,7 @@ def _add_reinstall(subparsers, venv_completer: VenvCompleter, shared_parser: arg
         parents=[shared_parser],
     )
     p.add_argument("package").completer = venv_completer
-    p.add_argument(
-        "--python",
-        default=DEFAULT_PYTHON,
-        help=(
-            "Python to reinstall with. Possible values can be the executable name (python3.11), "
-            "the version to pass to py launcher (3.11), or the full path to the executable."
-            f"Requires Python {MINIMUM_PYTHON_VERSION} or above."
-        ),
-    )
+    add_python_options(p)
 
 
 def _add_reinstall_all(subparsers: argparse._SubParsersAction, shared_parser: argparse.ArgumentParser) -> None:
@@ -548,15 +565,7 @@ def _add_reinstall_all(subparsers: argparse._SubParsersAction, shared_parser: ar
         ),
         parents=[shared_parser],
     )
-    p.add_argument(
-        "--python",
-        default=DEFAULT_PYTHON,
-        help=(
-            "Python to reinstall with. Possible values can be the executable name (python3.11), "
-            "the version to pass to py launcher (3.11), or the full path to the executable."
-            f"Requires Python {MINIMUM_PYTHON_VERSION} or above."
-        ),
-    )
+    add_python_options(p)
     p.add_argument("--skip", nargs="+", default=[], help="skip these packages")
 
 
@@ -622,15 +631,7 @@ def _add_run(subparsers: argparse._SubParsersAction, shared_parser: argparse.Arg
         help="Require app to be run from local __pypackages__ directory",
     )
     p.add_argument("--spec", help=SPEC_HELP)
-    p.add_argument(
-        "--python",
-        default=DEFAULT_PYTHON,
-        help=(
-            "Python to run with. Possible values can be the executable name (python3.11), "
-            "the version to pass to py launcher (3.11), or the full path to the executable. "
-            f"Requires Python {MINIMUM_PYTHON_VERSION} or above."
-        ),
-    )
+    add_python_options(p)
     add_pip_venv_args(p)
     p.set_defaults(subparser=p)
 
@@ -864,18 +865,23 @@ def setup(args: argparse.Namespace) -> None:
     mkdir(constants.LOCAL_BIN_DIR)
     mkdir(constants.LOCAL_MAN_DIR)
     mkdir(constants.PIPX_VENV_CACHEDIR)
-
-    cachedir_tag = constants.PIPX_VENV_CACHEDIR / "CACHEDIR.TAG"
-    if not cachedir_tag.exists():
-        logger.debug("Adding CACHEDIR.TAG to cache directory")
-        signature = (
-            "Signature: 8a477f597d28d172789f06886806bc55\n"
-            "# This file is a cache directory tag created by pipx.\n"
-            "# For information about cache directory tags, see:\n"
-            "#       https://bford.info/cachedir/\n"
-        )
-        with open(cachedir_tag, "w") as file:
-            file.write(signature)
+    mkdir(constants.PIPX_STANDALONE_PYTHON_CACHEDIR)
+
+    for cachedir in [
+        constants.PIPX_VENV_CACHEDIR,
+        constants.PIPX_STANDALONE_PYTHON_CACHEDIR,
+    ]:
+        cachedir_tag = cachedir / "CACHEDIR.TAG"
+        if not cachedir_tag.exists():
+            logger.debug("Adding CACHEDIR.TAG to cache directory")
+            signature = (
+                "Signature: 8a477f597d28d172789f06886806bc55\n"
+                "# This file is a cache directory tag created by pipx.\n"
+                "# For information about cache directory tags, see:\n"
+                "#       https://bford.info/cachedir/\n"
+            )
+            with open(cachedir_tag, "w") as file:
+                file.write(signature)
 
     rmdir(constants.PIPX_TRASH_DIR, False)