From 4039584a75675e0b9e28a9c05e216e02c74f314c Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 17 Sep 2024 17:42:55 -0700 Subject: [PATCH] Add the ability to specify the `--pip-log` path. Although `--pip-log` is just an alias for the pre-existing `--preserve-pip-download-log` option, the option gains the ability to accept an optional log path value. In addition to this pro-active means of starting a debuggable Pex session with Pip, the log is also made more useful in the face of a multi-target resolve by serializing the log on targets and prefixing log lines per target. --- pex/cli/commands/lock.py | 4 +- pex/pip/tool.py | 21 +++-- pex/resolve/configured_resolve.py | 2 +- pex/resolve/lockfile/create.py | 2 +- pex/resolve/lockfile/updater.py | 4 +- pex/resolve/resolver_configuration.py | 2 +- pex/resolve/resolver_options.py | 41 +++++++--- pex/resolver.py | 110 +++++++++++++++++++++----- testing/__init__.py | 27 +++---- 9 files changed, 154 insertions(+), 59 deletions(-) diff --git a/pex/cli/commands/lock.py b/pex/cli/commands/lock.py index 377454e4a..d8224d295 100644 --- a/pex/cli/commands/lock.py +++ b/pex/cli/commands/lock.py @@ -656,7 +656,7 @@ def _add_update_arguments(cls, update_parser): resolver_options.register_network_options(resolver_options_parser) resolver_options.register_max_jobs_option(resolver_options_parser) resolver_options.register_use_pip_config(resolver_options_parser) - resolver_options.register_preserve_pip_download_log(resolver_options_parser) + resolver_options.register_pip_log(resolver_options_parser) @classmethod def add_update_lock_options( @@ -1087,7 +1087,7 @@ def _create_lock_update_request( max_jobs=resolver_options.get_max_jobs_value(self.options), use_pip_config=resolver_options.get_use_pip_config_value(self.options), dependency_configuration=dependency_config, - preserve_log=resolver_options.get_preserve_pip_download_log(self.options), + pip_log=resolver_options.get_pip_log(self.options), ) target_configuration = target_options.configure(self.options) diff --git a/pex/pip/tool.py b/pex/pip/tool.py index 5ead11bd6..dbe925cf7 100644 --- a/pex/pip/tool.py +++ b/pex/pip/tool.py @@ -11,7 +11,6 @@ import subprocess import sys from collections import deque -from tempfile import mkdtemp from pex import targets from pex.atomic_directory import atomic_directory @@ -339,6 +338,7 @@ def _spawn_pip_isolated( args, # type: Iterable[str] package_index_configuration=None, # type: Optional[PackageIndexConfiguration] interpreter=None, # type: Optional[PythonInterpreter] + log=None, # type: Optional[str] pip_verbosity=0, # type: int extra_env=None, # type: Optional[Dict[str, str]] **popen_kwargs # type: Any @@ -371,6 +371,10 @@ def _spawn_pip_isolated( # `~/.config/pip/pip.conf`. pip_args.append("--isolated") + if log: + pip_args.append("--log") + pip_args.append(log) + # The max pip verbosity is -vvv and for pex it's -vvvvvvvvv; so we scale down by a factor # of 3. pip_verbosity = pip_verbosity or (ENV.PEX_VERBOSE // 3) @@ -442,6 +446,7 @@ def _spawn_pip_isolated_job( args, # type: Iterable[str] package_index_configuration=None, # type: Optional[PackageIndexConfiguration] interpreter=None, # type: Optional[PythonInterpreter] + log=None, # type: Optional[str] pip_verbosity=0, # type: int finalizer=None, # type: Optional[Callable[[int], None]] extra_env=None, # type: Optional[Dict[str, str]] @@ -452,6 +457,7 @@ def _spawn_pip_isolated_job( args, package_index_configuration=package_index_configuration, interpreter=interpreter, + log=log, pip_verbosity=pip_verbosity, extra_env=extra_env, **popen_kwargs @@ -501,7 +507,7 @@ def spawn_download_distributions( build_configuration=BuildConfiguration(), # type: BuildConfiguration observer=None, # type: Optional[DownloadObserver] dependency_configuration=DependencyConfiguration(), # type: DependencyConfiguration - preserve_log=False, # type: bool + log=None, # type: Optional[str] ): # type: (...) -> Job target = target or targets.current() @@ -583,17 +589,14 @@ def spawn_download_distributions( popen_kwargs = {} finalizer = None - prefix = "pex-pip-log." - log = os.path.join( - mkdtemp(prefix=prefix) if preserve_log else safe_mkdtemp(prefix=prefix), "pip.log" - ) + preserve_log = log is not None if preserve_log: TRACER.log( "Preserving `pip download` log at {log_path}".format(log_path=log), V=ENV.PEX_VERBOSE, ) + log = log or os.path.join(safe_mkdtemp(prefix="pex-pip-log."), "pip.log") - download_cmd = ["--log", log] + download_cmd # N.B.: The `pip -q download ...` command is quiet but # `pip -q --log log.txt download ...` leaks download progress bars to stdout. We work # around this by sending stdout to the bit bucket. @@ -627,6 +630,7 @@ def finalizer(_): download_cmd, package_index_configuration=package_index_configuration, interpreter=target.get_interpreter(), + log=log, pip_verbosity=0, extra_env=extra_env, **popen_kwargs @@ -696,6 +700,7 @@ def spawn_debug( self, platform, # type: Platform manylinux=None, # type: Optional[str] + log=None, # type: Optional[str] ): # type: (...) -> Job @@ -710,5 +715,5 @@ def spawn_debug( debug_command = ["debug"] debug_command.extend(foreign_platform.iter_platform_args(platform, manylinux=manylinux)) return self._spawn_pip_isolated_job( - debug_command, pip_verbosity=1, stdout=subprocess.PIPE, stderr=subprocess.PIPE + debug_command, log=log, pip_verbosity=1, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) diff --git a/pex/resolve/configured_resolve.py b/pex/resolve/configured_resolve.py index 7a7218835..d3b5c11d9 100644 --- a/pex/resolve/configured_resolve.py +++ b/pex/resolve/configured_resolve.py @@ -125,7 +125,7 @@ def resolve( compile=compile_pyc, max_parallel_jobs=resolver_configuration.max_jobs, ignore_errors=ignore_errors, - preserve_log=resolver_configuration.preserve_log, + pip_log=resolver_configuration.log, pip_version=resolver_configuration.version, resolver=ConfiguredResolver(pip_configuration=resolver_configuration), use_pip_config=resolver_configuration.use_pip_config, diff --git a/pex/resolve/lockfile/create.py b/pex/resolve/lockfile/create.py index 1f1ba0dc2..38e7371a9 100644 --- a/pex/resolve/lockfile/create.py +++ b/pex/resolve/lockfile/create.py @@ -421,7 +421,7 @@ def create( max_parallel_jobs=pip_configuration.max_jobs, observer=lock_observer, dest=download_dir, - preserve_log=pip_configuration.preserve_log, + pip_log=pip_configuration.log, pip_version=pip_configuration.version, resolver=configured_resolver, use_pip_config=pip_configuration.use_pip_config, diff --git a/pex/resolve/lockfile/updater.py b/pex/resolve/lockfile/updater.py index 77a812398..7fbcc1675 100644 --- a/pex/resolve/lockfile/updater.py +++ b/pex/resolve/lockfile/updater.py @@ -619,7 +619,7 @@ def create( max_jobs, # type: int use_pip_config, # type: bool dependency_configuration, # type: DependencyConfiguration - preserve_log, # type: bool + pip_log, # type: Optional[str] ): # type: (...) -> LockUpdater @@ -638,7 +638,7 @@ def create( network_configuration=network_configuration, max_jobs=max_jobs, use_pip_config=use_pip_config, - preserve_log=preserve_log, + log=pip_log, ) return cls( lock_file=lock_file, diff --git a/pex/resolve/resolver_configuration.py b/pex/resolve/resolver_configuration.py index 66f7983d2..35354b0b7 100644 --- a/pex/resolve/resolver_configuration.py +++ b/pex/resolve/resolver_configuration.py @@ -185,7 +185,7 @@ class PipConfiguration(object): allow_prereleases = attr.ib(default=False) # type: bool transitive = attr.ib(default=True) # type: bool max_jobs = attr.ib(default=DEFAULT_MAX_JOBS) # type: int - preserve_log = attr.ib(default=False) # type: bool + log = attr.ib(default=None) # type: Optional[str] version = attr.ib(default=None) # type: Optional[PipVersionValue] resolver_version = attr.ib(default=None) # type: Optional[ResolverVersion.Value] allow_version_fallback = attr.ib(default=True) # type: bool diff --git a/pex/resolve/resolver_options.py b/pex/resolve/resolver_options.py index 967e43fd7..c523088a4 100644 --- a/pex/resolve/resolver_options.py +++ b/pex/resolve/resolver_options.py @@ -5,7 +5,8 @@ import glob import os -from argparse import Action, ArgumentTypeError, Namespace, _ActionsContainer +import tempfile +from argparse import Action, ArgumentError, ArgumentTypeError, Namespace, _ActionsContainer from pex import pex_warnings from pex.argparse import HandleBoolAction @@ -316,23 +317,45 @@ def valid_project_name(arg): help="Whether to transitively resolve requirements.", ) register_max_jobs_option(parser) - register_preserve_pip_download_log(parser) + register_pip_log(parser) -def register_preserve_pip_download_log(parser): +class HandlePipDownloadLogAction(Action): + def __init__(self, *args, **kwargs): + kwargs["nargs"] = "?" + super(HandlePipDownloadLogAction, self).__init__(*args, **kwargs) + + def __call__(self, parser, namespace, value, option_str=None): + if option_str.startswith("--no"): + if value: + raise ArgumentError( + self, + "Cannot specify a Pip log path and turn off Pip log preservation at the same " + "time. Given: `{option_str} {value}`".format( + option_str=option_str, value=value + ), + ) + elif not value: + value = os.path.join(tempfile.mkdtemp(prefix="pex-pip-log."), "pip.log") + setattr(namespace, self.dest, value) + + +def register_pip_log(parser): # type: (_ActionsContainer) -> None parser.add_argument( + "--pip-log", "--preserve-pip-download-log", "--no-preserve-pip-download-log", - default=PipConfiguration().preserve_log, - action=HandleBoolAction, + dest="pip_log", + default=PipConfiguration().log, + action=HandlePipDownloadLogAction, help="Preserve the `pip download` log and print its location to stderr.", ) -def get_preserve_pip_download_log(options): - # type: (Namespace) -> bool - return cast(bool, options.preserve_pip_download_log) +def get_pip_log(options): + # type: (Namespace) -> Optional[str] + return cast("Optional[str]", options.pip_log) def register_use_pip_config(parser): @@ -618,7 +641,7 @@ def create_pip_configuration(options): build_configuration=build_configuration, transitive=options.transitive, max_jobs=get_max_jobs_value(options), - preserve_log=get_preserve_pip_download_log(options), + log=get_pip_log(options), version=pip_version, resolver_version=resolver_version, allow_version_fallback=options.allow_pip_version_fallback, diff --git a/pex/resolver.py b/pex/resolver.py index 47e495bd2..6b4f46171 100644 --- a/pex/resolver.py +++ b/pex/resolver.py @@ -2,7 +2,7 @@ # Copyright 2014 Pex project contributors. # Licensed under the Apache License, Version 2.0 (see LICENSE). -from __future__ import absolute_import +from __future__ import absolute_import, print_function import functools import glob @@ -17,7 +17,7 @@ from pex.atomic_directory import AtomicDirectory, atomic_directory from pex.auth import PasswordEntry from pex.cache.dirs import CacheDir -from pex.common import pluralize, safe_mkdir, safe_mkdtemp +from pex.common import pluralize, safe_mkdir, safe_mkdtemp, safe_open from pex.compatibility import url_unquote, urlparse from pex.dependency_configuration import DependencyConfiguration from pex.dist_metadata import DistMetadata, Distribution, Requirement, is_wheel @@ -44,7 +44,7 @@ Untranslatable, check_resolve, ) -from pex.targets import LocalInterpreter, Target, Targets +from pex.targets import AbbreviatedPlatform, CompletePlatform, LocalInterpreter, Target, Targets from pex.tracer import TRACER from pex.typing import TYPE_CHECKING from pex.util import CacheHelper @@ -52,6 +52,7 @@ if TYPE_CHECKING: from typing import ( DefaultDict, + Dict, Iterable, Iterator, List, @@ -74,6 +75,68 @@ def _uniqued_targets(targets=None): return tuple(OrderedSet(targets)) if targets is not None else () +@attr.s(frozen=True) +class PipLogManager(object): + @classmethod + def create( + cls, + log, # type: Optional[str] + targets, # type: Sequence[Target] + ): + # type: (...) -> PipLogManager + log_by_target = {} # type: Dict[Target, str] + if log and len(targets) == 1: + log_by_target[targets[0]] = log + elif log: + log_dir = safe_mkdtemp(prefix="pex-pip-log.") + log_by_target.update( + (target, os.path.join(log_dir, "pip.{target}.log".format(target=target.id))) + for target in targets + ) + return cls(log=log, log_by_target=log_by_target) + + log = attr.ib() # type: Optional[str] + _log_by_target = attr.ib() # type: Mapping[Target, str] + + @staticmethod + def _target_id(target): + # type: (Target) -> str + if isinstance(target, LocalInterpreter): + # e.g.: CPython 2.7.18 + return target.interpreter.version_string + if isinstance(target, AbbreviatedPlatform): + return str(target.platform) + if isinstance(target, CompletePlatform): + return str(target.platform.tag) + return target.id + + def finalize_log(self): + # type: () -> None + target_count = len(self._log_by_target) + if target_count <= 1: + return + + with safe_open(self.log, "a") as out_fp: + for index, (target, log) in enumerate(self._log_by_target.items(), start=1): + prefix = "{index}/{count}]{target}".format( + index=index, count=target_count, target=self._target_id(target) + ) + if not os.path.exists(log): + print( + "{prefix}: WARNING: no Pip log was generated!".format(prefix=prefix), + file=out_fp, + ) + continue + + with open(log) as in_fp: + for line in in_fp: + out_fp.write("{prefix}: {line}".format(prefix=prefix, line=line)) + + def get_log(self, target): + # type: (Target) -> Optional[str] + return self._log_by_target.get(target) + + @attr.s(frozen=True) class DownloadRequest(object): targets = attr.ib(converter=_uniqued_targets) # type: Tuple[Target, ...] @@ -86,7 +149,7 @@ class DownloadRequest(object): package_index_configuration = attr.ib(default=None) # type: Optional[PackageIndexConfiguration] build_configuration = attr.ib(default=BuildConfiguration()) # type: BuildConfiguration observer = attr.ib(default=None) # type: Optional[ResolveObserver] - preserve_log = attr.ib(default=False) # type: bool + pip_log = attr.ib(default=None) # type: Optional[str] pip_version = attr.ib(default=None) # type: Optional[PipVersionValue] resolver = attr.ib(default=None) # type: Optional[Resolver] dependency_configuration = attr.ib( @@ -109,24 +172,29 @@ def download_distributions(self, dest=None, max_parallel_jobs=None): dest = dest or safe_mkdtemp( prefix="resolver_download.", dir=safe_mkdir(get_downloads_dir()) ) - spawn_download = functools.partial(self._spawn_download, dest) + log_manager = PipLogManager.create(self.pip_log, self.targets) + spawn_download = functools.partial(self._spawn_download, dest, log_manager) with TRACER.timed( "Resolving for:\n {}".format( "\n ".join(target.render_description() for target in self.targets) ) ): - return list( - execute_parallel( - inputs=self.targets, - spawn_func=spawn_download, - error_handler=Raise[Target, DownloadResult](Unsatisfiable), - max_jobs=max_parallel_jobs, + try: + return list( + execute_parallel( + inputs=self.targets, + spawn_func=spawn_download, + error_handler=Raise[Target, DownloadResult](Unsatisfiable), + max_jobs=max_parallel_jobs, + ) ) - ) + finally: + log_manager.finalize_log() def _spawn_download( self, resolved_dists_dir, # type: str + log_manager, # type: PipLogManager target, # type: Target ): # type: (...) -> SpawnedJob[DownloadResult] @@ -159,7 +227,7 @@ def _spawn_download( build_configuration=self.build_configuration, observer=observer, dependency_configuration=self.dependency_configuration, - preserve_log=self.preserve_log, + log=log_manager.get_log(target), ) return SpawnedJob.wait(job=download_job, result=download_result) @@ -978,7 +1046,7 @@ def resolve( max_parallel_jobs=None, # type: Optional[int] ignore_errors=False, # type: bool verify_wheels=True, # type: bool - preserve_log=False, # type: bool + pip_log=None, # type: Optional[str] pip_version=None, # type: Optional[PipVersionValue] resolver=None, # type: Optional[Resolver] use_pip_config=False, # type: bool @@ -1018,7 +1086,7 @@ def resolve( building and installing distributions in a resolve. Defaults to the number of CPUs available. :keyword ignore_errors: Whether to ignore resolution solver errors. Defaults to ``False``. :keyword verify_wheels: Whether to verify wheels have valid metadata. Defaults to ``True``. - :keyword preserve_log: Preserve the `pip download` log and print its location to stderr. + :keyword pip_log: Preserve the `pip download` log and print its location to stderr. Defaults to ``False``. :returns: The installed distributions meeting all requirements and constraints. :raises Unsatisfiable: If ``requirements`` is not transitively satisfiable. @@ -1098,7 +1166,7 @@ def resolve( package_index_configuration=package_index_configuration, build_configuration=build_configuration, max_parallel_jobs=max_parallel_jobs, - preserve_log=preserve_log, + pip_log=pip_log, pip_version=pip_version, resolver=resolver, dependency_configuration=dependency_configuration, @@ -1152,7 +1220,7 @@ def _download_internal( dest=None, # type: Optional[str] max_parallel_jobs=None, # type: Optional[int] observer=None, # type: Optional[ResolveObserver] - preserve_log=False, # type: bool + pip_log=None, # type: Optional[str] pip_version=None, # type: Optional[PipVersionValue] resolver=None, # type: Optional[Resolver] dependency_configuration=DependencyConfiguration(), # type: DependencyConfiguration @@ -1171,7 +1239,7 @@ def _download_internal( package_index_configuration=package_index_configuration, build_configuration=build_configuration, observer=observer, - preserve_log=preserve_log, + pip_log=pip_log, pip_version=pip_version, resolver=resolver, dependency_configuration=dependency_configuration, @@ -1231,7 +1299,7 @@ def download( dest=None, # type: Optional[str] max_parallel_jobs=None, # type: Optional[int] observer=None, # type: Optional[ResolveObserver] - preserve_log=False, # type: bool + pip_log=None, # type: Optional[str] pip_version=None, # type: Optional[PipVersionValue] resolver=None, # type: Optional[Resolver] use_pip_config=False, # type: bool @@ -1265,7 +1333,7 @@ def download( :keyword max_parallel_jobs: The maximum number of parallel jobs to use when resolving, building and installing distributions in a resolve. Defaults to the number of CPUs available. :keyword observer: An optional observer of the download internals. - :keyword preserve_log: Preserve the `pip download` log and print its location to stderr. + :keyword pip_log: Preserve the `pip download` log and print its location to stderr. Defaults to ``False``. :returns: The local distributions meeting all requirements and constraints. :raises Unsatisfiable: If the resolution of download of distributions fails for any reason. @@ -1296,7 +1364,7 @@ def download( dest=dest, max_parallel_jobs=max_parallel_jobs, observer=observer, - preserve_log=preserve_log, + pip_log=pip_log, pip_version=pip_version, resolver=resolver, dependency_configuration=dependency_configuration, diff --git a/testing/__init__.py b/testing/__init__.py index 319ec3779..25f987f28 100644 --- a/testing/__init__.py +++ b/testing/__init__.py @@ -490,16 +490,14 @@ def run_simple_pex_test( def bootstrap_python_installer(dest): # type: (str) -> None - for _ in range(3): + for index in range(3): try: - subprocess.check_call(["git", "clone", "https://github.com/pyenv/pyenv.git", dest]) + subprocess.check_call(args=["git", "clone", "https://github.com/pyenv/pyenv", dest]) + return except subprocess.CalledProcessError as e: - print("caught exception: %r" % e) + print("Error cloning pyenv on attempt", index + 1, "of 3:", e, file=sys.stderr) continue - else: - break - else: - raise RuntimeError("Helper method could not clone pyenv from git after 3 tries") + raise RuntimeError("Could not clone pyenv from git after 3 tries.") # NB: We keep the pool of bootstrapped interpreters as small as possible to avoid timeouts in CI @@ -508,7 +506,7 @@ def bootstrap_python_installer(dest): # minutes for a shard. # N.B.: Make sure to stick to versions that have binary releases for all supported platforms to # support use of pyenv-win which does not build from source, just running released installers -# robotically instead. +# instead. PY27 = "2.7.18" PY38 = "3.8.10" PY39 = "3.9.13" @@ -539,14 +537,15 @@ def ensure_python_distribution(version): pip = os.path.join(interpreter_location, "bin", "pip") + with atomic_directory(target_dir=pyenv_root) as pyenv_root_atomic_dir: + if not pyenv_root_atomic_dir.is_finalized(): + bootstrap_python_installer(pyenv_root_atomic_dir.work_dir) + with atomic_directory(target_dir=interpreter_location) as interpreter_target_dir: if not interpreter_target_dir.is_finalized(): - with atomic_directory(target_dir=pyenv_root) as pyenv_root_target_dir: - if pyenv_root_target_dir.is_finalized(): - with pyenv_root_target_dir.locked(): - subprocess.check_call(args=["git", "pull", "--ff-only"], cwd=pyenv_root) - else: - bootstrap_python_installer(pyenv_root_target_dir.work_dir) + with pyenv_root_atomic_dir.locked(): + subprocess.check_call(args=["git", "pull", "--ff-only"], cwd=pyenv_root) + env = pyenv_env.copy() if sys.platform.lower().startswith("linux"): env["CONFIGURE_OPTS"] = "--enable-shared"