Add the ability to specify the --pip-log path. (#2536)

Although `--pip-log` is just an alias for the pre-existing
`--preserve-pip-download-log` option, the option gains the ability to
accept an optional log path value. In addition to this pro-active means
of starting a debuggable Pex session with Pip, the log is also made more
useful in the face of a multi-target resolve by serializing the log on
targets and prefixing log lines per target. The log now also includes
the compatibility tags calculated via `pip -v debug ...` for any 
abbreviated `--platform`s used in the resolve.

pex/cli/commands/lock.py

@@ -656,7 +656,7 @@ def _add_update_arguments(cls, update_parser):
         resolver_options.register_network_options(resolver_options_parser)
         resolver_options.register_max_jobs_option(resolver_options_parser)
         resolver_options.register_use_pip_config(resolver_options_parser)
-        resolver_options.register_preserve_pip_download_log(resolver_options_parser)
+        resolver_options.register_pip_log(resolver_options_parser)
 
     @classmethod
     def add_update_lock_options(
@@ -1087,7 +1087,7 @@ def _create_lock_update_request(
             max_jobs=resolver_options.get_max_jobs_value(self.options),
             use_pip_config=resolver_options.get_use_pip_config_value(self.options),
             dependency_configuration=dependency_config,
-            preserve_log=resolver_options.get_preserve_pip_download_log(self.options),
+            pip_log=resolver_options.get_pip_log(self.options),
         )
 
         target_configuration = target_options.configure(self.options)

pex/pip/tool.py

@@ -11,7 +11,6 @@
 import subprocess
 import sys
 from collections import deque
-from tempfile import mkdtemp
 
 from pex import targets
 from pex.atomic_directory import atomic_directory
@@ -339,6 +338,7 @@ def _spawn_pip_isolated(
         args,  # type: Iterable[str]
         package_index_configuration=None,  # type: Optional[PackageIndexConfiguration]
         interpreter=None,  # type: Optional[PythonInterpreter]
+        log=None,  # type: Optional[str]
         pip_verbosity=0,  # type: int
         extra_env=None,  # type: Optional[Dict[str, str]]
         **popen_kwargs  # type: Any
@@ -371,6 +371,10 @@ def _spawn_pip_isolated(
             # `~/.config/pip/pip.conf`.
             pip_args.append("--isolated")
 
+        if log:
+            pip_args.append("--log")
+            pip_args.append(log)
+
         # The max pip verbosity is -vvv and for pex it's -vvvvvvvvv; so we scale down by a factor
         # of 3.
         pip_verbosity = pip_verbosity or (ENV.PEX_VERBOSE // 3)
@@ -442,6 +446,7 @@ def _spawn_pip_isolated_job(
         args,  # type: Iterable[str]
         package_index_configuration=None,  # type: Optional[PackageIndexConfiguration]
         interpreter=None,  # type: Optional[PythonInterpreter]
+        log=None,  # type: Optional[str]
         pip_verbosity=0,  # type: int
         finalizer=None,  # type: Optional[Callable[[int], None]]
         extra_env=None,  # type: Optional[Dict[str, str]]
@@ -452,6 +457,7 @@ def _spawn_pip_isolated_job(
             args,
             package_index_configuration=package_index_configuration,
             interpreter=interpreter,
+            log=log,
             pip_verbosity=pip_verbosity,
             extra_env=extra_env,
             **popen_kwargs
@@ -501,7 +507,7 @@ def spawn_download_distributions(
         build_configuration=BuildConfiguration(),  # type: BuildConfiguration
         observer=None,  # type: Optional[DownloadObserver]
         dependency_configuration=DependencyConfiguration(),  # type: DependencyConfiguration
-        preserve_log=False,  # type: bool
+        log=None,  # type: Optional[str]
     ):
         # type: (...) -> Job
         target = target or targets.current()
@@ -583,17 +589,14 @@ def spawn_download_distributions(
         popen_kwargs = {}
         finalizer = None
 
-        prefix = "pex-pip-log."
-        log = os.path.join(
-            mkdtemp(prefix=prefix) if preserve_log else safe_mkdtemp(prefix=prefix), "pip.log"
-        )
+        preserve_log = log is not None
         if preserve_log:
             TRACER.log(
                 "Preserving `pip download` log at {log_path}".format(log_path=log),
                 V=ENV.PEX_VERBOSE,
             )
+        log = log or os.path.join(safe_mkdtemp(prefix="pex-pip-log."), "pip.log")
 
-        download_cmd = ["--log", log] + download_cmd
         # N.B.: The `pip -q download ...` command is quiet but
         # `pip -q --log log.txt download ...` leaks download progress bars to stdout. We work
         # around this by sending stdout to the bit bucket.
@@ -627,6 +630,7 @@ def finalizer(_):
             download_cmd,
             package_index_configuration=package_index_configuration,
             interpreter=target.get_interpreter(),
+            log=log,
             pip_verbosity=0,
             extra_env=extra_env,
             **popen_kwargs
@@ -696,6 +700,7 @@ def spawn_debug(
         self,
         platform,  # type: Platform
         manylinux=None,  # type: Optional[str]
+        log=None,  # type: Optional[str]
     ):
         # type: (...) -> Job
 
@@ -710,5 +715,5 @@ def spawn_debug(
         debug_command = ["debug"]
         debug_command.extend(foreign_platform.iter_platform_args(platform, manylinux=manylinux))
         return self._spawn_pip_isolated_job(
-            debug_command, pip_verbosity=1, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+            debug_command, log=log, pip_verbosity=1, stdout=subprocess.PIPE, stderr=subprocess.PIPE
         )

pex/resolve/configured_resolve.py

@@ -125,7 +125,7 @@ def resolve(
                 compile=compile_pyc,
                 max_parallel_jobs=resolver_configuration.max_jobs,
                 ignore_errors=ignore_errors,
-                preserve_log=resolver_configuration.preserve_log,
+                pip_log=resolver_configuration.log,
                 pip_version=resolver_configuration.version,
                 resolver=ConfiguredResolver(pip_configuration=resolver_configuration),
                 use_pip_config=resolver_configuration.use_pip_config,

pex/resolve/lockfile/create.py

@@ -421,7 +421,7 @@ def create(
             max_parallel_jobs=pip_configuration.max_jobs,
             observer=lock_observer,
             dest=download_dir,
-            preserve_log=pip_configuration.preserve_log,
+            pip_log=pip_configuration.log,
             pip_version=pip_configuration.version,
             resolver=configured_resolver,
             use_pip_config=pip_configuration.use_pip_config,

pex/resolve/lockfile/updater.py

@@ -619,7 +619,7 @@ def create(
         max_jobs,  # type: int
         use_pip_config,  # type: bool
         dependency_configuration,  # type: DependencyConfiguration
-        preserve_log,  # type: bool
+        pip_log,  # type: Optional[str]
     ):
         # type: (...) -> LockUpdater
 
@@ -638,7 +638,7 @@ def create(
             network_configuration=network_configuration,
             max_jobs=max_jobs,
             use_pip_config=use_pip_config,
-            preserve_log=preserve_log,
+            log=pip_log,
         )
         return cls(
             lock_file=lock_file,

pex/resolve/resolver_configuration.py

@@ -185,7 +185,7 @@ class PipConfiguration(object):
     allow_prereleases = attr.ib(default=False)  # type: bool
     transitive = attr.ib(default=True)  # type: bool
     max_jobs = attr.ib(default=DEFAULT_MAX_JOBS)  # type: int
-    preserve_log = attr.ib(default=False)  # type: bool
+    log = attr.ib(default=None)  # type: Optional[str]
     version = attr.ib(default=None)  # type: Optional[PipVersionValue]
     resolver_version = attr.ib(default=None)  # type: Optional[ResolverVersion.Value]
     allow_version_fallback = attr.ib(default=True)  # type: bool

pex/resolve/resolver_options.py

@@ -5,7 +5,8 @@
 
 import glob
 import os
-from argparse import Action, ArgumentTypeError, Namespace, _ActionsContainer
+import tempfile
+from argparse import Action, ArgumentError, ArgumentTypeError, Namespace, _ActionsContainer
 
 from pex import pex_warnings
 from pex.argparse import HandleBoolAction
@@ -316,23 +317,45 @@ def valid_project_name(arg):
         help="Whether to transitively resolve requirements.",
     )
     register_max_jobs_option(parser)
-    register_preserve_pip_download_log(parser)
+    register_pip_log(parser)
 
 
-def register_preserve_pip_download_log(parser):
+class HandlePipDownloadLogAction(Action):
+    def __init__(self, *args, **kwargs):
+        kwargs["nargs"] = "?"
+        super(HandlePipDownloadLogAction, self).__init__(*args, **kwargs)
+
+    def __call__(self, parser, namespace, value, option_str=None):
+        if option_str.startswith("--no"):
+            if value:
+                raise ArgumentError(
+                    self,
+                    "Cannot specify a Pip log path and turn off Pip log preservation at the same "
+                    "time. Given: `{option_str} {value}`".format(
+                        option_str=option_str, value=value
+                    ),
+                )
+        elif not value:
+            value = os.path.join(tempfile.mkdtemp(prefix="pex-pip-log."), "pip.log")
+        setattr(namespace, self.dest, value)
+
+
+def register_pip_log(parser):
     # type: (_ActionsContainer) -> None
     parser.add_argument(
+        "--pip-log",
         "--preserve-pip-download-log",
         "--no-preserve-pip-download-log",
-        default=PipConfiguration().preserve_log,
-        action=HandleBoolAction,
+        dest="pip_log",
+        default=PipConfiguration().log,
+        action=HandlePipDownloadLogAction,
         help="Preserve the `pip download` log and print its location to stderr.",
     )
 
 
-def get_preserve_pip_download_log(options):
-    # type: (Namespace) -> bool
-    return cast(bool, options.preserve_pip_download_log)
+def get_pip_log(options):
+    # type: (Namespace) -> Optional[str]
+    return cast("Optional[str]", options.pip_log)
 
 
 def register_use_pip_config(parser):
@@ -618,7 +641,7 @@ def create_pip_configuration(options):
         build_configuration=build_configuration,
         transitive=options.transitive,
         max_jobs=get_max_jobs_value(options),
-        preserve_log=get_preserve_pip_download_log(options),
+        log=get_pip_log(options),
         version=pip_version,
         resolver_version=resolver_version,
         allow_version_fallback=options.allow_pip_version_fallback,