Skip to content

Commit

Permalink
Surface Pip dependency conflict information.
Browse files Browse the repository at this point in the history 
Unfortunately, the useful information is sent to stdout which we suppress
since stdout is used to emit alot of other information that would be noise
for Pex users. Until pypa/pip#9420 is resolved
we work around this by plucking the information out of a --log file.

Fixes pex-tool#1159
  • Loading branch information
@jsirois
jsirois committed Jan 5, 2021
1 parent b2da656 commit 379f565
Show file tree
Hide file tree
Showing 2 changed files with 118 additions and 14 deletions.
96 changes: 82 additions & 14 deletions pex/pip.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,13 @@
from __future__ import absolute_import, print_function

import os
import re
import subprocess
from collections import deque
from textwrap import dedent

from pex import third_party
from pex.common import atomic_directory
from pex.common import atomic_directory, safe_mkdtemp
from pex.compatibility import urlparse
from pex.distribution_target import DistributionTarget
from pex.interpreter import PythonInterpreter
Expand Down Expand Up @@ -222,14 +224,23 @@ def __init__(self, pip_pex_path):
# type: (str) -> None
self._pip_pex_path = pip_pex_path # type: str

@staticmethod
def _calculate_resolver_version(package_index_configuration=None):
# type: (Optional[PackageIndexConfiguration]) -> ResolverVersion.Value
return (
package_index_configuration.resolver_version
if package_index_configuration
else ResolverVersion.PIP_LEGACY
)

def _spawn_pip_isolated(
self,
args, # type: Iterable[str]
package_index_configuration=None, # type: Optional[PackageIndexConfiguration]
cache=None, # type: Optional[str]
interpreter=None, # type: Optional[PythonInterpreter]
):
# type: (...) -> Job
# type: (...) -> Tuple[List[str], subprocess.Popen]
pip_args = [
# We vendor the version of pip we want so pip should never check for updates.
"--disable-pip-version-check",
Expand All @@ -244,12 +255,7 @@ def _spawn_pip_isolated(
"--exists-action",
"a",
]
resolver_version = (
package_index_configuration.resolver_version
if package_index_configuration
else ResolverVersion.PIP_LEGACY
)
pip_args.extend(resolver_version.pip_args)
pip_args.extend(self._calculate_resolver_version(package_index_configuration).pip_args)
if not package_index_configuration or package_index_configuration.isolated:
# Don't read PIP_ environment variables or pip configuration files like
# `~/.config/pip/pip.conf`.
Expand Down Expand Up @@ -291,9 +297,23 @@ def _spawn_pip_isolated(
from pex.pex import PEX

pip = PEX(pex=self._pip_pex_path, interpreter=interpreter)
return Job(
command=pip.cmdline(command), process=pip.run(args=command, env=env, blocking=False)
)
return pip.cmdline(command), pip.run(args=command, env=env, blocking=False)

def _spawn_pip_isolated_job(
self,
args, # type: Iterable[str]
package_index_configuration=None, # type: Optional[PackageIndexConfiguration]
cache=None, # type: Optional[str]
interpreter=None, # type: Optional[PythonInterpreter]
):
# type: (...) -> Job
command, process = self._spawn_pip_isolated(
args,
package_index_configuration=package_index_configuration,
cache=cache,
interpreter=interpreter,
)
return Job(command=command, process=process)

def spawn_download_distributions(
self,
Expand Down Expand Up @@ -363,12 +383,60 @@ def spawn_download_distributions(
if requirements:
download_cmd.extend(requirements)

return self._spawn_pip_isolated(
# The Pip 2020 resolver hides useful dependency conflict information in stdout interspersed
# with other information we want to suppress. We jump though some hoops here to get at that
# information and surface it on stderr. See: https://github.com/pypa/pip/issues/9420.
log = None
if (
self._calculate_resolver_version(package_index_configuration)
== ResolverVersion.PIP_2020
):
log = os.path.join(safe_mkdtemp(), "pip.log")
download_cmd = ["--log", log] + download_cmd

command, process = self._spawn_pip_isolated(
download_cmd,
package_index_configuration=package_index_configuration,
cache=cache,
interpreter=target.get_interpreter(),
)
return self._Issue9420Job(command, process, log) if log else Job(command, process)

class _Issue9420Job(Job):
def __init__(self, command, process, log):
self._log = log
super(Pip._Issue9420Job, self).__init__(command, process)

def _check_returncode(self, stderr=None):
# N.B.: Pip --log output looks like:
# 2021-01-04T16:12:01,119 ERROR: Cannot install pantsbuild-pants==1.24.0.dev2 and wheel==0.33.6 because these package versions have conflicting dependencies.
# 2021-01-04T16:12:01,119
# 2021-01-04T16:12:01,119 The conflict is caused by:
# 2021-01-04T16:12:01,119 The user requested wheel==0.33.6
# 2021-01-04T16:12:01,119 pantsbuild-pants 1.24.0.dev2 depends on wheel==0.31.1
# 2021-01-04T16:12:01,119
# 2021-01-04T16:12:01,119 To fix this you could try to:
# 2021-01-04T16:12:01,119 1. loosen the range of package versions you've specified
# 2021-01-04T16:12:01,119 2. remove package versions to allow pip attempt to solve the dependency conflict
# 2021-01-04T16:12:01,119 ERROR: ResolutionImpossible: for help visit https://pip.pypa.io/en/latest/user_guide/#fixing-conflicting-dependencies
if self._process.returncode != 0:
strip = None
collected = []
with open(self._log, "r") as fp:
for line in fp:
if not strip:
match = re.match(r"^(?P<timestamp>[^ ]+) ERROR: Cannot install ", line)
if match:
strip = len(match.group("timestamp"))
else:
match = re.match(r"^[^ ]+ ERROR: ResolutionImpossible: ", line)
if match:
break
else:
collected.append(line[strip:].encode("utf-8"))
os.unlink(self._log)
stderr = (stderr or b"") + b"".join(collected)
super(Pip._Issue9420Job, self)._check_returncode(stderr=stderr)

def spawn_build_wheels(
self,
Expand All @@ -382,7 +450,7 @@ def spawn_build_wheels(
wheel_cmd = ["wheel", "--no-deps", "--wheel-dir", wheel_dir]
wheel_cmd.extend(distributions)

return self._spawn_pip_isolated(
return self._spawn_pip_isolated_job(
wheel_cmd,
# If the build leverages PEP-518 it will need to resolve build requirements.
package_index_configuration=package_index_configuration,
Expand Down Expand Up @@ -451,7 +519,7 @@ def spawn_install_wheel(

install_cmd.append("--compile" if compile else "--no-compile")
install_cmd.append(wheel)
return self._spawn_pip_isolated(install_cmd, cache=cache, interpreter=interpreter)
return self._spawn_pip_isolated_job(install_cmd, cache=cache, interpreter=interpreter)


_PIP = None
Expand Down
36 changes: 36 additions & 0 deletions tests/test_integration.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2716,3 +2716,39 @@ def test_seed(
pex_stdout, pex_stderr = Executor.execute([pex_file] + isort_args)
assert pex_stdout == seed_stdout
assert pex_stderr == seed_stderr


def test_pip_issues_9420_workaround():
# type: () -> None

# N.B.: isort 5.7.0 needs Python >=3.6
python = ensure_python_interpreter(PY36)

results = run_pex_command(
args=["--resolver-version", "pip-2020-resolver", "isort[colors]==5.7.0", "colorama==0.4.1"],
python=python,
quiet=True,
)
results.assert_failure()
normalized_stderr = "\n".join(line.strip() for line in results.error.strip().splitlines())
assert normalized_stderr.startswith(
dedent(
"""\
ERROR: Cannot install colorama==0.4.1 and isort[colors]==5.7.0 because these package versions have conflicting dependencies.
ERROR: ResolutionImpossible: for help visit https://pip.pypa.io/en/latest/user_guide/#fixing-conflicting-dependencies
"""
)
)
assert normalized_stderr.endswith(
dedent(
"""\
The conflict is caused by:
The user requested colorama==0.4.1
isort[colors] 5.7.0 depends on colorama<0.5.0 and >=0.4.3; extra == "colors"
To fix this you could try to:
1. loosen the range of package versions you've specified
2. remove package versions to allow pip attempt to solve the dependency conflict
"""
).strip()
)

0 comments on commit 379f565

Please sign in to comment.