diff --git a/CHANGES.md b/CHANGES.md index 16a458096..91fae5d04 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,19 @@ # Release Notes +## 2.24.2 + +This release fixes a long-standing bug in "YOLO-mode" foreign platform +speculative wheel builds. Previously if the speculatively built wheel +had tags that did not match the foreign platform, the process errored +pre-emptively. This was correct for complete foreign platforms, where +all tag information is known, but not for all cases of abbreviated +platforms, where the failure was overly aggressive in some cases. Now +foreign abbreviated platform speculative builds are only rejected when +there is enough information to be sure the speculatively built wheel +definitely cannot work on the foreign abbreviated platform. + +* Accept more foreign `--platform` "YOLO-mode" wheels. (#2607) + ## 2.24.1 This release fixes `pex3 cache prune` handling of cached Pips. diff --git a/docker/user/create_docker_image_user.sh b/docker/user/create_docker_image_user.sh index e28bb91f2..dd4f7ccdc 100755 --- a/docker/user/create_docker_image_user.sh +++ b/docker/user/create_docker_image_user.sh @@ -3,13 +3,17 @@ set -xeuo pipefail if (( $# != 4 )); then - echo >2 "Usage $0 " + echo >&2 "Usage $0 " exit 1 fi uid=$2 gid=$4 +if id -un ubuntu; then + userdel -r ubuntu +fi + if ! id -g ${gid} >/dev/null; then group=$3 addgroup --gid=${gid} ${group} >&2 diff --git a/pex/environment.py b/pex/environment.py index 1b9317ab4..44cda7487 100644 --- a/pex/environment.py +++ b/pex/environment.py @@ -527,20 +527,18 @@ def _root_requirements_iter( ) unavailable_dists = self._unavailable_dists_by_project_name.get(project_name) if unavailable_dists: - message += ( - "\nFound {count} {distributions} for {project_name} that do not apply:\n" - "{unavailable_dists}".format( - count=len(unavailable_dists), - distributions=pluralize(unavailable_dists, "distribution"), - project_name=project_name, - unavailable_dists="\n".join( - "{index}.) {message}".format( - index=index, - message=unavailable_dist.render_message(self._target), - ) - for index, unavailable_dist in enumerate(unavailable_dists, start=1) - ), - ) + message += "\nFound {count} {distributions} for {project_name} that {does} not apply:\n" "{unavailable_dists}".format( + count=len(unavailable_dists), + distributions=pluralize(unavailable_dists, "distribution"), + project_name=project_name, + does="does" if len(unavailable_dists) == 1 else "do", + unavailable_dists="\n".join( + "{index}.) {message}".format( + index=index, + message=unavailable_dist.render_message(self._target), + ) + for index, unavailable_dist in enumerate(unavailable_dists, start=1) + ), ) raise ResolveError(message) candidates = [ diff --git a/pex/resolver.py b/pex/resolver.py index 9dacac413..c1f04513b 100644 --- a/pex/resolver.py +++ b/pex/resolver.py @@ -26,6 +26,7 @@ from pex.network_configuration import NetworkConfiguration from pex.orderedset import OrderedSet from pex.pep_376 import InstalledWheel +from pex.pep_425 import CompatibilityTags from pex.pep_427 import InstallableType, WheelError, install_wheel_chroot from pex.pep_503 import ProjectName from pex.pip.download_observer import DownloadObserver @@ -44,6 +45,7 @@ check_resolve, ) from pex.targets import AbbreviatedPlatform, CompletePlatform, LocalInterpreter, Target, Targets +from pex.third_party.packaging.tags import Tag from pex.tracer import TRACER from pex.typing import TYPE_CHECKING from pex.util import CacheHelper @@ -53,6 +55,7 @@ from typing import ( DefaultDict, Dict, + FrozenSet, Iterable, Iterator, List, @@ -368,7 +371,41 @@ def finalize_build(self, check_compatible=True): wheel_path = wheels[0] if check_compatible and self.request.target.is_foreign: wheel = Distribution.load(wheel_path) - if not self.request.target.wheel_applies(wheel): + wheel_tag_match = self.request.target.wheel_applies(wheel) + incompatible = isinstance(self.request.target, CompletePlatform) and not wheel_tag_match + if ( + not incompatible + and not wheel_tag_match + and isinstance(self.request.target, AbbreviatedPlatform) + ): + + def collect_platforms(tags): + # type: (Iterable[Tag]) -> Tuple[FrozenSet[str], bool] + platforms = [] # type: List[str] + is_linux = False + for tag in tags: + platforms.append(tag.platform) + if "linux" in tag.platform: + is_linux = True + return frozenset(platforms), is_linux + + wheel_platform_tags, is_linux_wheel = collect_platforms( + CompatibilityTags.from_wheel(wheel.location) + ) + abbreviated_target_platform_tags, is_linux_abbreviated_target = collect_platforms( + self.request.target.supported_tags + ) + + # N.B.: We can't say much about whether an abbreviated platform will match in the + # end unless the platform is a mismatch (i.e. linux vs mac). We check only for that + # sort of mismatch here. Further, we don't wade into manylinux compatibility and + # just consider a locally built linux wheel may match a linux target. + if not (is_linux_wheel and is_linux_abbreviated_target): + incompatible = ( + is_linux_wheel ^ is_linux_abbreviated_target + or not abbreviated_target_platform_tags.intersection(wheel_platform_tags) + ) + if incompatible: raise ValueError( "No pre-built wheel was available for {project_name} {version}.{eol}" "Successfully built the wheel {wheel} from the sdist {sdist} but it is not " diff --git a/pex/version.py b/pex/version.py index a2dc23914..dc4606e32 100644 --- a/pex/version.py +++ b/pex/version.py @@ -1,4 +1,4 @@ # Copyright 2015 Pex project contributors. # Licensed under the Apache License, Version 2.0 (see LICENSE). -__version__ = "2.24.1" +__version__ = "2.24.2" diff --git a/tests/integration/resolve/test_issue_2532.py b/tests/integration/resolve/test_issue_2532.py index fac7518f2..4966fe68f 100644 --- a/tests/integration/resolve/test_issue_2532.py +++ b/tests/integration/resolve/test_issue_2532.py @@ -45,7 +45,7 @@ def test_resolved_wheel_tag_platform_mismatch_warns( ENV PATH=/pex/venv/bin:$PATH RUN mkdir /work - WORKDIR = /work + WORKDIR /work """.format( pex_wheel=os.path.basename(pex_wheel) ) @@ -77,7 +77,8 @@ def test_resolved_wheel_tag_platform_mismatch_warns( stderr=subprocess.PIPE, ) stdout, stderr = process.communicate() - assert 0 == process.returncode + error = stderr.decode("utf-8") + assert 0 == process.returncode, error # N.B.: The tags calculated for manylinux_2_28_x86_64-cp-3.11.9-cp311 via `pip -v debug ...` # are: @@ -93,7 +94,6 @@ def test_resolved_wheel_tag_platform_mismatch_warns( # # Instead of failing the resolve check though, we should just see a warning since both of these # tags may be compatible at runtime, and, in fact, they are. - error = stderr.decode("utf-8") assert ( dedent( """\ diff --git a/tests/integration/test_issue_996.py b/tests/integration/test_issue_996.py index 4f9a72e56..011c81104 100644 --- a/tests/integration/test_issue_996.py +++ b/tests/integration/test_issue_996.py @@ -3,26 +3,19 @@ import multiprocessing import os +import subprocess -from pex.common import temporary_dir from pex.interpreter import PythonInterpreter from pex.typing import TYPE_CHECKING -from testing import ( - PY39, - PY310, - IntegResults, - ensure_python_interpreter, - make_env, - run_pex_command, - run_simple_pex, -) +from testing import PY39, PY310, IntegResults, ensure_python_interpreter, make_env, run_pex_command +from testing.pytest.tmp import Tempdir if TYPE_CHECKING: from typing import List -def test_resolve_local_platform(): - # type: () -> None +def test_resolve_local_platform(tmpdir): + # type: (Tempdir) -> None python39 = ensure_python_interpreter(PY39) python310 = ensure_python_interpreter(PY310) pex_python_path = os.pathsep.join((python39, python310)) @@ -35,27 +28,28 @@ def create_platform_pex(args): env=make_env(PEX_PYTHON_PATH=pex_python_path), ) - with temporary_dir() as td: - pex_file = os.path.join(td, "pex_file") - - # N.B.: We use psutil since only an sdist is available for linux and osx and the - # distribution has no dependencies. - args = ["psutil==5.7.0", "-o", pex_file] - - # By default, no --platforms are resolved and so distributions must be available in binary - # form. - results = create_platform_pex(args) - results.assert_failure() - - # If --platform resolution is enabled however, we should be able to find a corresponding - # local interpreter to perform a full-featured resolve with. - results = create_platform_pex(["--resolve-local-platforms"] + args) - results.assert_success() - - output, returncode = run_simple_pex( - pex=pex_file, - args=("-c", "import psutil; print(psutil.cpu_count())"), - interpreter=PythonInterpreter.from_binary(python310), - ) - assert 0 == returncode - assert int(output.strip()) >= multiprocessing.cpu_count() + pex_file = tmpdir.join("pex_file") + + # N.B.: We use psutil since only an sdist is available for linux and osx and the + # distribution has no dependencies. + build_args = ["psutil==5.7.0", "-o", pex_file] + check_args = [python310, pex_file, "-c", "import psutil; print(psutil.cpu_count())"] + check_env = make_env(PEX_PYTHON_PATH=python310, PEX_VERBOSE=1) + + # By default, no --platforms are resolved and so the yolo build process will produce a wheel + # using Python 3.9, which is not compatible with Python 3.10. + create_platform_pex(build_args).assert_success() + process = subprocess.Popen(check_args, env=check_env, stderr=subprocess.PIPE) + _, stderr = process.communicate() + error = stderr.decode("utf-8") + assert process.returncode != 0, error + assert ( + "Found 1 distribution for psutil that does not apply:\n" + " 1.) The wheel tags for psutil 5.7.0 are " + ) in error + + # If --platform resolution is enabled however, we should be able to find a corresponding + # local interpreter to perform a full-featured resolve with. + create_platform_pex(["--resolve-local-platforms"] + build_args).assert_success() + output = subprocess.check_output(check_args, env=check_env).decode("utf-8") + assert int(output.strip()) >= multiprocessing.cpu_count() diff --git a/tests/integration/test_pex_bootstrapper.py b/tests/integration/test_pex_bootstrapper.py index 9769cc17d..91966ac49 100644 --- a/tests/integration/test_pex_bootstrapper.py +++ b/tests/integration/test_pex_bootstrapper.py @@ -393,14 +393,14 @@ def test_boot_resolve_fail( r"No interpreter compatible with the requested constraints was found:\n" r"\n" r" A distribution for psutil could not be resolved for {py39_exe}.\n" - r" Found 1 distribution for psutil that do not apply:\n" + r" Found 1 distribution for psutil that does not apply:\n" r" 1\.\) The wheel tags for psutil 5\.9\.0 are .+ which do not match the supported tags " r"of {py39_exe}:\n" r" cp39-cp39-.+\n" r" ... \d+ more ...\n" r"\n" r" A distribution for psutil could not be resolved for {py310_exe}.\n" - r" Found 1 distribution for psutil that do not apply:\n" + r" Found 1 distribution for psutil that does not apply:\n" r" 1\.\) The wheel tags for psutil 5\.9\.0 are .+ which do not match the supported tags " r"of {py310_exe}:\n" r" cp310-cp310-.+\n"