From 0dbac1edc4458958c0107fe72348912a9eeb8960 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Thu, 2 Mar 2023 12:29:36 -0800 Subject: [PATCH] Attempt "cross-builds" of sdists for foreign platforms. (#2075) The "cross-building" is in scare-quotes because this is not actually cross-building, it is just attempting a build of the sdist, and, if successful, seeing if the resulting wheel matches the foreign platform. This enables sdist-only projects to be used in foreign platform Pex operations when it turns out the sdist is multi-platform. Closes #2073 --- pex/pip/download_observer.py | 83 ++++++++- .../__init__.py} | 93 ++++++---- pex/pip/foreign_platform/markers.py | 19 ++ pex/pip/foreign_platform/requires_python.py | 112 ++++++++++++ pex/pip/foreign_platform/tags.py | 26 +++ pex/pip/foreign_platform_patches.py | 47 ----- pex/pip/installation.py | 8 +- pex/pip/tool.py | 47 +++-- pex/resolve/downloads.py | 2 +- pex/resolve/locked_resolve.py | 20 +-- pex/resolve/locker.py | 99 +++++------ pex/resolve/locker_patches.py | 163 +++--------------- pex/resolve/lockfile/create.py | 4 +- pex/resolver.py | 25 ++- pex/venv/pex.py | 2 +- tests/integration/cli/commands/test_export.py | 7 +- tests/integration/cli/commands/test_lock.py | 155 ++++++++++------- tests/integration/test_issue_2073.py | 115 ++++++++++++ tests/resolve/test_locked_resolve.py | 7 - tests/test_pip.py | 76 ++++---- 20 files changed, 660 insertions(+), 450 deletions(-) rename pex/pip/{foreign_platform.py => foreign_platform/__init__.py} (60%) create mode 100644 pex/pip/foreign_platform/markers.py create mode 100644 pex/pip/foreign_platform/requires_python.py create mode 100644 pex/pip/foreign_platform/tags.py delete mode 100644 pex/pip/foreign_platform_patches.py create mode 100644 tests/integration/test_issue_2073.py diff --git a/pex/pip/download_observer.py b/pex/pip/download_observer.py index 67bbaebcb..927e73ffc 100644 --- a/pex/pip/download_observer.py +++ b/pex/pip/download_observer.py @@ -1,13 +1,17 @@ # Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -from __future__ import absolute_import +from __future__ import absolute_import, print_function +import os +import pkgutil + +from pex.common import safe_mkdtemp from pex.pip.log_analyzer import LogAnalyzer from pex.typing import TYPE_CHECKING if TYPE_CHECKING: - from typing import Iterable, Mapping, Optional, Text + from typing import Dict, Mapping, Optional, Text, Tuple import attr # vendor:skip else: @@ -16,12 +20,81 @@ @attr.s(frozen=True) class Patch(object): - code = attr.ib(default=None) # type: Optional[Text] - args = attr.ib(default=()) # type: Iterable[str] + @classmethod + def from_code_resource( + cls, + package, # type: str + resource, # type: str + **env # type: str + ): + # type: (...) -> Patch + module, ext = os.path.splitext(resource) + if ext != ".py": + raise ValueError( + "Code resources must be `.py` files, asked to load: {resource}".format( + resource=resource + ) + ) + code = pkgutil.get_data(package, resource) + assert code is not None, ( + "The resource {resource} relative to {package} should always be present in a " + "Pex distribution or source tree.".format(resource=resource, package=package) + ) + return cls(module=module, code=code.decode("utf-8"), env=env) + + module = attr.ib() # type: str + code = attr.ib() # type: Text env = attr.ib(factory=dict) # type: Mapping[str, str] +@attr.s(frozen=True) +class PatchSet(object): + @classmethod + def create(cls, *patches): + # type: (*Patch) -> PatchSet + return cls(patches=patches) + + patches = attr.ib(default=()) # type: Tuple[Patch, ...] + + @property + def env(self): + # type: () -> Dict[str, str] + env = {} # type: Dict[str, str] + for patch in self.patches: + env.update(patch.env) + return env + + def emit_patches(self, package): + # type: (str) -> Optional[str] + + if not self.patches: + return None + + if not package or "." in package: + raise ValueError( + "The `package` argument must be a non-empty, non-nested package name. " + "Given: {package!r}".format(package=package) + ) + + patches_dir = safe_mkdtemp() + patches_package = os.path.join(patches_dir, package) + os.mkdir(patches_package) + + for patch in self.patches: + python_file = "{module}.py".format(module=patch.module) + with open(os.path.join(patches_package, python_file), "wb") as code_fp: + code_fp.write(patch.code.encode("utf-8")) + + with open(os.path.join(patches_package, "__init__.py"), "w") as fp: + print("from __future__ import absolute_import", file=fp) + for patch in self.patches: + print("from . import {module}".format(module=patch.module), file=fp) + print("{module}.patch()".format(module=patch.module), file=fp) + + return patches_dir + + @attr.s(frozen=True) class DownloadObserver(object): analyzer = attr.ib() # type: Optional[LogAnalyzer] - patch = attr.ib() # type: Patch + patch_set = attr.ib() # type: PatchSet diff --git a/pex/pip/foreign_platform.py b/pex/pip/foreign_platform/__init__.py similarity index 60% rename from pex/pip/foreign_platform.py rename to pex/pip/foreign_platform/__init__.py index 08760bad1..caf532c41 100644 --- a/pex/pip/foreign_platform.py +++ b/pex/pip/foreign_platform/__init__.py @@ -5,12 +5,12 @@ import json import os -import pkgutil import re from pex.common import safe_mkdtemp +from pex.interpreter_constraints import iter_compatible_versions from pex.pep_425 import CompatibilityTags -from pex.pip.download_observer import DownloadObserver, Patch +from pex.pip.download_observer import DownloadObserver, Patch, PatchSet from pex.pip.log_analyzer import ErrorAnalyzer, ErrorMessage from pex.platforms import Platform from pex.targets import AbbreviatedPlatform, CompletePlatform, Target @@ -18,7 +18,7 @@ from pex.typing import TYPE_CHECKING if TYPE_CHECKING: - from typing import Iterator, Optional, Text, Tuple + from typing import Iterable, Iterator, Optional import attr # vendor:skip @@ -91,54 +91,83 @@ def analyze(self, line): return self.Continue() -_CODE = None # type: Optional[Text] - - -def _code(): - # type: () -> Text - global _CODE - if _CODE is None: - code = pkgutil.get_data(__name__, "foreign_platform_patches.py") - assert code is not None, ( - "The sibling resource foreign_platform_patches.py of {} should always be present in a " - "Pex distribution or source tree.".format(__name__) - ) - _CODE = code.decode("utf-8") - return _CODE - - def patch(target): # type: (Target) -> Optional[DownloadObserver] if not isinstance(target, (AbbreviatedPlatform, CompletePlatform)): return None analyzer = _Issue10050Analyzer(target.platform) - args = () # type: Tuple[str, ...] + patches = [] patches_dir = safe_mkdtemp() + patched_environment = target.marker_environment.as_dict() with open(os.path.join(patches_dir, "markers.json"), "w") as markers_fp: json.dump(patched_environment, markers_fp) - env = dict(_PEX_PATCHED_MARKERS_FILE=markers_fp.name) - - if isinstance(target, AbbreviatedPlatform): - args = tuple(iter_platform_args(target.platform, target.manylinux)) + patches.append( + Patch.from_code_resource(__name__, "markers.py", _PEX_PATCHED_MARKERS_FILE=markers_fp.name) + ) compatible_tags = target.supported_tags if compatible_tags: - env.update(patch_tags(compatible_tags).env) + patches.append(patch_tags(compatible_tags=compatible_tags, patches_dir=patches_dir)) + + assert ( + target.marker_environment.python_full_version or target.marker_environment.python_version + ), ( + "A complete platform should always have both `python_full_version` and `python_version` " + "environment markers defined and an abbreviated platform should always have at least the" + "`python_version` environment marker defined. Given: {target}".format(target=target) + ) + requires_python = ( + "=={full_version}".format(full_version=target.marker_environment.python_full_version) + if target.marker_environment.python_full_version + else "=={version}.*".format(version=target.marker_environment.python_version) + ) + patches.append( + patch_requires_python(requires_python=[requires_python], patches_dir=patches_dir) + ) TRACER.log( "Patching environment markers for {} with {}".format(target, patched_environment), V=3, ) - return DownloadObserver(analyzer=analyzer, patch=Patch(code=_code(), args=args, env=env)) + return DownloadObserver(analyzer=analyzer, patch_set=PatchSet(patches=tuple(patches))) -def patch_tags(compatible_tags): - # type: (CompatibilityTags) -> Patch - patches_dir = safe_mkdtemp() - with open(os.path.join(patches_dir, "tags.json"), "w") as tags_fp: +def patch_tags( + compatible_tags, # type: CompatibilityTags + patches_dir=None, # type: Optional[str] +): + # type: (...) -> Patch + with open(os.path.join(patches_dir or safe_mkdtemp(), "tags.json"), "w") as tags_fp: json.dump(compatible_tags.to_string_list(), tags_fp) - env = dict(_PEX_PATCHED_TAGS_FILE=tags_fp.name) - return Patch(env=env, code=_code()) + return Patch.from_code_resource(__name__, "tags.py", _PEX_PATCHED_TAGS_FILE=tags_fp.name) + + +def patch_requires_python( + requires_python, # type: Iterable[str] + patches_dir=None, # type: Optional[str] +): + # type: (...) -> Patch + """N.B.: This Path exports Python version information in the `requires_python` module. + + Exports: + + PYTHON_FULL_VERSIONS: List[Tuple[int, int, int]] + A sorted list of Python full versions compatible with the given `requires_python`. + + PYTHON_VERSIONS: List[Tuple[int, int]] + A sorted list of Python versions compatible with the given `requires_python`. + """ + with TRACER.timed( + "Calculating compatible python versions for {requires_python}".format( + requires_python=requires_python + ) + ): + python_full_versions = list(iter_compatible_versions(requires_python)) + with open( + os.path.join(patches_dir or safe_mkdtemp(), "python_full_versions.json"), "w" + ) as fp: + json.dump(python_full_versions, fp) + return Patch.from_code_resource( + __name__, "requires_python.py", _PEX_PYTHON_VERSIONS_FILE=fp.name + ) diff --git a/pex/pip/foreign_platform/markers.py b/pex/pip/foreign_platform/markers.py new file mode 100644 index 000000000..8a978d777 --- /dev/null +++ b/pex/pip/foreign_platform/markers.py @@ -0,0 +1,19 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import json +import os + + +def patch(): + from pip._vendor.packaging import markers # type: ignore[import] + + # N.B.: The following environment variable is used by the Pex runtime to control Pip and must be + # kept in-sync with `__init__.py`. + patched_markers_file = os.environ.pop("_PEX_PATCHED_MARKERS_FILE") + with open(patched_markers_file) as fp: + patched_markers = json.load(fp) + + markers.default_environment = patched_markers.copy diff --git a/pex/pip/foreign_platform/requires_python.py b/pex/pip/foreign_platform/requires_python.py new file mode 100644 index 000000000..df6fb2319 --- /dev/null +++ b/pex/pip/foreign_platform/requires_python.py @@ -0,0 +1,112 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import json +import os + +# N.B.: The following environment variable is used by the Pex runtime to control Pip and must be +# kept in-sync with `__init__.py`. +with open(os.environ.pop("_PEX_PYTHON_VERSIONS_FILE")) as fp: + PYTHON_FULL_VERSIONS = json.load(fp) +PYTHON_VERSIONS = sorted(set((version[0], version[1]) for version in PYTHON_FULL_VERSIONS)) + + +def patch(): + # The pip-legacy-resolver patch. + from pip._internal.utils import packaging # type: ignore[import] + + if PYTHON_FULL_VERSIONS: + orig_check_requires_python = packaging.check_requires_python + + def check_requires_python(requires_python, *_args, **_kw): + # Ensure any dependency we lock is compatible with the full interpreter range + # specified since we have no way to force Pip to backtrack and follow paths for any + # divergences. Most (all?) true divergences should be covered by forked environment + # markers. + return all( + orig_check_requires_python(requires_python, python_full_version) + for python_full_version in PYTHON_FULL_VERSIONS + ) + + packaging.check_requires_python = check_requires_python + else: + packaging.check_requires_python = lambda *_args, **_kw: True + + # The pip-2020-resolver patch. + from pip._internal.resolution.resolvelib.candidates import ( # type: ignore[import] + RequiresPythonCandidate, + ) + from pip._internal.resolution.resolvelib.requirements import ( # type: ignore[import] + RequiresPythonRequirement, + ) + + if PYTHON_FULL_VERSIONS: + orig_get_candidate_lookup = RequiresPythonRequirement.get_candidate_lookup + orig_is_satisfied_by = RequiresPythonRequirement.is_satisfied_by + + # Ensure we do a proper, but minimal, comparison for Python versions. Previously we + # always tested all `Requires-Python` specifier sets against Python full versions. + # That can be pathologically slow (see: + # https://github.com/pantsbuild/pants/issues/14998); so we avoid using Python full + # versions unless the `Requires-Python` specifier set requires that data. In other + # words: + # + # Need full versions to evaluate properly: + # + Requires-Python: >=3.7.6 + # + Requires-Python: >=3.7,!=3.7.6,<4 + # + # Do not need full versions to evaluate properly: + # + Requires-Python: >=3.7,<4 + # + Requires-Python: ==3.7.* + # + Requires-Python: >=3.6.0 + # + def needs_full_versions(spec): + components = spec.version.split(".", 2) + if len(components) < 3: + return False + major_, minor_, patch = components + if spec.operator in ("<", "<=", ">", ">=") and patch == "0": + return False + return patch != "*" + + def _py_versions(self): + if not hasattr(self, "__py_versions"): + self.__py_versions = ( + version + for version in ( + PYTHON_FULL_VERSIONS + if any(needs_full_versions(spec) for spec in self.specifier) + else PYTHON_VERSIONS + ) + if ".".join(map(str, version)) in self.specifier + ) + return self.__py_versions + + def get_candidate_lookup(self): + for py_version in self._py_versions(): + delegate = RequiresPythonRequirement( + self.specifier, RequiresPythonCandidate(py_version) + ) + candidate_lookup = orig_get_candidate_lookup(delegate) + if candidate_lookup != (None, None): + return candidate_lookup + return None, None + + def is_satisfied_by(self, *_args, **_kw): + # Ensure any dependency we lock is compatible with the full interpreter range + # specified since we have no way to force Pip to backtrack and follow paths for any + # divergences. Most (all?) true divergences should be covered by forked environment + # markers. + return all( + orig_is_satisfied_by(self, RequiresPythonCandidate(py_version)) + for py_version in self._py_versions() + ) + + RequiresPythonRequirement._py_versions = _py_versions + RequiresPythonRequirement.get_candidate_lookup = get_candidate_lookup + RequiresPythonRequirement.is_satisfied_by = is_satisfied_by + else: + RequiresPythonRequirement.get_candidate_lookup = lambda self: (self._candidate, None) + RequiresPythonRequirement.is_satisfied_by = lambda *_args, **_kw: True diff --git a/pex/pip/foreign_platform/tags.py b/pex/pip/foreign_platform/tags.py new file mode 100644 index 000000000..bd5629793 --- /dev/null +++ b/pex/pip/foreign_platform/tags.py @@ -0,0 +1,26 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import itertools +import json +import os + + +def patch(): + from pip._internal.utils import compatibility_tags # type: ignore[import] + from pip._vendor.packaging import tags # type: ignore[import] + + # N.B.: The following environment variable is used by the Pex runtime to control Pip and must be + # kept in-sync with `__init__.py`. + patched_tags_file = os.environ.pop("_PEX_PATCHED_TAGS_FILE") + with open(patched_tags_file) as fp: + patched_tags = tuple( + itertools.chain.from_iterable(tags.parse_tag(tag) for tag in json.load(fp)) + ) + + def get_supported(*_args, **_kwargs): + return list(patched_tags) + + compatibility_tags.get_supported = get_supported diff --git a/pex/pip/foreign_platform_patches.py b/pex/pip/foreign_platform_patches.py deleted file mode 100644 index 68a77b60d..000000000 --- a/pex/pip/foreign_platform_patches.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -import os - -# N.B.: The following environment variables are used by the Pex runtime to control Pip and must be -# kept in-sync with `foreign_platform.py`. -patched_markers_file = os.environ.pop("_PEX_PATCHED_MARKERS_FILE", None) -patched_tags_file = os.environ.pop("_PEX_PATCHED_TAGS_FILE", None) - -if patched_markers_file: - - def patch_markers_default_environment(): - import json - - from pip._vendor.packaging import markers # type: ignore[import] - - with open(patched_markers_file) as fp: - patched_markers = json.load(fp) - - markers.default_environment = patched_markers.copy - - patch_markers_default_environment() - del patch_markers_default_environment - - -if patched_tags_file: - - def patch_compatibility_tags(): - import itertools - import json - - from pip._internal.utils import compatibility_tags # type: ignore[import] - from pip._vendor.packaging import tags # type: ignore[import] - - with open(patched_tags_file) as fp: - tags = tuple( - itertools.chain.from_iterable(tags.parse_tag(tag) for tag in json.load(fp)) - ) - - def get_supported(*args, **kwargs): - return list(tags) - - compatibility_tags.get_supported = get_supported - - patch_compatibility_tags() - del patch_compatibility_tags diff --git a/pex/pip/installation.py b/pex/pip/installation.py index a931fec78..a2530257d 100644 --- a/pex/pip/installation.py +++ b/pex/pip/installation.py @@ -53,15 +53,15 @@ def _pip_venv( import os import runpy - patches_module = os.environ.pop({patches_module_env_var_name!r}, None) - if patches_module: + patches_package = os.environ.pop({patches_package_env_var_name!r}, None) + if patches_package: # Apply runtime patches to Pip to work around issues or else bend # Pip to Pex's needs. - __import__(patches_module) + __import__(patches_package) runpy.run_module(mod_name="pip", run_name="__main__", alter_sys=True) """ - ).format(patches_module_env_var_name=Pip._PATCHES_MODULE_ENV_VAR_NAME) + ).format(patches_package_env_var_name=Pip._PATCHES_PACKAGE_ENV_VAR_NAME) ) fp.close() isolated_pip_builder.set_executable(fp.name, "__pex_patched_pip__.py") diff --git a/pex/pip/tool.py b/pex/pip/tool.py index f07d9cc4e..9ecc4322e 100644 --- a/pex/pip/tool.py +++ b/pex/pip/tool.py @@ -22,7 +22,7 @@ from pex.pep_425 import CompatibilityTags from pex.pex_bootstrapper import VenvPex from pex.pip import foreign_platform -from pex.pip.download_observer import DownloadObserver +from pex.pip.download_observer import DownloadObserver, PatchSet from pex.pip.log_analyzer import ErrorAnalyzer, ErrorMessage, LogAnalyzer, LogScrapeJob from pex.pip.tailer import Tailer from pex.pip.version import PipVersion, PipVersionValue @@ -221,17 +221,8 @@ def analyze(self, line): @attr.s(frozen=True) class Pip(object): - _PATCHES_MODULE_ENV_VAR_NAME = "_PEX_PIP_RUNTIME_PATCHES" - - @classmethod - def _patch_code(cls, code): - # type: (Text) -> Mapping[str, str] - patches_dir = safe_mkdtemp() - patches_module = "_pex_pip_patches" - python_file = "{patches_module}.py".format(patches_module=patches_module) - with open(os.path.join(patches_dir, python_file), "wb") as code_fp: - code_fp.write(code.encode("utf-8")) - return {"PEX_EXTRA_SYS_PATH": patches_dir, cls._PATCHES_MODULE_ENV_VAR_NAME: patches_module} + _PATCHES_PACKAGE_ENV_VAR_NAME = "_PEX_PIP_RUNTIME_PATCHES_PACKAGE" + _PATCHES_PACKAGE_NAME = "_pex_pip_patches" _pip_pex = attr.ib() # type: VenvPex @@ -425,8 +416,7 @@ def spawn_download_distributions( download_cmd = ["download", "--dest", download_dir] extra_env = {} # type: Dict[str, str] - if not isinstance(target, LocalInterpreter) or not build: - # If we're not targeting a local interpreter, we can't build wheels from sdists. + if not build: download_cmd.extend(["--only-binary", ":all:"]) if not use_wheel: @@ -462,9 +452,9 @@ def spawn_download_distributions( foreign_platform_observer = foreign_platform.patch(target) if ( foreign_platform_observer - and foreign_platform_observer.patch.code + and foreign_platform_observer.patch_set.patches and observer - and observer.patch.code + and observer.patch_set.patches ): raise ValueError( "Can only have one patch for Pip code, but, in addition to patching for a foreign " @@ -472,17 +462,19 @@ def spawn_download_distributions( ) log_analyzers = [] # type: List[LogAnalyzer] - code = None # type: Optional[Text] + pex_extra_sys_path = [] # type: List[str] for obs in (foreign_platform_observer, observer): if obs: if obs.analyzer: log_analyzers.append(obs.analyzer) - download_cmd.extend(obs.patch.args) - extra_env.update(obs.patch.env) - code = code or obs.patch.code + extra_env.update(obs.patch_set.env) + extra_sys_path = obs.patch_set.emit_patches(package=self._PATCHES_PACKAGE_NAME) + if extra_sys_path: + pex_extra_sys_path.append(extra_sys_path) - if code: - extra_env.update(self._patch_code(code)) + if pex_extra_sys_path: + extra_env["PEX_EXTRA_SYS_PATH"] = os.pathsep.join(pex_extra_sys_path) + extra_env[self._PATCHES_PACKAGE_ENV_VAR_NAME] = self._PATCHES_PACKAGE_NAME # The Pip 2020 resolver hides useful dependency conflict information in stdout interspersed # with other information we want to suppress. We jump though some hoops here to get at that @@ -659,11 +651,12 @@ def spawn_install_wheel( compatible_tags = CompatibilityTags.from_wheel(wheel).extend( interpreter.identity.supported_tags ) - patch = foreign_platform.patch_tags(compatible_tags) - extra_env = dict(patch.env) - if patch.code: - extra_env.update(self._patch_code(patch.code)) - install_cmd.extend(patch.args) + patch_set = PatchSet.create(foreign_platform.patch_tags(compatible_tags)) + extra_env = dict(patch_set.env) + extra_sys_path = patch_set.emit_patches(package=self._PATCHES_PACKAGE_NAME) + if extra_sys_path: + extra_env["PEX_EXTRA_SYS_PATH"] = extra_sys_path + extra_env[self._PATCHES_PACKAGE_ENV_VAR_NAME] = self._PATCHES_PACKAGE_NAME install_cmd.append("--compile" if compile else "--no-compile") install_cmd.append(wheel) diff --git a/pex/resolve/downloads.py b/pex/resolve/downloads.py index cd2b21f5d..9d54408bc 100644 --- a/pex/resolve/downloads.py +++ b/pex/resolve/downloads.py @@ -110,7 +110,7 @@ def _download( # restrictions. download_observer = DownloadObserver( analyzer=None, - patch=locker.patch(lock_configuration=LockConfiguration(style=LockStyle.UNIVERSAL)), + patch_set=locker.patch(lock_configuration=LockConfiguration(style=LockStyle.UNIVERSAL)), ) return self.pip.spawn_download_distributions( download_dir=download_dir, diff --git a/pex/resolve/locked_resolve.py b/pex/resolve/locked_resolve.py index 9141c8501..04920d8d9 100644 --- a/pex/resolve/locked_resolve.py +++ b/pex/resolve/locked_resolve.py @@ -551,21 +551,11 @@ def resolve( # type: (...) -> Union[Resolved, Error] is_local_interpreter = isinstance(target, LocalInterpreter) - if not use_wheel: - if not build: - return Error( - "Cannot both ignore wheels (use_wheel=False) and refrain from building " - "distributions (build=False)." - ) - elif not is_local_interpreter: - return Error( - "Cannot ignore wheels (use_wheel=False) when resolving for a platform: given " - "{platform_description}".format( - platform_description=target.render_description() - ) - ) - if not is_local_interpreter: - build = False + if not use_wheel and not build: + return Error( + "Cannot both ignore wheels (use_wheel=False) and refrain from building " + "distributions (build=False)." + ) repository = defaultdict(list) # type: DefaultDict[ProjectName, List[LockedRequirement]] for locked_requirement in self.locked_requirements: diff --git a/pex/resolve/locker.py b/pex/resolve/locker.py index e7cb51a54..38b164a17 100644 --- a/pex/resolve/locker.py +++ b/pex/resolve/locker.py @@ -6,24 +6,23 @@ import itertools import json import os -import pkgutil import re from collections import OrderedDict, defaultdict from pex import hashing from pex.common import safe_mkdtemp -from pex.compatibility import unquote, urlparse +from pex.compatibility import urlparse from pex.dist_metadata import ProjectNameAndVersion, Requirement from pex.hashing import Sha256 -from pex.interpreter_constraints import iter_compatible_versions from pex.orderedset import OrderedSet from pex.pep_440 import Version -from pex.pip.download_observer import Patch +from pex.pip import foreign_platform +from pex.pip.download_observer import Patch, PatchSet from pex.pip.local_project import digest_local_project from pex.pip.log_analyzer import LogAnalyzer from pex.pip.vcs import fingerprint_downloaded_vcs_archive from pex.pip.version import PipVersionValue -from pex.requirements import ArchiveScheme, VCSRequirement, VCSScheme, parse_scheme +from pex.requirements import ArchiveScheme, VCSRequirement, VCSScheme from pex.resolve.locked_resolve import LockConfiguration, LockStyle, TargetSystem from pex.resolve.pep_691.fingerprint_service import FingerprintService from pex.resolve.pep_691.model import Endpoint @@ -35,7 +34,6 @@ ResolvedRequirement, ) from pex.resolve.resolvers import Resolver -from pex.tracer import TRACER from pex.typing import TYPE_CHECKING if TYPE_CHECKING: @@ -556,58 +554,47 @@ def lock_result(self): def patch(lock_configuration): - # type: (LockConfiguration) -> Patch + # type: (LockConfiguration) -> PatchSet - code = None # type: Optional[Text] - env = {} # type: Dict[str, str] + if lock_configuration.style != LockStyle.UNIVERSAL: + return PatchSet() - if lock_configuration.style == LockStyle.UNIVERSAL: - code_bytes = pkgutil.get_data(__name__, "locker_patches.py") - assert code_bytes is not None, ( - "The sibling resource locker_patches.py of {} should always be present in a Pex " - "distribution or source tree.".format(__name__) + patches_dir = safe_mkdtemp() + patches = [] + if lock_configuration.requires_python: + patches.append( + foreign_platform.patch_requires_python( + requires_python=lock_configuration.requires_python, patches_dir=patches_dir + ) ) - code = code_bytes.decode("utf-8") - if lock_configuration.requires_python: - version_info_dir = safe_mkdtemp() - with TRACER.timed( - "Calculating compatible python versions for {requires_python}".format( - requires_python=lock_configuration.requires_python - ) - ): - python_full_versions = list( - iter_compatible_versions(lock_configuration.requires_python) - ) - with open(os.path.join(version_info_dir, "python_full_versions.json"), "w") as fp: - json.dump(python_full_versions, fp) - env.update(_PEX_PYTHON_VERSIONS_FILE=fp.name) - - if lock_configuration.target_systems and set(lock_configuration.target_systems) != set( - TargetSystem.values() - ): - target_systems = { - "os_names": [ - _OS_NAME[target_system] for target_system in lock_configuration.target_systems - ], - "platform_systems": [ - _PLATFORM_SYSTEM[target_system] - for target_system in lock_configuration.target_systems - ], - "sys_platforms": list( - itertools.chain.from_iterable( - _SYS_PLATFORMS[target_system] - for target_system in lock_configuration.target_systems - ) - ), - "platform_tag_regexps": [ - _PLATFORM_TAG_REGEXP[target_system] + env = {} # type: Dict[str, str] + if lock_configuration.target_systems and set(lock_configuration.target_systems) != set( + TargetSystem.values() + ): + target_systems = { + "os_names": [ + _OS_NAME[target_system] for target_system in lock_configuration.target_systems + ], + "platform_systems": [ + _PLATFORM_SYSTEM[target_system] + for target_system in lock_configuration.target_systems + ], + "sys_platforms": list( + itertools.chain.from_iterable( + _SYS_PLATFORMS[target_system] for target_system in lock_configuration.target_systems - ], - } - target_systems_info_dir = safe_mkdtemp() - with open(os.path.join(target_systems_info_dir, "target_systems.json"), "w") as fp: - json.dump(target_systems, fp) - env.update(_PEX_TARGET_SYSTEMS_FILE=fp.name) - - return Patch(code=code, env=env) + ) + ), + "platform_tag_regexps": [ + _PLATFORM_TAG_REGEXP[target_system] + for target_system in lock_configuration.target_systems + ], + } + with open(os.path.join(patches_dir, "target_systems.json"), "w") as fp: + json.dump(target_systems, fp) + env.update(_PEX_TARGET_SYSTEMS_FILE=fp.name) + + patches.append(Patch.from_code_resource(__name__, "locker_patches.py", **env)) + + return PatchSet(patches=tuple(patches)) diff --git a/pex/resolve/locker_patches.py b/pex/resolve/locker_patches.py index da0cea014..12f112e42 100644 --- a/pex/resolve/locker_patches.py +++ b/pex/resolve/locker_patches.py @@ -5,9 +5,16 @@ import os -python_full_versions = [] -python_versions = [] -python_majors = [] +try: + from . import requires_python # type:ignore[attr-defined] # This file will be relocated. + + python_full_versions = requires_python.PYTHON_FULL_VERSIONS + python_versions = requires_python.PYTHON_VERSIONS + python_majors = sorted(set(version[0] for version in python_full_versions)) +except ImportError: + python_full_versions = [] + python_versions = [] + python_majors = [] os_names = [] platform_systems = [] @@ -16,17 +23,8 @@ # N.B.: The following environment variables are used by the Pex runtime to control Pip and must be # kept in-sync with `locker.py`. -python_versions_file = os.environ.pop("_PEX_PYTHON_VERSIONS_FILE", None) target_systems_file = os.environ.pop("_PEX_TARGET_SYSTEMS_FILE", None) -if python_versions_file: - import json - - with open(python_versions_file) as fp: - python_full_versions = json.load(fp) - python_versions = sorted(set((version[0], version[1]) for version in python_full_versions)) - python_majors = sorted(set(version[0] for version in python_full_versions)) - if target_systems_file: import json @@ -38,13 +36,6 @@ platform_tag_regexps = target_systems["platform_tag_regexps"] -# 1.) Universal dependency environment marker applicability. -# -# Allows all dependencies in metadata to be followed regardless -# of whether they apply to this system. For example, if this is -# Python 3.10 but a marker says a dependency is only for -# 'python_version < "3.6"' we still want to lock that dependency -# subgraph too. def patch_marker_evaluate(): from pip._vendor.packaging import markers # type: ignore[import] @@ -90,14 +81,6 @@ def _eval_op(lhs, op, rhs): markers._eval_op = _eval_op -patch_marker_evaluate() -del patch_marker_evaluate - - -# 2.) Universal wheel tag applicability. -# -# Allows all wheel URLs to be checked even when the wheel does not -# match system tags. def patch_wheel_model(): from pip._internal.models.wheel import Wheel # type: ignore[import] @@ -162,114 +145,18 @@ def supported_platform_tag(self, *_args, **_kwargs): Wheel.find_most_preferred_tag = lambda *args, **kwargs: 0 -patch_wheel_model() -del patch_wheel_model - - -# 3.) Universal Python version applicability. -# -# Much like 2 (wheel applicability), we want to gather distributions -# even when they require different Pythons than the system Python. -# -# Unlike the other two patches, this patch diverges between the pip-legacy-resolver and the -# pip-2020-resolver. -def patch_requires_python(): - # The pip-legacy-resolver patch. - from pip._internal.utils import packaging # type: ignore[import] - - if python_full_versions: - orig_check_requires_python = packaging.check_requires_python - - def check_requires_python(requires_python, *_args, **_kw): - # Ensure any dependency we lock is compatible with the full interpreter range - # specified since we have no way to force Pip to backtrack and follow paths for any - # divergences. Most (all?) true divergences should be covered by forked environment - # markers. - return all( - orig_check_requires_python(requires_python, python_full_version) - for python_full_version in python_full_versions - ) - - packaging.check_requires_python = check_requires_python - else: - packaging.check_requires_python = lambda *_args, **_kw: True - - # The pip-2020-resolver patch. - from pip._internal.resolution.resolvelib.candidates import ( # type: ignore[import] - RequiresPythonCandidate, - ) - from pip._internal.resolution.resolvelib.requirements import ( # type: ignore[import] - RequiresPythonRequirement, - ) - - if python_full_versions: - orig_get_candidate_lookup = RequiresPythonRequirement.get_candidate_lookup - orig_is_satisfied_by = RequiresPythonRequirement.is_satisfied_by - - # Ensure we do a proper, but minimal, comparison for Python versions. Previously we - # always tested all `Requires-Python` specifier sets against Python full versions. That - # can be pathologically slow (see: https://github.com/pantsbuild/pants/issues/14998); so - # we avoid using Python full versions unless the `Requires-Python` specifier set - # requires that data. In other words: - # - # Need full versions to evaluate properly: - # + Requires-Python: >=3.7.6 - # + Requires-Python: >=3.7,!=3.7.6,<4 - # - # Do not need full versions to evaluate properly: - # + Requires-Python: >=3.7,<4 - # + Requires-Python: ==3.7.* - # + Requires-Python: >=3.6.0 - # - def needs_full_versions(spec): - components = spec.version.split(".", 2) - if len(components) < 3: - return False - major_, minor_, patch = components - if spec.operator in ("<", "<=", ">", ">=") and patch == "0": - return False - return patch != "*" - - def _py_versions(self): - if not hasattr(self, "__py_versions"): - self.__py_versions = ( - version - for version in ( - python_full_versions - if any(needs_full_versions(spec) for spec in self.specifier) - else python_versions - ) - if ".".join(map(str, version)) in self.specifier - ) - return self.__py_versions - - def get_candidate_lookup(self): - for py_version in self._py_versions(): - delegate = RequiresPythonRequirement( - self.specifier, RequiresPythonCandidate(py_version) - ) - candidate_lookup = orig_get_candidate_lookup(delegate) - if candidate_lookup != (None, None): - return candidate_lookup - return None, None - - def is_satisfied_by(self, *_args, **_kw): - # Ensure any dependency we lock is compatible with the full interpreter range - # specified since we have no way to force Pip to backtrack and follow paths for any - # divergences. Most (all?) true divergences should be covered by forked environment - # markers. - return all( - orig_is_satisfied_by(self, RequiresPythonCandidate(py_version)) - for py_version in self._py_versions() - ) - - RequiresPythonRequirement._py_versions = _py_versions - RequiresPythonRequirement.get_candidate_lookup = get_candidate_lookup - RequiresPythonRequirement.is_satisfied_by = is_satisfied_by - else: - RequiresPythonRequirement.get_candidate_lookup = lambda self: (self._candidate, None) - RequiresPythonRequirement.is_satisfied_by = lambda *_args, **_kw: True - - -patch_requires_python() -del patch_requires_python +def patch(): + # 1.) Universal dependency environment marker applicability. + # + # Allows all dependencies in metadata to be followed regardless + # of whether they apply to this system. For example, if this is + # Python 3.10 but a marker says a dependency is only for + # 'python_version < "3.6"' we still want to lock that dependency + # subgraph too. + patch_marker_evaluate() + + # 2.) Universal wheel tag applicability. + # + # Allows all wheel URLs to be checked even when the wheel does not + # match system tags. + patch_wheel_model() diff --git a/pex/resolve/lockfile/create.py b/pex/resolve/lockfile/create.py index 9ca0dc356..742454cab 100644 --- a/pex/resolve/lockfile/create.py +++ b/pex/resolve/lockfile/create.py @@ -204,8 +204,8 @@ def observe_download( max_parallel_jobs=self.max_parallel_jobs, ), ) - patch = locker.patch(lock_configuration=self.lock_configuration) - observer = DownloadObserver(analyzer=analyzer, patch=patch) + patch_set = locker.patch(lock_configuration=self.lock_configuration) + observer = DownloadObserver(analyzer=analyzer, patch_set=patch_set) self._analysis.add( _LockAnalysis(target=target, analyzer=analyzer, download_dir=download_dir) ) diff --git a/pex/resolver.py b/pex/resolver.py index 40d1914d4..403abecf3 100644 --- a/pex/resolver.py +++ b/pex/resolver.py @@ -17,11 +17,12 @@ from pex.auth import PasswordEntry from pex.common import safe_mkdir, safe_mkdtemp from pex.compatibility import unquote, urlparse -from pex.dist_metadata import DistMetadata, Distribution, Requirement +from pex.dist_metadata import DistMetadata, Distribution, ProjectNameAndVersion, Requirement from pex.fingerprinted_distribution import FingerprintedDistribution from pex.jobs import Raise, SpawnedJob, execute_parallel from pex.network_configuration import NetworkConfiguration from pex.orderedset import OrderedSet +from pex.pep_425 import CompatibilityTags from pex.pep_503 import ProjectName from pex.pex_info import PexInfo from pex.pip.download_observer import DownloadObserver @@ -323,7 +324,27 @@ def finalize_build(self): ) ) wheel = wheels[0] - return InstallRequest.create(self.request.target, os.path.join(self.dist_dir, wheel)) + wheel_path = os.path.join(self.dist_dir, wheel) + if self.request.target.is_foreign: + wheel_tags = CompatibilityTags.from_wheel(wheel_path) + if not self.request.target.supported_tags.compatible_tags(wheel_tags): + project_name_and_version = ProjectNameAndVersion.from_filename(wheel_path) + raise ValueError( + "No pre-built wheel was available for {project_name} {version}.{eol}" + "Successfully built the wheel {wheel} from the sdist {sdist} but it is not " + "compatible with the requested foreign target {target}.{eol}" + "You'll need to build a wheel from {sdist} on the foreign target platform and " + "make it available to Pex via a `--find-links` repo or a custom " + "`--index`.".format( + project_name=project_name_and_version.project_name, + version=project_name_and_version.version, + eol=os.linesep, + wheel=wheel, + sdist=os.path.basename(self.request.source_path), + target=self.request.target.render_description(), + ) + ) + return InstallRequest.create(self.request.target, wheel_path) @attr.s(frozen=True) diff --git a/pex/venv/pex.py b/pex/venv/pex.py index 4bbe9e3c7..e0d832f10 100644 --- a/pex/venv/pex.py +++ b/pex/venv/pex.py @@ -438,7 +438,7 @@ def sys_executable_paths(): "_PEX_TEST_PYENV_ROOT", "_PEX_PIP_VERSION", # This is used by Pex's Pip to inject runtime patches dynamically. - "_PEX_PIP_RUNTIME_PATCHES", + "_PEX_PIP_RUNTIME_PATCHES_PACKAGE", # These are used by Pex's Pip venv to provide foreign platform support and work # around https://github.com/pypa/pip/issues/10050. "_PEX_PATCHED_MARKERS_FILE", diff --git a/tests/integration/cli/commands/test_export.py b/tests/integration/cli/commands/test_export.py index f373e5bcd..c1730fb9b 100644 --- a/tests/integration/cli/commands/test_export.py +++ b/tests/integration/cli/commands/test_export.py @@ -269,7 +269,8 @@ def test_export_respects_target(tmpdir): assert dedent( """\ ansicolors==1.1.8 \\ - --hash=md5:abcd1234 + --hash=md5:abcd1234 \\ + --hash=sha1:ef567890 pywin32==227 \\ --hash=sha256:spameggs """ @@ -284,8 +285,8 @@ def test_export_respects_target(tmpdir): } ), ), ( - "A win32 foreign target should get all wheels but no sdists since we can't cross-build " - "sdists." + "A win32 foreign target should get both ansicolors cross-platform artifacts as well as " + "the platform-specific pywin32 wheel." ) assert ( diff --git a/tests/integration/cli/commands/test_lock.py b/tests/integration/cli/commands/test_lock.py index 51c581891..95f01a30e 100644 --- a/tests/integration/cli/commands/test_lock.py +++ b/tests/integration/cli/commands/test_lock.py @@ -19,6 +19,7 @@ from pex.pep_440 import Version from pex.pep_503 import ProjectName from pex.pip.version import PipVersion +from pex.platforms import Platform from pex.resolve.locked_resolve import Artifact, LockedRequirement from pex.resolve.lockfile import json_codec from pex.resolve.lockfile.download_manager import DownloadedArtifact @@ -27,8 +28,9 @@ from pex.resolve.resolver_configuration import ResolverVersion from pex.resolve.testing import normalize_locked_resolve from pex.sorted_tuple import SortedTuple -from pex.targets import LocalInterpreter +from pex.targets import AbbreviatedPlatform, LocalInterpreter from pex.testing import ( + IS_LINUX, IS_LINUX_ARM64, IS_MAC, IS_PYPY, @@ -249,6 +251,11 @@ def test_create_universal_python_unsupported(): def test_create_universal_platform_check(tmpdir): # type: (Any) -> None + foreign_platform_310 = ( + "macosx_10.9_x86_64-cp-310-cp310" if IS_LINUX else "linux_x86_64-cp-310-cp310" + ) + abbreviated_platform_310 = AbbreviatedPlatform.create(Platform.create(foreign_platform_310)) + complete_platform = os.path.join(str(tmpdir), "complete-platform.json") run_pex3("interpreter", "inspect", "--markers", "--tags", "-v", "-i2", "-o", complete_platform) @@ -260,7 +267,7 @@ def test_create_universal_platform_check(tmpdir): "--style", "universal", "--platform", - "linux_x86_64-cp-310-cp310", + foreign_platform_310, "ansicolors==1.1.8", ).assert_success() run_pex3( @@ -273,28 +280,76 @@ def test_create_universal_platform_check(tmpdir): "ansicolors==1.1.8", ).assert_success() - # But if we exclude wheels, we should now fail any platform check. + # If we exclude wheels for an abbreviated platform, we should still be OK for ansicolors since + # it is cross-platform, and we can "cross-build" it. The one exception is Python 2.7. Even + # though the ansicolors wheel should be universal - it does build for py2 and py3, it is not + # marked as such and thus Python 2 builds ansicolors-1.1.8-py2-none-any.whl and Python 3 builds + # ansicolors-1.1.8-py3-none-any.whl. result = run_pex3( "lock", "create", "--style", "universal", "--platform", - "linux_x86_64-cp-310-cp310", + foreign_platform_310, "--no-wheel", "ansicolors==1.1.8", ) + if sys.version_info[0] == 2: + result.assert_failure() + assert ( + re.search( + r"No pre-built wheel was available for ansicolors 1\.1\.8\.{eol}" + r"Successfully built the wheel ansicolors-1\.1\.8-py2-none-any\.whl from the sdist " + r"ansicolors-1\.1\.8\.zip but it is not compatible with the requested foreign target " + r"{foreign_target}\.{eol}" + r"You'll need to build a wheel from ansicolors-1\.1\.8\.zip on the foreign target " + r"platform and make it available to Pex via a `--find-links` repo or a custom " + r"`--index`\." + r"".format( + foreign_target=re.escape(abbreviated_platform_310.render_description()), + eol=os.linesep, + ), + result.error, + ) + is not None + ), result.error + else: + result.assert_success() + + # But not for psutil because it is platform-specific, and we cannot "cross-build" it. + result = run_pex3( + "lock", + "create", + "--style", + "universal", + "--platform", + foreign_platform_310, + "--no-wheel", + "psutil==5.9.1", + ) result.assert_failure() + assert ( - dedent( - """\ - Failed to resolve compatible artifacts from lock for 1 target: - 1. cp310-cp310-linux_x86_64: - Cannot ignore wheels (use_wheel=False) when resolving for a platform: given abbreviated platform cp310-cp310-linux_x86_64 - """ + re.search( + r"No pre-built wheel was available for psutil 5\.9\.1\.{eol}" + r"Successfully built the wheel psutil-5\.9\.1-\S+\.whl from the sdist " + r"psutil-5\.9\.1\.tar\.gz but it is not compatible with the requested foreign target " + r"{foreign_target}\.{eol}" + r"You'll need to build a wheel from psutil-5\.9\.1\.tar\.gz on the foreign target " + r"platform and make it available to Pex via a `--find-links` repo or a custom " + r"`--index`\." + r"".format( + foreign_target=re.escape(abbreviated_platform_310.render_description()), + eol=os.linesep, + ), + result.error, ) - in result.error - ) + is not None + ), result.error + + # If we exclude wheels for a complete platform, we should still be OK for ansicolors since it is + # cross-platform. run_pex3( "lock", "create", @@ -304,79 +359,47 @@ def test_create_universal_platform_check(tmpdir): complete_platform, "--no-wheel", "ansicolors==1.1.8", - ).assert_failure() + ).assert_success() + # There are 3.10 wheels for all platforms we test here. run_pex3( "lock", "create", "--style", "universal", "--platform", - "macosx_10.9_x86_64-cp-310-cp310", + foreign_platform_310, "psutil==5.9.1", ).assert_success() - # Here there is no pre-built wheel for CPython 3.11 on any platform; so we expect failure when - # checking the --platform macosx_10.9_x86_64-cp-311-cp311 will fail. + # Here there is no pre-built wheel for CPython 3.11 on any platform; so we expect failure from + # the "cross-build" attempt. + foreign_platform_311 = ( + "macosx_10.9_x86_64-cp-311-cp311" if IS_LINUX else "linux_x86_64-cp-311-cp311" + ) result = run_pex3( "lock", "create", "--style", "universal", "--platform", - "macosx_10.9_x86_64-cp-311-cp311", + foreign_platform_311, "psutil==5.9.1", ) result.assert_failure() - assert ( - dedent( - """\ - Failed to resolve compatible artifacts from lock for 1 target: - 1. cp311-cp311-macosx_10_9_x86_64: - Failed to resolve all requirements for abbreviated platform cp311-cp311-macosx_10_9_x86_64: - - Configured with: - build: False - use_wheel: True - - Dependency on psutil not satisfied, 1 incompatible candidate found: - 1.) psutil 5.9.1 (via: psutil==5.9.1) does not have any compatible artifacts: - https://files.pythonhosted.org/packages/77/06/f9fd79449440d7217d6bf2c90998d540e125cfeffe39d214a328dadc46f4/psutil-5.9.1-cp27-cp27m-manylinux2010_i686.whl - https://files.pythonhosted.org/packages/13/71/c25adbd9b33a2e27edbe1fc84b3111a5ad97611885d7abcbdd8d1f2bb7ca/psutil-5.9.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl - https://files.pythonhosted.org/packages/14/06/39d7e963a6a8bbf26519de208593cdb0ddfe22918b8989f4b2363d4ab49f/psutil-5.9.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl - https://files.pythonhosted.org/packages/1b/53/8f0772df0a6d593bc2fcdf12f4f790bab5c4f6a77bb61a8ddaad2cbba7f8/psutil-5.9.1-cp27-cp27m-win_amd64.whl - https://files.pythonhosted.org/packages/26/b4/a58cf15ea649faa92c54f00c627aef1d50b9f1abf207485f10c967a50c95/psutil-5.9.1-cp310-cp310-win32.whl - https://files.pythonhosted.org/packages/2a/32/136cd5bf55728ea64a22b1d817890e35fc17314c46a24ee3268b65f9076f/psutil-5.9.1-cp37-cp37m-win32.whl - https://files.pythonhosted.org/packages/2c/9d/dc329b7da284677ea843f3ff4b35b8ab3b96b65a58a544b3c3f86d9d032f/psutil-5.9.1-cp27-cp27mu-manylinux2010_x86_64.whl - https://files.pythonhosted.org/packages/2d/56/54b4ed8102ce5a2f5367b4e766c1873c18f9c32cde321435d0e0ee2abcc5/psutil-5.9.1-cp27-cp27mu-manylinux2010_i686.whl - https://files.pythonhosted.org/packages/41/ec/5fd3e9388d0ed1edfdeae71799df374f4a117932646a63413fa95a121e9f/psutil-5.9.1-cp38-cp38-win32.whl - https://files.pythonhosted.org/packages/46/80/1de3a9bac336b5c8e4f7b0ff2e80c85ba237f18f2703be68884ee6798432/psutil-5.9.1-cp38-cp38-macosx_10_9_x86_64.whl - https://files.pythonhosted.org/packages/62/1f/f14225bda76417ab9bd808ff21d5cd59d5435a9796ca09b34d4cb0edcd88/psutil-5.9.1-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl - https://files.pythonhosted.org/packages/65/1d/6a112f146faee6292a6c3ee2a7f24a8e572697adb7e1c5de3d8508f647cc/psutil-5.9.1-cp36-cp36m-macosx_10_9_x86_64.whl - https://files.pythonhosted.org/packages/6b/76/a8cb69ed3566877dcbccf408f5f9d6055227ad4fed694e88809fa8506b0b/psutil-5.9.1-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl - https://files.pythonhosted.org/packages/6d/c6/6a4e46802e8690d50ba6a56c7f79ac283e703fcfa0fdae8e41909c8cef1f/psutil-5.9.1-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl - https://files.pythonhosted.org/packages/73/1a/d78f2f2de2aad6628415d2a48917cabc2c7fb0c3a31c7cdf187cffa4eb36/psutil-5.9.1-cp36-cp36m-win_amd64.whl - https://files.pythonhosted.org/packages/7e/52/a02dc53e26714a339c8b4972d8e3f268e4db8905f5d1a3a100f1e40b6fa7/psutil-5.9.1-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl - https://files.pythonhosted.org/packages/7e/8d/e0a66123fa98e309597815de518b47a7a6c571a8f886fc8d4db2331fd2ab/psutil-5.9.1-cp27-cp27m-win32.whl - https://files.pythonhosted.org/packages/85/4d/78173e3dffb74c5fa87914908f143473d0b8b9183f9d275333679a4e4649/psutil-5.9.1-cp36-cp36m-win32.whl - https://files.pythonhosted.org/packages/97/f6/0180e58dd1359da7d6fbc27d04dac6fb500dc758b6f4b65407608bb13170/psutil-5.9.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl - https://files.pythonhosted.org/packages/9d/41/d5f2db2ab7f5dff2fa795993a0cd6fa8a8f39ca197c3a86857875333ec10/psutil-5.9.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl - https://files.pythonhosted.org/packages/9f/ca/84ce3e48b3ca2f0f74314d89929b3a523220f3f4a8dff395d6ef74dadef3/psutil-5.9.1-cp39-cp39-macosx_10_9_x86_64.whl - https://files.pythonhosted.org/packages/a9/97/b7e3532d97d527349701d2143c3f868733b94e2db6f531b07811b698f549/psutil-5.9.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl - https://files.pythonhosted.org/packages/b1/d2/c5374a784567c1e42ee8a589b1b42e2bd6e14c7be3c234d84360ab3a0a39/psutil-5.9.1-cp39-cp39-win32.whl - https://files.pythonhosted.org/packages/b2/ad/65e2b2b97677f98d718388dc11b2a9d7f177ebbae5eef72547a32bc28911/psutil-5.9.1-cp38-cp38-win_amd64.whl - https://files.pythonhosted.org/packages/c0/5a/2ac88d5265b711c8aa4e786825b38d5d0b1e5ecbdd0ce78e9b04a820d247/psutil-5.9.1-cp310-cp310-win_amd64.whl - https://files.pythonhosted.org/packages/cf/29/ad704a45960bfb52ef8bf0beb9c41c09ce92d61c40333f03e9a03f246c22/psutil-5.9.1-cp27-cp27m-manylinux2010_x86_64.whl - https://files.pythonhosted.org/packages/d1/16/6239e76ab5d990dc7866bc22a80585f73421588d63b42884d607f5f815e2/psutil-5.9.1-cp310-cp310-macosx_10_9_x86_64.whl - https://files.pythonhosted.org/packages/d6/de/0999ea2562b96d7165812606b18f7169307b60cd378bc29cf3673322c7e9/psutil-5.9.1.tar.gz - https://files.pythonhosted.org/packages/d6/ef/fd4dc9085e3879c3af63fe60667dd3b71adf50d030b5549315f4a619271b/psutil-5.9.1-cp37-cp37m-macosx_10_9_x86_64.whl - https://files.pythonhosted.org/packages/df/88/427f3959855fcb3ab04891e00c026a246892feb11b20433db814b7a24405/psutil-5.9.1-cp37-cp37m-win_amd64.whl - https://files.pythonhosted.org/packages/e0/ac/fd6f098969d49f046083ac032e6788d9f861903596fb9555a02bf50a1238/psutil-5.9.1-cp39-cp39-win_amd64.whl - https://files.pythonhosted.org/packages/fd/ba/c5a3f46f351ab609cc0be6a563e492900c57e3d5c9bda0b79b84d8c3eae9/psutil-5.9.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl - """ - ) - in result.error - ) + abbreviated_platform_311 = AbbreviatedPlatform.create(Platform.create(foreign_platform_311)) + assert re.search( + r"No pre-built wheel was available for psutil 5\.9\.1\.{eol}" + r"Successfully built the wheel psutil-5\.9\.1-\S+\.whl from the sdist " + r"psutil-5\.9\.1\.tar\.gz but it is not compatible with the requested foreign target " + r"{foreign_target}\.{eol}" + r"You'll need to build a wheel from psutil-5\.9\.1\.tar\.gz on the foreign target " + r"platform and make it available to Pex via a `--find-links` repo or a custom " + r"`--index`\.".format( + foreign_target=abbreviated_platform_311.render_description(), eol=os.linesep + ), + result.error, + ), result.error UPDATE_LOCKFILE_CONTENTS = """\ diff --git a/tests/integration/test_issue_2073.py b/tests/integration/test_issue_2073.py new file mode 100644 index 000000000..38f97fe00 --- /dev/null +++ b/tests/integration/test_issue_2073.py @@ -0,0 +1,115 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +import os.path +import re +import subprocess + +from pex.cli.testing import run_pex3 +from pex.platforms import Platform +from pex.targets import AbbreviatedPlatform +from pex.testing import IS_LINUX, IntegResults, run_pex_command +from pex.typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any + + +FOREIGN_PLATFORM_311 = ( + "macosx_10.9_x86_64-cp-311-cp311" if IS_LINUX else "linux_x86_64-cp-311-cp311" +) +ABBREVIATED_FOREIGN_PLATFORM_311 = AbbreviatedPlatform.create(Platform.create(FOREIGN_PLATFORM_311)) + + +def assert_psutil_cross_build_failure(result): + # type: (IntegResults) -> None + result.assert_failure() + assert ( + re.search( + r"No pre-built wheel was available for psutil 5\.9\.1\.{eol}" + r"Successfully built the wheel psutil-5\.9\.1-\S+\.whl from the sdist " + r"psutil-5\.9\.1\.tar\.gz but it is not compatible with the requested foreign target " + r"{foreign_target}\.{eol}" + r"You'll need to build a wheel from psutil-5\.9\.1\.tar\.gz on the foreign target platform " + r"and make it available to Pex via a `--find-links` repo or a custom `--index`\.".format( + eol=os.linesep, + foreign_target=ABBREVIATED_FOREIGN_PLATFORM_311.render_description(), + ), + result.error, + ) + is not None + ), result.error + + +def assert_cowsay_cross_build_success( + tmpdir, # type: Any + *args # type: str +): + pex = os.path.join(str(tmpdir), "pex") + run_pex_command( + args=["cowsay==5.0", "-c", "cowsay", "--platform", FOREIGN_PLATFORM_311, "-o", pex] + + list(args) + ).assert_success() + assert "5.0" == subprocess.check_output(args=[pex, "--version"]).decode("utf-8").strip() + + +def test_standard_resolve_foreign_platform_yolo_cross_build(tmpdir): + # type: (Any) -> None + + # There is no pre-built wheel for CPython 3.11 on any platform; so we expect failure from the + # "cross-build" attempt. + assert_psutil_cross_build_failure( + run_pex_command(args=["psutil==5.9.1", "--platform", FOREIGN_PLATFORM_311]) + ) + + # The cowsay 5.0 distribution is sdist-only. We should grab this and attempt a build to see if + # we succeed and if the resulting wheel is compatible, which it should be since cowsay 5.0 is + # known to build to a py2.py3 universal wheel. + assert_cowsay_cross_build_success(tmpdir) + + +def create_lock( + lock, # type: str + *args # type: str +): + # type: (...) -> IntegResults + return run_pex3( + "lock", + "create", + "--style", + "universal", + "--target-system", + "linux", + "--target-system", + "mac", + "-o", + lock, + "--indent", + "2", + *args + ) + + +def test_lock_create_foreign_platform_yolo_cross_build(tmpdir): + # type: (Any) -> None + + lock = os.path.join(str(tmpdir), "lock") + + assert_psutil_cross_build_failure( + create_lock(lock, "--platform", FOREIGN_PLATFORM_311, "psutil==5.9.1") + ) + + create_lock(lock, "--platform", FOREIGN_PLATFORM_311, "cowsay==5.0").assert_success() + + +def test_lock_resolve_foreign_platform_yolo_cross_build(tmpdir): + # type: (Any) -> None + + lock = os.path.join(str(tmpdir), "lock") + create_lock(lock, "psutil==5.9.1", "cowsay==5.0").assert_success() + + assert_psutil_cross_build_failure( + run_pex_command(args=["psutil==5.9.1", "--platform", FOREIGN_PLATFORM_311, "--lock", lock]) + ) + + assert_cowsay_cross_build_success(tmpdir, "--lock", lock) diff --git a/tests/resolve/test_locked_resolve.py b/tests/resolve/test_locked_resolve.py index b23aa529d..79c7c0e4f 100644 --- a/tests/resolve/test_locked_resolve.py +++ b/tests/resolve/test_locked_resolve.py @@ -279,13 +279,6 @@ def test_invalid_configuration( "(build=False).", ) - platform_target = platform("linux-x86_64-cp-37-m") - assert_error( - ansicolors_exotic.resolve(platform_target, [req("ansicolors")], use_wheel=False), - "Cannot ignore wheels (use_wheel=False) when resolving for a platform: given " - "{target_description}".format(target_description=platform_target.render_description()), - ) - def test_platform_resolve(ansicolors_exotic): # type: (LockedResolve) -> None diff --git a/tests/test_pip.py b/tests/test_pip.py index 1541fc829..2f83b2307 100644 --- a/tests/test_pip.py +++ b/tests/test_pip.py @@ -3,7 +3,6 @@ from __future__ import absolute_import -import glob import hashlib import json import os @@ -11,7 +10,6 @@ import pytest -from pex import targets from pex.common import safe_rmtree from pex.interpreter import PythonInterpreter from pex.jobs import Job @@ -21,7 +19,7 @@ from pex.platforms import Platform from pex.resolve.configured_resolver import ConfiguredResolver from pex.targets import AbbreviatedPlatform, LocalInterpreter, Target -from pex.testing import PY310, ensure_python_interpreter, environment_as +from pex.testing import IS_LINUX, PY310, ensure_python_interpreter, environment_as from pex.typing import TYPE_CHECKING from pex.variables import ENV @@ -106,18 +104,28 @@ def test_no_duplicate_constraints_pex_warnings( ) +@pytest.mark.skipif( + not IS_LINUX + or not any( + ( + "manylinux2014_x86_64" == platform.platform + for platform in PythonInterpreter.get().supported_platforms + ) + ), + reason="Test requires a manylinux2014_x86_64 compatible interpreter.", +) @applicable_pip_versions def test_download_platform_issues_1355( create_pip, # type: CreatePip version, # type: PipVersionValue - current_interpreter, # type: PythonInterpreter + py38, # type: PythonInterpreter tmpdir, # type: Any ): # type: (...) -> None - pip = create_pip(current_interpreter, version=version) + pip = create_pip(py38, version=version) download_dir = os.path.join(str(tmpdir), "downloads") - def download_ansicolors( + def download_pyarrow( target=None, # type: Optional[Target] package_index_configuration=None, # type: Optional[PackageIndexConfiguration] ): @@ -125,49 +133,29 @@ def download_ansicolors( safe_rmtree(download_dir) return pip.spawn_download_distributions( download_dir=download_dir, - requirements=["ansicolors==1.0.2"], + requirements=["pyarrow==4.0.1"], transitive=False, target=target, package_index_configuration=package_index_configuration, ) - def assert_ansicolors_downloaded(target=None): - # type: (Optional[Target]) -> None - download_ansicolors(target=target).wait() - assert ["ansicolors-1.0.2.tar.gz"] == os.listdir(download_dir) - - # The only ansicolors 1.0.2 dist on PyPI is an sdist and we should be able to download one of - # those with the current interpreter since we have an interpreter in hand to build a wheel from - # it with later. - assert_ansicolors_downloaded() - assert_ansicolors_downloaded(target=targets.current()) - assert_ansicolors_downloaded(target=LocalInterpreter.create(current_interpreter)) - - wheel_dir = os.path.join(str(tmpdir), "wheels") - pip.spawn_build_wheels( - distributions=glob.glob(os.path.join(download_dir, "*.tar.gz")), - wheel_dir=wheel_dir, - interpreter=current_interpreter, - ).wait() - built_wheels = glob.glob(os.path.join(wheel_dir, "*.whl")) - assert len(built_wheels) == 1 - - ansicolors_wheel = built_wheels[0] - local_wheel_repo = PackageIndexConfiguration.create(find_links=[wheel_dir]) - current_platform = AbbreviatedPlatform.create(current_interpreter.platform) - - # We should fail to find a wheel for ansicolors 1.0.2 and thus fail to download for a target - # Platform, even if that target platform happens to match the current interpreter we're - # executing Pip with. - with pytest.raises(Job.Error): - download_ansicolors(target=current_platform).wait() - - # If we point the target Platform to a find-links repo with the wheel just-built though, the - # download should proceed without error. - download_ansicolors( - target=current_platform, package_index_configuration=local_wheel_repo - ).wait() - assert [os.path.basename(ansicolors_wheel)] == os.listdir(download_dir) + def assert_pyarrow_downloaded( + expected_wheel, # type: str + target=None, # type: Optional[Target] + ): + # type: (...) -> None + download_pyarrow(target=target).wait() + assert [expected_wheel] == os.listdir(download_dir) + + assert_pyarrow_downloaded( + "pyarrow-4.0.1-cp38-cp38-manylinux2014_x86_64.whl", target=LocalInterpreter.create(py38) + ) + assert_pyarrow_downloaded( + "pyarrow-4.0.1-cp38-cp38-manylinux2010_x86_64.whl", + target=AbbreviatedPlatform.create( + Platform.create("linux-x86_64-cp-38-cp38"), manylinux="manylinux2010" + ), + ) def assert_download_platform_markers_issue_1366(