Attempt "cross-builds" of sdists for foreign platforms. (#2075)

The "cross-building" is in scare-quotes because this is not actually
cross-building, it is just attempting a build of the sdist, and, if
successful, seeing if the resulting wheel matches the foreign platform.
This enables sdist-only projects to be used in foreign platform Pex
operations when it turns out the sdist is multi-platform.

Closes #2073

pex/pip/download_observer.py

@@ -1,13 +1,17 @@
 # Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-from __future__ import absolute_import
+from __future__ import absolute_import, print_function
 
+import os
+import pkgutil
+
+from pex.common import safe_mkdtemp
 from pex.pip.log_analyzer import LogAnalyzer
 from pex.typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from typing import Iterable, Mapping, Optional, Text
+    from typing import Dict, Mapping, Optional, Text, Tuple
 
     import attr  # vendor:skip
 else:
@@ -16,12 +20,81 @@
 
 @attr.s(frozen=True)
 class Patch(object):
-    code = attr.ib(default=None)  # type: Optional[Text]
-    args = attr.ib(default=())  # type: Iterable[str]
+    @classmethod
+    def from_code_resource(
+        cls,
+        package,  # type: str
+        resource,  # type: str
+        **env  # type: str
+    ):
+        # type: (...) -> Patch
+        module, ext = os.path.splitext(resource)
+        if ext != ".py":
+            raise ValueError(
+                "Code resources must be `.py` files, asked to load: {resource}".format(
+                    resource=resource
+                )
+            )
+        code = pkgutil.get_data(package, resource)
+        assert code is not None, (
+            "The resource {resource} relative to {package} should always be present in a "
+            "Pex distribution or source tree.".format(resource=resource, package=package)
+        )
+        return cls(module=module, code=code.decode("utf-8"), env=env)
+
+    module = attr.ib()  # type: str
+    code = attr.ib()  # type: Text
     env = attr.ib(factory=dict)  # type: Mapping[str, str]
 
 
+@attr.s(frozen=True)
+class PatchSet(object):
+    @classmethod
+    def create(cls, *patches):
+        # type: (*Patch) -> PatchSet
+        return cls(patches=patches)
+
+    patches = attr.ib(default=())  # type: Tuple[Patch, ...]
+
+    @property
+    def env(self):
+        # type: () -> Dict[str, str]
+        env = {}  # type: Dict[str, str]
+        for patch in self.patches:
+            env.update(patch.env)
+        return env
+
+    def emit_patches(self, package):
+        # type: (str) -> Optional[str]
+
+        if not self.patches:
+            return None
+
+        if not package or "." in package:
+            raise ValueError(
+                "The `package` argument must be a non-empty, non-nested package name. "
+                "Given: {package!r}".format(package=package)
+            )
+
+        patches_dir = safe_mkdtemp()
+        patches_package = os.path.join(patches_dir, package)
+        os.mkdir(patches_package)
+
+        for patch in self.patches:
+            python_file = "{module}.py".format(module=patch.module)
+            with open(os.path.join(patches_package, python_file), "wb") as code_fp:
+                code_fp.write(patch.code.encode("utf-8"))
+
+        with open(os.path.join(patches_package, "__init__.py"), "w") as fp:
+            print("from __future__ import absolute_import", file=fp)
+            for patch in self.patches:
+                print("from . import {module}".format(module=patch.module), file=fp)
+                print("{module}.patch()".format(module=patch.module), file=fp)
+
+        return patches_dir
+
+
 @attr.s(frozen=True)
 class DownloadObserver(object):
     analyzer = attr.ib()  # type: Optional[LogAnalyzer]
-    patch = attr.ib()  # type: Patch
+    patch_set = attr.ib()  # type: PatchSet

pex/pip/foreign_platform.py → pex/pip/foreign_platform/__init__.py

@@ -5,20 +5,20 @@
 
 import json
 import os
-import pkgutil
 import re
 
 from pex.common import safe_mkdtemp
+from pex.interpreter_constraints import iter_compatible_versions
 from pex.pep_425 import CompatibilityTags
-from pex.pip.download_observer import DownloadObserver, Patch
+from pex.pip.download_observer import DownloadObserver, Patch, PatchSet
 from pex.pip.log_analyzer import ErrorAnalyzer, ErrorMessage
 from pex.platforms import Platform
 from pex.targets import AbbreviatedPlatform, CompletePlatform, Target
 from pex.tracer import TRACER
 from pex.typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from typing import Iterator, Optional, Text, Tuple
+    from typing import Iterable, Iterator, Optional
 
     import attr  # vendor:skip
 
@@ -91,54 +91,83 @@ def analyze(self, line):
         return self.Continue()
 
 
-_CODE = None  # type: Optional[Text]
-
-
-def _code():
-    # type: () -> Text
-    global _CODE
-    if _CODE is None:
-        code = pkgutil.get_data(__name__, "foreign_platform_patches.py")
-        assert code is not None, (
-            "The sibling resource foreign_platform_patches.py of {} should always be present in a "
-            "Pex distribution or source tree.".format(__name__)
-        )
-        _CODE = code.decode("utf-8")
-    return _CODE
-
-
 def patch(target):
     # type: (Target) -> Optional[DownloadObserver]
     if not isinstance(target, (AbbreviatedPlatform, CompletePlatform)):
         return None
 
     analyzer = _Issue10050Analyzer(target.platform)
-    args = ()  # type: Tuple[str, ...]
 
+    patches = []
     patches_dir = safe_mkdtemp()
+
     patched_environment = target.marker_environment.as_dict()
     with open(os.path.join(patches_dir, "markers.json"), "w") as markers_fp:
         json.dump(patched_environment, markers_fp)
-    env = dict(_PEX_PATCHED_MARKERS_FILE=markers_fp.name)
-
-    if isinstance(target, AbbreviatedPlatform):
-        args = tuple(iter_platform_args(target.platform, target.manylinux))
+    patches.append(
+        Patch.from_code_resource(__name__, "markers.py", _PEX_PATCHED_MARKERS_FILE=markers_fp.name)
+    )
 
     compatible_tags = target.supported_tags
     if compatible_tags:
-        env.update(patch_tags(compatible_tags).env)
+        patches.append(patch_tags(compatible_tags=compatible_tags, patches_dir=patches_dir))
+
+    assert (
+        target.marker_environment.python_full_version or target.marker_environment.python_version
+    ), (
+        "A complete platform should always have both `python_full_version` and `python_version` "
+        "environment markers defined and an abbreviated platform should always have at least the"
+        "`python_version` environment marker defined. Given: {target}".format(target=target)
+    )
+    requires_python = (
+        "=={full_version}".format(full_version=target.marker_environment.python_full_version)
+        if target.marker_environment.python_full_version
+        else "=={version}.*".format(version=target.marker_environment.python_version)
+    )
+    patches.append(
+        patch_requires_python(requires_python=[requires_python], patches_dir=patches_dir)
+    )
 
     TRACER.log(
         "Patching environment markers for {} with {}".format(target, patched_environment),
         V=3,
     )
-    return DownloadObserver(analyzer=analyzer, patch=Patch(code=_code(), args=args, env=env))
+    return DownloadObserver(analyzer=analyzer, patch_set=PatchSet(patches=tuple(patches)))
 
 
-def patch_tags(compatible_tags):
-    # type: (CompatibilityTags) -> Patch
-    patches_dir = safe_mkdtemp()
-    with open(os.path.join(patches_dir, "tags.json"), "w") as tags_fp:
+def patch_tags(
+    compatible_tags,  # type: CompatibilityTags
+    patches_dir=None,  # type: Optional[str]
+):
+    # type: (...) -> Patch
+    with open(os.path.join(patches_dir or safe_mkdtemp(), "tags.json"), "w") as tags_fp:
         json.dump(compatible_tags.to_string_list(), tags_fp)
-    env = dict(_PEX_PATCHED_TAGS_FILE=tags_fp.name)
-    return Patch(env=env, code=_code())
+    return Patch.from_code_resource(__name__, "tags.py", _PEX_PATCHED_TAGS_FILE=tags_fp.name)
+
+
+def patch_requires_python(
+    requires_python,  # type: Iterable[str]
+    patches_dir=None,  # type: Optional[str]
+):
+    # type: (...) -> Patch
+    """N.B.: This Path exports Python version information in the `requires_python` module.
+
+    Exports:
+    + PYTHON_FULL_VERSIONS: List[Tuple[int, int, int]]
+        A sorted list of Python full versions compatible with the given `requires_python`.
+    + PYTHON_VERSIONS: List[Tuple[int, int]]
+        A sorted list of Python versions compatible with the given `requires_python`.
+    """
+    with TRACER.timed(
+        "Calculating compatible python versions for {requires_python}".format(
+            requires_python=requires_python
+        )
+    ):
+        python_full_versions = list(iter_compatible_versions(requires_python))
+        with open(
+            os.path.join(patches_dir or safe_mkdtemp(), "python_full_versions.json"), "w"
+        ) as fp:
+            json.dump(python_full_versions, fp)
+        return Patch.from_code_resource(
+            __name__, "requires_python.py", _PEX_PYTHON_VERSIONS_FILE=fp.name
+        )

pex/pip/foreign_platform/markers.py

@@ -0,0 +1,19 @@
+# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import
+
+import json
+import os
+
+
+def patch():
+    from pip._vendor.packaging import markers  # type: ignore[import]
+
+    # N.B.: The following environment variable is used by the Pex runtime to control Pip and must be
+    # kept in-sync with `__init__.py`.
+    patched_markers_file = os.environ.pop("_PEX_PATCHED_MARKERS_FILE")
+    with open(patched_markers_file) as fp:
+        patched_markers = json.load(fp)
+
+    markers.default_environment = patched_markers.copy

pex/pip/foreign_platform/requires_python.py

@@ -0,0 +1,112 @@
+# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import
+
+import json
+import os
+
+# N.B.: The following environment variable is used by the Pex runtime to control Pip and must be
+# kept in-sync with `__init__.py`.
+with open(os.environ.pop("_PEX_PYTHON_VERSIONS_FILE")) as fp:
+    PYTHON_FULL_VERSIONS = json.load(fp)
+PYTHON_VERSIONS = sorted(set((version[0], version[1]) for version in PYTHON_FULL_VERSIONS))
+
+
+def patch():
+    # The pip-legacy-resolver patch.
+    from pip._internal.utils import packaging  # type: ignore[import]
+
+    if PYTHON_FULL_VERSIONS:
+        orig_check_requires_python = packaging.check_requires_python
+
+        def check_requires_python(requires_python, *_args, **_kw):
+            # Ensure any dependency we lock is compatible with the full interpreter range
+            # specified since we have no way to force Pip to backtrack and follow paths for any
+            # divergences. Most (all?) true divergences should be covered by forked environment
+            # markers.
+            return all(
+                orig_check_requires_python(requires_python, python_full_version)
+                for python_full_version in PYTHON_FULL_VERSIONS
+            )
+
+        packaging.check_requires_python = check_requires_python
+    else:
+        packaging.check_requires_python = lambda *_args, **_kw: True
+
+    # The pip-2020-resolver patch.
+    from pip._internal.resolution.resolvelib.candidates import (  # type: ignore[import]
+        RequiresPythonCandidate,
+    )
+    from pip._internal.resolution.resolvelib.requirements import (  # type: ignore[import]
+        RequiresPythonRequirement,
+    )
+
+    if PYTHON_FULL_VERSIONS:
+        orig_get_candidate_lookup = RequiresPythonRequirement.get_candidate_lookup
+        orig_is_satisfied_by = RequiresPythonRequirement.is_satisfied_by
+
+        # Ensure we do a proper, but minimal, comparison for Python versions. Previously we
+        # always tested all `Requires-Python` specifier sets against Python full versions.
+        # That can be pathologically slow (see:
+        # https://github.com/pantsbuild/pants/issues/14998); so we avoid using Python full
+        # versions unless the `Requires-Python` specifier set requires that data. In other
+        # words:
+        #
+        # Need full versions to evaluate properly:
+        # + Requires-Python: >=3.7.6
+        # + Requires-Python: >=3.7,!=3.7.6,<4
+        #
+        # Do not need full versions to evaluate properly:
+        # + Requires-Python: >=3.7,<4
+        # + Requires-Python: ==3.7.*
+        # + Requires-Python: >=3.6.0
+        #
+        def needs_full_versions(spec):
+            components = spec.version.split(".", 2)
+            if len(components) < 3:
+                return False
+            major_, minor_, patch = components
+            if spec.operator in ("<", "<=", ">", ">=") and patch == "0":
+                return False
+            return patch != "*"
+
+        def _py_versions(self):
+            if not hasattr(self, "__py_versions"):
+                self.__py_versions = (
+                    version
+                    for version in (
+                        PYTHON_FULL_VERSIONS
+                        if any(needs_full_versions(spec) for spec in self.specifier)
+                        else PYTHON_VERSIONS
+                    )
+                    if ".".join(map(str, version)) in self.specifier
+                )
+            return self.__py_versions
+
+        def get_candidate_lookup(self):
+            for py_version in self._py_versions():
+                delegate = RequiresPythonRequirement(
+                    self.specifier, RequiresPythonCandidate(py_version)
+                )
+                candidate_lookup = orig_get_candidate_lookup(delegate)
+                if candidate_lookup != (None, None):
+                    return candidate_lookup
+            return None, None
+
+        def is_satisfied_by(self, *_args, **_kw):
+            # Ensure any dependency we lock is compatible with the full interpreter range
+            # specified since we have no way to force Pip to backtrack and follow paths for any
+            # divergences. Most (all?) true divergences should be covered by forked environment
+            # markers.
+            return all(
+                orig_is_satisfied_by(self, RequiresPythonCandidate(py_version))
+                for py_version in self._py_versions()
+            )
+
+        RequiresPythonRequirement._py_versions = _py_versions
+        RequiresPythonRequirement.get_candidate_lookup = get_candidate_lookup
+        RequiresPythonRequirement.is_satisfied_by = is_satisfied_by
+    else:
+        RequiresPythonRequirement.get_candidate_lookup = lambda self: (self._candidate, None)
+        RequiresPythonRequirement.is_satisfied_by = lambda *_args, **_kw: True