Add a requires_dists function.

This is needed to support pex-tool#1020 and pex-tool#1108.

pex/dist_metadata.py

@@ -7,11 +7,27 @@
 import email
 
 from pex.third_party.packaging.specifiers import SpecifierSet
-from pex.third_party.pkg_resources import DistInfoDistribution, Distribution
+from pex.third_party.pkg_resources import DistInfoDistribution, Distribution, Requirement
 from pex.typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from typing import Optional
+    from typing import Dict, Iterator, Optional
+
+
+_PKG_INFO_BY_DIST = {}  # type: Dict[Distribution, Optional[email.message.Message]]
+
+
+def _parse_pkg_info(dist):
+    # type: (Distribution) -> Optional[email.message.Message]
+    global _PKG_INFO_BY_DIST
+    if dist not in _PKG_INFO_BY_DIST:
+        if not dist.has_metadata(DistInfoDistribution.PKG_INFO):
+            pkg_info = None
+        else:
+            metadata = dist.get_metadata(DistInfoDistribution.PKG_INFO)
+            pkg_info = email.parser.Parser().parsestr(metadata)
+        _PKG_INFO_BY_DIST[dist] = pkg_info
+    return _PKG_INFO_BY_DIST[dist]
 
 
 def requires_python(dist):
@@ -23,12 +39,40 @@ def requires_python(dist):
     :param dist: A distribution to check for `Python-Requires` metadata.
     :return: The required python version specifiers.
     """
-    if not dist.has_metadata(DistInfoDistribution.PKG_INFO):
+    pkg_info = _parse_pkg_info(dist)
+    if pkg_info is None:
         return None
 
-    metadata = dist.get_metadata(DistInfoDistribution.PKG_INFO)
-    pkg_info = email.parser.Parser().parsestr(metadata)
-    python_requirement = pkg_info.get("Requires-Python")
-    if not python_requirement:
+    python_requirement = pkg_info.get("Requires-Python", None)
+    if python_requirement is None:
         return None
     return SpecifierSet(python_requirement)
+
+
+def requires_dists(
+    dist,  # type: Distribution
+    include_1_1_requires=True,  # type: bool
+):
+    # type: (...) -> Iterator[Requirement]
+    """Examines dist for and returns any declared requirements.
+
+    Looks for `Requires-Dist` metadata and, optionally, the older `Requires` metadata if
+    `include_1_1_requires`.
+
+    See:
+    + https://www.python.org/dev/peps/pep-0345/#requires-dist-multiple-use
+    + https://www.python.org/dev/peps/pep-0314/#requires-multiple-use
+
+    :param dist: A distribution to check for requirement metadata.
+    :return: All requirements found.
+    """
+    pkg_info = _parse_pkg_info(dist)
+    if pkg_info is None:
+        return
+
+    for requires_dist in pkg_info.get_all("Requires-Dist", ()):
+        yield Requirement.parse(requires_dist)
+
+    if include_1_1_requires:
+        for requires in pkg_info.get_all("Requires", ()):
+            yield Requirement.parse(requires)

tests/test_dist_metadata.py

@@ -0,0 +1,86 @@
+# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import
+
+import os
+from contextlib import contextmanager
+
+from pex.common import temporary_dir
+from pex.dist_metadata import requires_dists, requires_python
+from pex.pip import get_pip
+from pex.third_party.packaging.specifiers import SpecifierSet
+from pex.third_party.pkg_resources import Distribution, Requirement
+from pex.util import DistributionHelper
+
+
+def install_wheel(
+    wheel_path,  # type: str
+    install_dir,  # type: str
+):
+    # type: (...) -> Distribution
+    get_pip().spawn_install_wheel(wheel=wheel_path, install_dir=install_dir).wait()
+    dist = DistributionHelper.distribution_from_path(install_dir)
+    assert dist is not None, "Could not load a distribution from {}".format(install_dir)
+    return dist
+
+
+def example_package(name):
+    # type: (str) -> str
+    return os.path.join("./tests/example_packages", name)
+
+
+@contextmanager
+def example_distribution(name):
+    # type: (str) -> Distribution
+    wheel_path = example_package(name)
+    with temporary_dir() as install_dir:
+        yield install_wheel(wheel_path, install_dir=install_dir)
+
+
+@contextmanager
+def resolved_distribution(requirement):
+    # type: (str) -> Distribution
+    with temporary_dir() as td:
+        download_dir = os.path.join(td, "download")
+        get_pip().spawn_download_distributions(
+            download_dir=download_dir, requirements=[requirement], transitive=False
+        ).wait()
+        wheels = os.listdir(download_dir)
+        assert len(wheels) == 1, "Expected 1 wheel to be downloaded for {}".format(requirement)
+        wheel_path = os.path.join(download_dir, wheels[0])
+        install_dir = os.path.join(td, "install")
+        yield install_wheel(wheel_path, install_dir=install_dir)
+
+
+def test_requires_python():
+    # type: () -> None
+    with resolved_distribution("pex==2.1.21") as dist:
+        assert SpecifierSet(
+            ">=2.7,<=3.9,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*"
+        ) == requires_python(dist)
+
+
+def test_requires_python_none():
+    # type: () -> None
+    with example_distribution("aws_cfn_bootstrap-1.4-py2-none-any.whl") as dist:
+        assert requires_python(dist) is None
+
+
+def test_requires_dists():
+    # type: () -> None
+    with example_distribution("aws_cfn_bootstrap-1.4-py2-none-any.whl") as dist:
+        assert [
+            Requirement.parse(req)
+            for req in (
+                "python-daemon>=1.5.2,<2.0",
+                "pystache>=0.4.0",
+                "setuptools",
+            )
+        ] == list(requires_dists(dist))
+
+
+def test_requires_dists_none():
+    # type: () -> None
+    with example_distribution("MarkupSafe-1.0-cp27-cp27mu-linux_x86_64.whl") as dist:
+        assert [] == list(requires_dists(dist))

tests/test_environment.py

@@ -426,37 +426,55 @@ def test_present_but_empty_namespace_packages_metadata_does_not_warn():
     assert_namespace_packages_warning("pycodestyle", "2.5.0", expected_warning=False)
 
 
+def create_dist(
+    location,  # type: str
+    version="1.0.0",  # type: Optional[str]
+):
+    # type: (...) -> Distribution
+    # N.B.: version must be set simply so that __hash__ / __eq__ work correctly in the
+    # `pex.dist_metadata` module.
+    return Distribution(location=location, version=version)
+
+
 @pytest.mark.parametrize(
-    ("wheel_filename", "wheel_is_linux"),
+    ("wheel_distribution", "wheel_is_linux"),
     [
         pytest.param(
-            "llvmlite-0.29.0-cp35-cp35m-linux_x86_64.whl", True, id="without_build_tag_linux"
+            create_dist("llvmlite-0.29.0-cp35-cp35m-linux_x86_64.whl", "0.29.0"),
+            True,
+            id="without_build_tag_linux",
         ),
         pytest.param(
-            "llvmlite-0.29.0-1-cp35-cp35m-linux_x86_64.whl", True, id="with_build_tag_linux"
+            create_dist("llvmlite-0.29.0-1-cp35-cp35m-linux_x86_64.whl", "0.29.0"),
+            True,
+            id="with_build_tag_linux",
         ),
         pytest.param(
-            "llvmlite-0.29.0-cp35-cp35m-macosx_10.9_x86_64.whl", False, id="without_build_tag_osx"
+            create_dist("llvmlite-0.29.0-cp35-cp35m-macosx_10.9_x86_64.whl", "0.29.0"),
+            False,
+            id="without_build_tag_osx",
         ),
         pytest.param(
-            "llvmlite-0.29.0-1-cp35-cp35m-macosx_10.9_x86_64.whl", False, id="with_build_tag_osx"
+            create_dist("llvmlite-0.29.0-1-cp35-cp35m-macosx_10.9_x86_64.whl", "0.29.0"),
+            False,
+            id="with_build_tag_osx",
         ),
     ],
 )
 def test_can_add_handles_optional_build_tag_in_wheel(
-    python_35_interpreter, wheel_filename, wheel_is_linux
+    python_35_interpreter, wheel_distribution, wheel_is_linux
 ):
     # type: (PythonInterpreter, str, bool) -> None
     pex_environment = PEXEnvironment(
         pex="", pex_info=PexInfo.default(python_35_interpreter), interpreter=python_35_interpreter
     )
     native_wheel = IS_LINUX and wheel_is_linux
-    assert pex_environment.can_add(Distribution(wheel_filename)) is native_wheel
+    assert pex_environment.can_add(wheel_distribution) is native_wheel
 
 
 def test_can_add_handles_invalid_wheel_filename(python_35_interpreter):
     # type: (PythonInterpreter) -> None
     pex_environment = PEXEnvironment(
         pex="", pex_info=PexInfo.default(python_35_interpreter), interpreter=python_35_interpreter
     )
-    assert pex_environment.can_add(Distribution("pep427-invalid.whl")) is False
+    assert pex_environment.can_add(create_dist("pep427-invalid.whl")) is False