diff --git a/pex/dist_metadata.py b/pex/dist_metadata.py index 79569f04b..a732059e2 100644 --- a/pex/dist_metadata.py +++ b/pex/dist_metadata.py @@ -7,11 +7,27 @@ import email from pex.third_party.packaging.specifiers import SpecifierSet -from pex.third_party.pkg_resources import DistInfoDistribution, Distribution +from pex.third_party.pkg_resources import DistInfoDistribution, Distribution, Requirement from pex.typing import TYPE_CHECKING if TYPE_CHECKING: - from typing import Optional + from typing import Dict, Iterator, Optional + + +_PKG_INFO_BY_DIST = {} # type: Dict[Distribution, Optional[email.message.Message]] + + +def _parse_pkg_info(dist): + # type: (Distribution) -> Optional[email.message.Message] + global _PKG_INFO_BY_DIST + if dist not in _PKG_INFO_BY_DIST: + if not dist.has_metadata(DistInfoDistribution.PKG_INFO): + pkg_info = None + else: + metadata = dist.get_metadata(DistInfoDistribution.PKG_INFO) + pkg_info = email.parser.Parser().parsestr(metadata) + _PKG_INFO_BY_DIST[dist] = pkg_info + return _PKG_INFO_BY_DIST[dist] def requires_python(dist): @@ -23,12 +39,40 @@ def requires_python(dist): :param dist: A distribution to check for `Python-Requires` metadata. :return: The required python version specifiers. """ - if not dist.has_metadata(DistInfoDistribution.PKG_INFO): + pkg_info = _parse_pkg_info(dist) + if pkg_info is None: return None - metadata = dist.get_metadata(DistInfoDistribution.PKG_INFO) - pkg_info = email.parser.Parser().parsestr(metadata) - python_requirement = pkg_info.get("Requires-Python") - if not python_requirement: + python_requirement = pkg_info.get("Requires-Python", None) + if python_requirement is None: return None return SpecifierSet(python_requirement) + + +def requires_dists( + dist, # type: Distribution + include_1_1_requires=True, # type: bool +): + # type: (...) -> Iterator[Requirement] + """Examines dist for and returns any declared requirements. + + Looks for `Requires-Dist` metadata and, optionally, the older `Requires` metadata if + `include_1_1_requires`. + + See: + + https://www.python.org/dev/peps/pep-0345/#requires-dist-multiple-use + + https://www.python.org/dev/peps/pep-0314/#requires-multiple-use + + :param dist: A distribution to check for requirement metadata. + :return: All requirements found. + """ + pkg_info = _parse_pkg_info(dist) + if pkg_info is None: + return + + for requires_dist in pkg_info.get_all("Requires-Dist", ()): + yield Requirement.parse(requires_dist) + + if include_1_1_requires: + for requires in pkg_info.get_all("Requires", ()): + yield Requirement.parse(requires) diff --git a/tests/test_dist_metadata.py b/tests/test_dist_metadata.py new file mode 100644 index 000000000..d19508d2c --- /dev/null +++ b/tests/test_dist_metadata.py @@ -0,0 +1,86 @@ +# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import os +from contextlib import contextmanager + +from pex.common import temporary_dir +from pex.dist_metadata import requires_dists, requires_python +from pex.pip import get_pip +from pex.third_party.packaging.specifiers import SpecifierSet +from pex.third_party.pkg_resources import Distribution, Requirement +from pex.util import DistributionHelper + + +def install_wheel( + wheel_path, # type: str + install_dir, # type: str +): + # type: (...) -> Distribution + get_pip().spawn_install_wheel(wheel=wheel_path, install_dir=install_dir).wait() + dist = DistributionHelper.distribution_from_path(install_dir) + assert dist is not None, "Could not load a distribution from {}".format(install_dir) + return dist + + +def example_package(name): + # type: (str) -> str + return os.path.join("./tests/example_packages", name) + + +@contextmanager +def example_distribution(name): + # type: (str) -> Distribution + wheel_path = example_package(name) + with temporary_dir() as install_dir: + yield install_wheel(wheel_path, install_dir=install_dir) + + +@contextmanager +def resolved_distribution(requirement): + # type: (str) -> Distribution + with temporary_dir() as td: + download_dir = os.path.join(td, "download") + get_pip().spawn_download_distributions( + download_dir=download_dir, requirements=[requirement], transitive=False + ).wait() + wheels = os.listdir(download_dir) + assert len(wheels) == 1, "Expected 1 wheel to be downloaded for {}".format(requirement) + wheel_path = os.path.join(download_dir, wheels[0]) + install_dir = os.path.join(td, "install") + yield install_wheel(wheel_path, install_dir=install_dir) + + +def test_requires_python(): + # type: () -> None + with resolved_distribution("pex==2.1.21") as dist: + assert SpecifierSet( + ">=2.7,<=3.9,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" + ) == requires_python(dist) + + +def test_requires_python_none(): + # type: () -> None + with example_distribution("aws_cfn_bootstrap-1.4-py2-none-any.whl") as dist: + assert requires_python(dist) is None + + +def test_requires_dists(): + # type: () -> None + with example_distribution("aws_cfn_bootstrap-1.4-py2-none-any.whl") as dist: + assert [ + Requirement.parse(req) + for req in ( + "python-daemon>=1.5.2,<2.0", + "pystache>=0.4.0", + "setuptools", + ) + ] == list(requires_dists(dist)) + + +def test_requires_dists_none(): + # type: () -> None + with example_distribution("MarkupSafe-1.0-cp27-cp27mu-linux_x86_64.whl") as dist: + assert [] == list(requires_dists(dist)) diff --git a/tests/test_environment.py b/tests/test_environment.py index 119fb5dc3..a70e57bdc 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -426,32 +426,50 @@ def test_present_but_empty_namespace_packages_metadata_does_not_warn(): assert_namespace_packages_warning("pycodestyle", "2.5.0", expected_warning=False) +def create_dist( + location, # type: str + version="1.0.0", # type: Optional[str] +): + # type: (...) -> Distribution + # N.B.: version must be set simply so that __hash__ / __eq__ work correctly in the + # `pex.dist_metadata` module. + return Distribution(location=location, version=version) + + @pytest.mark.parametrize( - ("wheel_filename", "wheel_is_linux"), + ("wheel_distribution", "wheel_is_linux"), [ pytest.param( - "llvmlite-0.29.0-cp35-cp35m-linux_x86_64.whl", True, id="without_build_tag_linux" + create_dist("llvmlite-0.29.0-cp35-cp35m-linux_x86_64.whl", "0.29.0"), + True, + id="without_build_tag_linux", ), pytest.param( - "llvmlite-0.29.0-1-cp35-cp35m-linux_x86_64.whl", True, id="with_build_tag_linux" + create_dist("llvmlite-0.29.0-1-cp35-cp35m-linux_x86_64.whl", "0.29.0"), + True, + id="with_build_tag_linux", ), pytest.param( - "llvmlite-0.29.0-cp35-cp35m-macosx_10.9_x86_64.whl", False, id="without_build_tag_osx" + create_dist("llvmlite-0.29.0-cp35-cp35m-macosx_10.9_x86_64.whl", "0.29.0"), + False, + id="without_build_tag_osx", ), pytest.param( - "llvmlite-0.29.0-1-cp35-cp35m-macosx_10.9_x86_64.whl", False, id="with_build_tag_osx" + create_dist("llvmlite-0.29.0-1-cp35-cp35m-macosx_10.9_x86_64.whl", "0.29.0"), + False, + id="with_build_tag_osx", ), ], ) def test_can_add_handles_optional_build_tag_in_wheel( - python_35_interpreter, wheel_filename, wheel_is_linux + python_35_interpreter, wheel_distribution, wheel_is_linux ): # type: (PythonInterpreter, str, bool) -> None pex_environment = PEXEnvironment( pex="", pex_info=PexInfo.default(python_35_interpreter), interpreter=python_35_interpreter ) native_wheel = IS_LINUX and wheel_is_linux - assert pex_environment.can_add(Distribution(wheel_filename)) is native_wheel + assert pex_environment.can_add(wheel_distribution) is native_wheel def test_can_add_handles_invalid_wheel_filename(python_35_interpreter): @@ -459,4 +477,4 @@ def test_can_add_handles_invalid_wheel_filename(python_35_interpreter): pex_environment = PEXEnvironment( pex="", pex_info=PexInfo.default(python_35_interpreter), interpreter=python_35_interpreter ) - assert pex_environment.can_add(Distribution("pep427-invalid.whl")) is False + assert pex_environment.can_add(create_dist("pep427-invalid.whl")) is False