From f27aabd4c9f5b2c8953e7c409574819928153c56 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 19 Jan 2021 19:48:32 -0800 Subject: [PATCH] Support resolving from a PEX file repository. Introduce a `--pex-repository` option to the Pex CLI to switch requirement resolution from using index servers and find-links repositories to using a local PEX file with pre-resolved requirements. This can be useful when a number of projects share a consistent resolve via a shared requirement file. You can resolve the full requirement file into a requirements PEX and then later resolve just the portions needed by each individual project from the fully resolved requirements PEX. Fixes #1108 --- pex/bin/pex.py | 78 ++++++++--- pex/distribution_target.py | 8 +- pex/environment.py | 147 ++++++++++++++------ pex/resolver.py | 134 ++++++++++++++++--- pex/util.py | 2 +- tests/test_resolver.py | 265 ++++++++++++++++++++++++++++++++++++- 6 files changed, 546 insertions(+), 88 deletions(-) diff --git a/pex/bin/pex.py b/pex/bin/pex.py index a23580d32..8a0a4fa49 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -30,7 +30,7 @@ from pex.pex_builder import PEXBuilder from pex.pip import ResolverVersion from pex.platforms import Platform -from pex.resolver import Unsatisfiable, parsed_platform, resolve_multi +from pex.resolver import Unsatisfiable, parsed_platform, resolve_from_pex, resolve_multi from pex.tracer import TRACER from pex.typing import TYPE_CHECKING from pex.variables import ENV, Variables @@ -175,6 +175,18 @@ def configure_clp_pex_resolution(parser): help="Additional cheeseshop indices to use to satisfy requirements.", ) + parser.add_argument( + "--pex-repository", + dest="pex_repository", + metavar="FILE", + default=None, + type=str, + help=( + "Resolve requirements from the given PEX file instead of from --index servers or " + "--find-links repos." + ), + ) + default_net_config = NetworkConfiguration.create() group.add_argument( @@ -912,26 +924,44 @@ def to_python_interpreter(full_path_or_basename): ) try: - resolveds = resolve_multi( - requirements=reqs, - requirement_files=options.requirement_files, - constraint_files=options.constraint_files, - allow_prereleases=options.allow_prereleases, - transitive=options.transitive, - interpreters=interpreters, - platforms=list(platforms), - indexes=indexes, - find_links=options.find_links, - resolver_version=ResolverVersion.for_value(options.resolver_version), - network_configuration=network_configuration, - cache=cache, - build=options.build, - use_wheel=options.use_wheel, - compile=options.compile, - manylinux=options.manylinux, - max_parallel_jobs=options.max_parallel_jobs, - ignore_errors=options.ignore_errors, - ) + if options.pex_repository: + with TRACER.timed( + "Resolving requirements from PEX {}.".format(options.pex_repository) + ): + resolveds = resolve_from_pex( + pex=options.pex_repository, + requirements=reqs, + requirement_files=options.requirement_files, + constraint_files=options.constraint_files, + network_configuration=network_configuration, + transitive=options.transitive, + interpreters=interpreters, + platforms=list(platforms), + manylinux=options.manylinux, + ignore_errors=options.ignore_errors, + ) + else: + with TRACER.timed("Resolving requirements."): + resolveds = resolve_multi( + requirements=reqs, + requirement_files=options.requirement_files, + constraint_files=options.constraint_files, + allow_prereleases=options.allow_prereleases, + transitive=options.transitive, + interpreters=interpreters, + platforms=list(platforms), + indexes=indexes, + find_links=options.find_links, + resolver_version=ResolverVersion.for_value(options.resolver_version), + network_configuration=network_configuration, + cache=cache, + build=options.build, + use_wheel=options.use_wheel, + compile=options.compile, + manylinux=options.manylinux, + max_parallel_jobs=options.max_parallel_jobs, + ignore_errors=options.ignore_errors, + ) for resolved_dist in resolveds: pex_builder.add_distribution(resolved_dist.distribution) @@ -1024,6 +1054,12 @@ def warn_ignore_pex_root(set_via): if options.python and options.interpreter_constraint: die('The "--python" and "--interpreter-constraint" options cannot be used together.') + if options.pex_repository and (options.indexes or options.find_links): + die( + 'The "--pex-repository" option cannot be used together with the "--index" or ' + '"--find-links" options.' + ) + with ENV.patch( PEX_VERBOSE=str(options.verbosity), PEX_ROOT=pex_root, TMPDIR=tmpdir ) as patched_env: diff --git a/pex/distribution_target.py b/pex/distribution_target.py index 81419c076..b213695f1 100644 --- a/pex/distribution_target.py +++ b/pex/distribution_target.py @@ -94,17 +94,21 @@ def requirement_applies( requirement, # type: Requirement extras=None, # type: Optional[Tuple[str, ...]] ): - # type: (...) -> bool + # type: (...) -> Optional[bool] """Determines if the given requirement applies to this distribution target. :param requirement: The requirement to evaluate. :param extras: Optional active extras. + :returns: `True` if the requirement definitely applies, `False` if it definitely does not + and `None` if it might apply but not enough information is at hand to determine + if it does apply. """ if requirement.marker is None: return True if self._platform is not None: - return True + # We can have no opinion for foreign platforms. + return None if not extras: # Provide an empty extra to safely evaluate the markers without matching any extra. diff --git a/pex/environment.py b/pex/environment.py index a3d146f3e..ec01373c9 100644 --- a/pex/environment.py +++ b/pex/environment.py @@ -31,6 +31,7 @@ Iterable, Iterator, List, + MutableMapping, Optional, Tuple, Union, @@ -73,6 +74,31 @@ def satisfies(self, requirement): return self.distribution in requirement +class _QualifiedRequirement(namedtuple("_QualifiedRequirement", ["requirement", "required"])): + @classmethod + def create( + cls, + requirement, # type: Requirement + required=True, # type: Optional[bool] + ): + # type: (...) -> _QualifiedRequirement + return cls(requirement=requirement, required=required) + + @property + def requirement(self): + # type: () -> Requirement + return cast(Requirement, super(_QualifiedRequirement, self).requirement) + + @property + def required(self): + # type: () -> Optional[bool] + return cast("Optional[bool]", super(_QualifiedRequirement, self).required) + + +if TYPE_CHECKING: + QualifiedRequirementOrNotFound = Union[_QualifiedRequirement, _DistributionNotFound] + + class _DistributionNotFound(namedtuple("_DistributionNotFound", ["requirement", "required_by"])): @classmethod def create( @@ -88,6 +114,28 @@ class ResolveError(Exception): """Indicates an error resolving requirements for a PEX.""" +class _RequirementKey(namedtuple("_RequirementKey", ["key", "extras"])): + @classmethod + def create(cls, requirement): + # type: (Requirement) -> _RequirementKey + return cls(requirement.key, frozenset(requirement.extras)) + + def satisfied_keys(self): + # type: () -> Iterator[_RequirementKey] + + # If we resolve a requirement with extras then we've satisfied resolves for the powerset of + # the extras. + # For example, if we resolve `cake[birthday,wedding]` then we satisfy resolves for: + # `cake[]` + # `cake[birthday]` + # `cake[wedding]` + # `cake[birthday,wedding]` + items = list(self.extras) + for size in range(len(items) + 1): + for combination_of_size in itertools.combinations(items, size): + yield _RequirementKey(self.key, frozenset(combination_of_size)) + + class PEXEnvironment(object): class _CachingZipImporter(object): class _CachingLoader(object): @@ -335,29 +383,37 @@ def _evaluate_marker( requirement, # type: Requirement extras=None, # type: Optional[Tuple[str, ...]] ): - # type: (...) -> bool + # type: (...) -> Optional[bool] applies = self._target.requirement_applies(requirement, extras=extras) - if not applies: + if applies is False: TRACER.log( "Skipping activation of `{}` due to environment marker de-selection".format( requirement - ) + ), + V=3, ) return applies def _resolve_requirement( self, requirement, # type: Requirement + resolved_dists_by_key, # type: MutableMapping[Distribution, _RequirementKey] + required, # type: Optional[bool] required_by=None, # type: Optional[Distribution] ): - # type: (...) -> Iterator[Union[Distribution, _DistributionNotFound]] + # type: (...) -> Iterator[_DistributionNotFound] + requirement_key = _RequirementKey.create(requirement) + if requirement_key in resolved_dists_by_key: + return + available_distributions = [ ranked_dist for ranked_dist in self._available_ranked_dists_by_key.get(requirement.key, []) if ranked_dist.satisfies(requirement) ] if not available_distributions: - yield _DistributionNotFound.create(requirement, required_by=required_by) + if required is True: + yield _DistributionNotFound.create(requirement, required_by=required_by) return resolved_distribution = sorted(available_distributions, reverse=True)[0].distribution @@ -373,7 +429,6 @@ def _resolve_requirement( V=9, ) - yield resolved_distribution for dep_requirement in dist_metadata.requires_dists(resolved_distribution): # A note regarding extras and why they're passed down one level (we don't pass / use # dep_requirement.extras for example): @@ -391,16 +446,24 @@ def _resolve_requirement( # We want to recurse and resolve all standard requests requirements but also those that # are part of the 'security' extra. In order to resolve the latter we need to include # the 'security' extra environment marker. - if not self._evaluate_marker(dep_requirement, extras=requirement.extras): + required = self._evaluate_marker(dep_requirement, extras=requirement.extras) + if required is False: continue - for dependency in self._resolve_requirement( - dep_requirement, required_by=resolved_distribution + for not_found in self._resolve_requirement( + dep_requirement, + resolved_dists_by_key, + required, + required_by=resolved_distribution, ): - yield dependency + yield not_found + + resolved_dists_by_key.update( + (key, resolved_distribution) for key in requirement_key.satisfied_keys() + ) def _root_requirements_iter(self, reqs): - # type: (Iterable[Requirement]) -> (Iterator[Union[Requirement, _DistributionNotFound]]) + # type: (Iterable[Requirement]) -> Iterator[QualifiedRequirementOrNotFound] # We want to pick one requirement for each key (required project) to then resolve # recursively. @@ -413,19 +476,20 @@ def _root_requirements_iter(self, reqs): # "setuptools==44.1.1; python_version<'3.6'", # "isort==5.6.4; python_version>='3.6'", # } - reqs_by_key = OrderedDict() # type: OrderedDict[str, List[Requirement]] + qualified_reqs_by_key = OrderedDict() # type: OrderedDict[str, List[_QualifiedRequirement]] for req in reqs: - if not self._evaluate_marker(req): + required = self._evaluate_marker(req) + if required is False: continue - requirements = reqs_by_key.get(req.key) + requirements = qualified_reqs_by_key.get(req.key) if requirements is None: - reqs_by_key[req.key] = requirements = [] - requirements.append(req) + qualified_reqs_by_key[req.key] = requirements = [] + requirements.append(_QualifiedRequirement.create(req, required=required)) # Next, from among the remaining applicable requirements for a given project, we want to # select the most tailored (highest ranked) available distribution. That distribution's # transitive requirements will later fill in the full resolve. - for key, requirements in reqs_by_key.items(): + for key, qualified_requirements in qualified_reqs_by_key.items(): ranked_dists = self._available_ranked_dists_by_key.get(key) if ranked_dists is None: # We've winnowed down reqs_by_key to just those requirements whose environment @@ -434,30 +498,34 @@ def _root_requirements_iter(self, reqs): "A distribution for {} could not be resolved in this environment.".format(key) ) candidates = [ - (ranked_dist, requirement) - for requirement in requirements + (ranked_dist, qualified_requirement) + for qualified_requirement in qualified_requirements for ranked_dist in ranked_dists - if ranked_dist.satisfies(requirement) + if ranked_dist.satisfies(qualified_requirement.requirement) ] if not candidates: - for requirement in requirements: - yield _DistributionNotFound.create(requirement) + for qualified_requirement in qualified_requirements: + yield _DistributionNotFound.create(qualified_requirement.requirement) continue - ranked_dist, requirement = sorted(candidates, key=lambda tup: tup[0], reverse=True)[0] + ranked_dist, qualified_requirement = sorted( + candidates, key=lambda tup: tup[0], reverse=True + )[0] if len(candidates) > 1: TRACER.log( "Selected {dist} via {req} and discarded {discarded}.".format( - req=requirement, + req=qualified_requirement.requirement, dist=ranked_dist.distribution, discarded=", ".join( - "{dist} via {req}".format(req=req, dist=ranked_dist.distribution) - for ranked_dist, req in candidates[1:] + "{dist} via {req}".format( + req=qualified_req.requirement, dist=ranked_dist.distribution + ) + for ranked_dist, qualified_req in candidates[1:] ), ), V=9, ) - yield requirement + yield qualified_requirement def resolve(self, reqs): # type: (Iterable[Requirement]) -> Iterable[Distribution] @@ -476,20 +544,19 @@ def record_unresolved(dist_not_found): if dist_not_found.required_by: requirers.add(dist_not_found.required_by) - resolveds = OrderedSet() # type: OrderedSet[Distribution] - - for req_or_not_found in self._root_requirements_iter(reqs): - if isinstance(req_or_not_found, _DistributionNotFound): - record_unresolved(req_or_not_found) + resolved_dists_by_key = OrderedDict() # type: OrderedDict[_RequirementKey, Distribution] + for qualified_req_or_not_found in self._root_requirements_iter(reqs): + if isinstance(qualified_req_or_not_found, _DistributionNotFound): + record_unresolved(qualified_req_or_not_found) continue - with TRACER.timed("Resolving {}".format(req_or_not_found), V=2): - for dist_or_not_found in self._resolve_requirement(req_or_not_found): - if isinstance(dist_or_not_found, _DistributionNotFound): - record_unresolved(dist_or_not_found) - continue - - resolveds.add(dist_or_not_found) + with TRACER.timed("Resolving {}".format(qualified_req_or_not_found.requirement), V=2): + for not_found in self._resolve_requirement( + requirement=qualified_req_or_not_found.requirement, + required=qualified_req_or_not_found.required, + resolved_dists_by_key=resolved_dists_by_key, + ): + record_unresolved(not_found) if unresolved_reqs: TRACER.log("Unresolved requirements:") @@ -545,7 +612,7 @@ def record_unresolved(dist_not_found): "{items}".format(pex=self._pex, platform=self._platform, items="\n".join(items)) ) - return resolveds + return OrderedSet(resolved_dists_by_key.values()) _NAMESPACE_PACKAGE_METADATA_RESOURCE = "namespace_packages.txt" diff --git a/pex/resolver.py b/pex/resolver.py index 547298eba..84eec2a4c 100644 --- a/pex/resolver.py +++ b/pex/resolver.py @@ -5,12 +5,14 @@ from __future__ import absolute_import import functools +import itertools import os import zipfile from collections import OrderedDict, defaultdict, namedtuple from pex.common import AtomicDirectory, atomic_directory, safe_mkdtemp from pex.distribution_target import DistributionTarget +from pex.environment import PEXEnvironment, ResolveError from pex.interpreter import PythonInterpreter from pex.jobs import Raise, SpawnedJob, execute_parallel from pex.network_configuration import NetworkConfiguration @@ -110,7 +112,7 @@ def parsed_platform(platform=None): class DownloadRequest(object): def __init__( self, - targets, # type: Iterable[DistributionTarget] + targets, # type: OrderedSet[DistributionTarget] direct_requirements, # type: Iterable[ParsedRequirement] requirements=None, # type: Optional[Iterable[str]] requirement_files=None, # type: Optional[Iterable[str]] @@ -123,7 +125,7 @@ def __init__( use_wheel=True, # type: bool ): # type: (...) -> None - self.targets = targets + self.targets = tuple(targets) self.direct_requirements = direct_requirements self.requirements = requirements self.requirement_files = requirement_files @@ -1111,24 +1113,12 @@ def resolve_multi( ) -def _download_internal( - direct_requirements, # type: Iterable[ParsedRequirement] - requirements=None, # type: Optional[Iterable[str]] - requirement_files=None, # type: Optional[Iterable[str]] - constraint_files=None, # type: Optional[Iterable[str]] - allow_prereleases=False, # type: bool - transitive=True, # type: bool +def _unique_targets( interpreters=None, # type: Optional[Iterable[PythonInterpreter]] - platforms=None, # type: Optional[Iterable[str]] - package_index_configuration=None, # type: Optional[PackageIndexConfiguration] - cache=None, # type: Optional[str] - build=True, # type: bool - use_wheel=True, # type: bool + platforms=None, # type: Optional[Iterable[Union[str, Platform]]] manylinux=None, # type: Optional[str] - dest=None, # type: Optional[str] - max_parallel_jobs=None, # type: Optional[int] ): - # type: (...) -> Tuple[List[BuildRequest], List[DownloadResult]] + # type: (...) -> OrderedSet[DistributionTarget] parsed_platforms = [parsed_platform(platform) for platform in platforms] if platforms else [] def iter_targets(): @@ -1154,11 +1144,31 @@ def iter_targets(): # Build for specific platforms. yield DistributionTarget.for_platform(platform, manylinux=manylinux) - # Only download for each target once. The download code assumes this unique targets optimization - # when spawning parallel downloads. - # TODO(John Sirois): centralize the de-deuping in the DownloadRequest constructor when we drop - # python 2.7 and move from namedtuples to dataclasses. - unique_targets = OrderedSet(iter_targets()) # type: Iterable[DistributionTarget] + return OrderedSet(iter_targets()) + + +def _download_internal( + direct_requirements, # type: Iterable[ParsedRequirement] + requirements=None, # type: Optional[Iterable[str]] + requirement_files=None, # type: Optional[Iterable[str]] + constraint_files=None, # type: Optional[Iterable[str]] + allow_prereleases=False, # type: bool + transitive=True, # type: bool + interpreters=None, # type: Optional[Iterable[PythonInterpreter]] + platforms=None, # type: Optional[Iterable[Union[str, Platform]]] + package_index_configuration=None, # type: Optional[PackageIndexConfiguration] + cache=None, # type: Optional[str] + build=True, # type: bool + use_wheel=True, # type: bool + manylinux=None, # type: Optional[str] + dest=None, # type: Optional[str] + max_parallel_jobs=None, # type: Optional[int] +): + # type: (...) -> Tuple[List[BuildRequest], List[DownloadResult]] + + unique_targets = _unique_targets( + interpreters=interpreters, platforms=platforms, manylinux=manylinux + ) download_request = DownloadRequest( targets=unique_targets, direct_requirements=direct_requirements, @@ -1382,3 +1392,83 @@ def install( ignore_errors=ignore_errors, max_parallel_jobs=max_parallel_jobs ) ) + + +def resolve_from_pex( + pex, # type: str + requirements=None, # type: Optional[Iterable[str]] + requirement_files=None, # type: Optional[Iterable[str]] + constraint_files=None, # type: Optional[Iterable[str]] + network_configuration=None, # type: Optional[NetworkConfiguration] + transitive=True, # type: bool + interpreters=None, # type: Optional[Iterable[PythonInterpreter]] + platforms=None, # type: Optional[Iterable[Union[str, Platform]]] + manylinux=None, # type: Optional[str] + ignore_errors=False, # type: bool +): + # type: (...) -> List[ResolvedDistribution] + + direct_requirements = _parse_reqs(requirements, requirement_files, network_configuration) + direct_requirements_by_key = OrderedDict() + for direct_requirement in direct_requirements: + if isinstance(direct_requirement, LocalProjectRequirement): + raise Untranslatable( + "Cannot resolve local project {path} from the PEX repository {pex}.".format( + path=direct_requirement.path, pex=pex + ) + ) + direct_requirements_by_key[ + direct_requirement.requirement.key + ] = direct_requirement.requirement + + constraints_by_key = defaultdict(list) # type: DefaultDict[str, List[Constraint]] + if not ignore_errors and (requirement_files or constraint_files): + fetcher = URLFetcher(network_configuration=network_configuration) + for location, is_constraints in itertools.chain( + ((requirement_file, False) for requirement_file in requirement_files or ()), + ((constraint_file, True) for constraint_file in constraint_files or ()), + ): + for parsed_item in parse_requirement_file( + location, is_constraints=is_constraints, fetcher=fetcher + ): + if isinstance(parsed_item, Constraint): + constraints_by_key[parsed_item.requirement.key].append(parsed_item) + + all_reqs = direct_requirements_by_key.values() + unique_targets = _unique_targets( + interpreters=interpreters, platforms=platforms, manylinux=manylinux + ) + resolved_distributions = OrderedSet() # type: OrderedSet[ResolvedDistribution] + for target in unique_targets: + pex_env = PEXEnvironment(pex, target=target) + try: + distributions = pex_env.resolve(all_reqs) + except ResolveError as e: + raise Unsatisfiable(str(e)) + + for distribution in distributions: + direct_requirement = direct_requirements_by_key.get(distribution.key, None) + if not transitive and not direct_requirement: + continue + + unmet_constraints = [ + constraint + for constraint in constraints_by_key.get(distribution.key, ()) + if distribution not in constraint.requirement + ] + if unmet_constraints: + raise Unsatisfiable( + "The following constraints were not satisfied by {dist} resolved from " + "{pex}:\n{constraints}".format( + dist=distribution.location, + pex=pex, + constraints="\n".join(map(str, unmet_constraints)), + ) + ) + + resolved_distributions.add( + ResolvedDistribution.create( + target, distribution, direct_requirement=direct_requirement + ) + ) + return list(resolved_distributions) diff --git a/pex/util.py b/pex/util.py index 3963358f6..25fb1ac6e 100644 --- a/pex/util.py +++ b/pex/util.py @@ -181,7 +181,7 @@ def cache_distribution(cls, zf, source, target_dir): """ with atomic_directory(target_dir, source=source, exclusive=True) as target_dir_tmp: if target_dir_tmp is None: - TRACER.log("Using cached {}".format(target_dir)) + TRACER.log("Using cached {}".format(target_dir), V=3) else: with TRACER.timed("Caching {}:{} in {}".format(zf.filename, source, target_dir)): for name in zf.namelist(): diff --git a/tests/test_resolver.py b/tests/test_resolver.py index cb08860ff..f2d729e52 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -14,6 +14,9 @@ from pex.compatibility import nested from pex.distribution_target import DistributionTarget from pex.interpreter import PythonInterpreter, spawn_python_job +from pex.pex_builder import PEXBuilder +from pex.pex_info import PexInfo +from pex.platforms import Platform from pex.resolver import ( InstalledDistribution, IntegrityError, @@ -21,6 +24,7 @@ Unsatisfiable, download, install, + resolve_from_pex, resolve_multi, ) from pex.testing import ( @@ -36,10 +40,10 @@ make_source_dir, ) from pex.third_party.pkg_resources import Requirement -from pex.typing import TYPE_CHECKING +from pex.typing import TYPE_CHECKING, cast if TYPE_CHECKING: - from typing import Any, List, Union + from typing import Any, DefaultDict, Iterable, List, Optional, Set, Union def create_sdist(**kwargs): @@ -528,3 +532,260 @@ def test_resolve_arbitrary_equality_issues_940(): ) assert [("===", "1.0.2-fba4511")] == requirement.specs assert requirement.marker is None + + +def create_pex_repository( + interpreters=None, # type: Optional[Iterable[PythonInterpreter]] + platforms=None, # type: Optional[Iterable[Platform]] + requirements=None, # type: Optional[Iterable[str]] + requirement_files=None, # type: Optional[Iterable[str]] + constraint_files=None, # type: Optional[Iterable[str]] +): + # type: (...) -> str + pex_builder = PEXBuilder() + for resolved_dist in resolve_multi( + interpreters=interpreters, + platforms=platforms, + requirements=requirements, + requirement_files=requirement_files, + constraint_files=constraint_files, + ): + pex_builder.add_distribution(resolved_dist.distribution) + if resolved_dist.direct_requirement: + pex_builder.add_requirement(resolved_dist.direct_requirement) + pex_builder.freeze() + return cast(str, pex_builder.path()) + + +def create_constraints_file(*requirements): + # type: (*str) -> str + constraints_file = os.path.join(safe_mkdtemp(), "constraints.txt") + with open(constraints_file, "w") as fp: + for requirement in requirements: + fp.write(requirement + os.linesep) + return constraints_file + + +@pytest.fixture(scope="module") +def py27(): + # type: () -> PythonInterpreter + return PythonInterpreter.from_binary(ensure_python_interpreter(PY27)) + + +@pytest.fixture(scope="module") +def py36(): + # type: () -> PythonInterpreter + return PythonInterpreter.from_binary(ensure_python_interpreter(PY36)) + + +@pytest.fixture(scope="module") +def macosx(): + # type: () -> Platform + return Platform.create("macosx-10.13-x86_64-cp-36-m") + + +@pytest.fixture(scope="module") +def linux(): + # type: () -> Platform + return Platform.create("linux-x86_64-cp-36-m") + + +@pytest.fixture(scope="module") +def foreign_platform( + macosx, # type: Platform + linux, # type: Platform +): + # type: (...) -> Platform + return macosx if IS_LINUX else linux + + +@pytest.fixture(scope="module") +def pex_repository(py27, py36, foreign_platform): + # type () -> str + + # N.B.: requests 2.25.1 constrains urllib3 to <1.27,>=1.21.1 and pick 1.26.2 on its own as of + # this writing. + constraints_file = create_constraints_file("urllib3==1.26.1") + + return create_pex_repository( + interpreters=[py27, py36], + platforms=[foreign_platform], + requirements=["requests[security,socks]==2.25.1"], + constraint_files=[constraints_file], + ) + + +def test_resolve_from_pex( + pex_repository, # type: str + py27, # type: PythonInterpreter + py36, # type: PythonInterpreter + foreign_platform, # type: Platform +): + # type: (...) -> None + pex_info = PexInfo.from_pex(pex_repository) + direct_requirements = pex_info.requirements + assert 1 == len(direct_requirements) + + resolved_distributions = resolve_from_pex( + pex=pex_repository, + requirements=direct_requirements, + interpreters=[py27, py36], + platforms=[foreign_platform], + ) + + distribution_locations_by_key = defaultdict(set) # type: DefaultDict[str, Set[str]] + for resolved_distribution in resolved_distributions: + distribution_locations_by_key[resolved_distribution.distribution.key].add( + resolved_distribution.distribution.location + ) + + assert { + os.path.basename(location) + for locations in distribution_locations_by_key.values() + for location in locations + } == set(pex_info.distributions.keys()), ( + "Expected to resolve the same full set of distributions from the pex repository as make " + "it up when using the same requirements." + ) + + assert "requests" in distribution_locations_by_key + assert 1 == len(distribution_locations_by_key["requests"]) + + assert "pysocks" in distribution_locations_by_key + assert 2 == len(distribution_locations_by_key["pysocks"]), ( + "PySocks has a non-platform-specific Python 2.7 distribution and a non-platform-specific " + "Python 3 distribution; so we expect to resolve two distributions - one covering " + "Python 2.7 and one covering local Python 3.6 and our cp36 foreign platform." + ) + + assert "cryptography" in distribution_locations_by_key + assert 3 == len(distribution_locations_by_key["cryptography"]), ( + "The cryptography requirement of the security extra is platform specific; so we expect a " + "unique distribution to be resolved for each of the three distributin targets" + ) + + +def test_resolve_from_pex_subset( + pex_repository, # type: str + foreign_platform, # type: Platform +): + # type: (...) -> None + + resolved_distributions = resolve_from_pex( + pex=pex_repository, + requirements=["cffi"], + platforms=[foreign_platform], + ) + + assert {"cffi", "pycparser"} == { + resolved_distribution.distribution.key for resolved_distribution in resolved_distributions + } + + +def test_resolve_from_pex_not_found( + pex_repository, # type: str + py36, # type: PythonInterpreter +): + # type: (...) -> None + + with pytest.raises(Unsatisfiable) as exec_info: + resolve_from_pex( + pex=pex_repository, + requirements=["pex"], + interpreters=[py36], + ) + assert "A distribution for pex could not be resolved in this environment." in str( + exec_info.value + ) + + with pytest.raises(Unsatisfiable) as exec_info: + resolve_from_pex( + pex=pex_repository, + requirements=["requests==1.0.0"], + interpreters=[py36], + ) + message = str(exec_info.value) + assert ( + "Failed to resolve requirements from PEX environment @ {}".format(pex_repository) in message + ) + assert "Needed {} compatible dependencies for:".format(py36.platform) in message + assert "1: requests==1.0.0" in message + assert "But this pex only contains:" in message + assert "requests-2.25.1-py2.py3-none-any.whl" in message + + +def test_resolve_from_pex_intransitive( + pex_repository, # type: str + py27, # type: PythonInterpreter + py36, # type: PythonInterpreter + foreign_platform, # type: Platform +): + # type: (...) -> None + + resolved_distributions = resolve_from_pex( + pex=pex_repository, + requirements=["requests"], + transitive=False, + interpreters=[py27, py36], + platforms=[foreign_platform], + ) + assert 3 == len( + resolved_distributions + ), "Expected one resolved distribution per distribution target." + assert 1 == len( + frozenset( + resolved_distribution.distribution.location + for resolved_distribution in resolved_distributions + ) + ), ( + "Expected one underlying resolved universal distribution usable on Linux and macOs by " + "both Python 2.7 and Python 3.6." + ) + for resolved_distribution in resolved_distributions: + assert ( + Requirement.parse("requests==2.25.1") + == resolved_distribution.distribution.as_requirement() + ) + assert Requirement.parse("requests") == resolved_distribution.direct_requirement + + +def test_resolve_from_pex_constraints( + pex_repository, # type: str + py27, # type: PythonInterpreter +): + # type: (...) -> None + + with pytest.raises(Unsatisfiable) as exec_info: + resolve_from_pex( + pex=pex_repository, + requirements=["requests"], + constraint_files=[create_constraints_file("urllib3==1.26.2")], + interpreters=[py27], + ) + message = str(exec_info.value) + assert "The following constraints were not satisfied by " in message + assert " resolved from {}:".format(pex_repository) in message + assert "urllib3==1.26.2" in message + + +def test_resolve_from_pex_ignore_errors( + pex_repository, # type: str + py27, # type: PythonInterpreter +): + # type: (...) -> None + + # See test_resolve_from_pex_constraints above for the failure this would otherwise cause. + resolved_distributions = resolve_from_pex( + pex=pex_repository, + requirements=["requests"], + constraint_files=[create_constraints_file("urllib3==1.26.2")], + interpreters=[py27], + ignore_errors=True, + ) + resolved_distributions_by_key = { + resolved_distribution.distribution.key: resolved_distribution.distribution.as_requirement() + for resolved_distribution in resolved_distributions + } + assert len(resolved_distributions_by_key) > 1, "We should resolve at least requests and urllib3" + assert "requests" in resolved_distributions_by_key + assert Requirement.parse("urllib3==1.26.1") == resolved_distributions_by_key["urllib3"]