diff --git a/pex/bin/pex.py b/pex/bin/pex.py index a22fdd70c..89c82f58c 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -53,9 +53,8 @@ from pex.resolve.resolver_configuration import ( LockRepositoryConfiguration, PexRepositoryConfiguration, - PreResolvedConfiguration, + PipConfiguration, ) -from pex.resolve.resolver_options import create_pip_configuration from pex.resolve.resolvers import Unsatisfiable, sorted_requirements from pex.result import Error, ResultError, catch, try_ from pex.scie import ScieConfiguration @@ -1014,20 +1013,11 @@ def build_pex( DependencyConfiguration.from_pex_info(requirements_pex_info) ) - if isinstance(resolver_configuration, (LockRepositoryConfiguration, PreResolvedConfiguration)): - pip_configuration = resolver_configuration.pip_configuration - elif isinstance(resolver_configuration, PexRepositoryConfiguration): - # TODO(John Sirois): Consider finding a way to support custom --index and --find-links in - # this case. I.E.: I use a corporate index to build a PEX repository and now I want to - # build a --project PEX whose pyproject.toml build-system.requires should be resolved from - # that corporate index. - pip_configuration = try_( - finalize_resolve_config( - create_pip_configuration(options), targets=targets, context="--project building" - ) - ) - else: - pip_configuration = resolver_configuration + pip_configuration = ( + resolver_configuration + if isinstance(resolver_configuration, PipConfiguration) + else resolver_configuration.pip_configuration + ) project_dependencies = OrderedSet() # type: OrderedSet[Requirement] with TRACER.timed( @@ -1157,11 +1147,14 @@ def _compatible_with_current_platform(interpreter, platforms): return current_platforms.intersection(platforms) -def configure_requirements_and_targets(options): - # type: (Namespace) -> Union[Tuple[RequirementConfiguration, InterpreterConstraints, Targets], Error] +def configure_requirements_and_targets( + options, # type: Namespace + pip_configuration, # type: PipConfiguration +): + # type: (...) -> Union[Tuple[RequirementConfiguration, InterpreterConstraints, Targets], Error] requirement_configuration = requirement_options.configure(options) - target_config = target_options.configure(options) + target_config = target_options.configure(options, pip_configuration=pip_configuration) script_metadata = ScriptMetadata() if options.executable and options.enable_script_metadata: @@ -1260,15 +1253,22 @@ def main(args=None): try: with global_environment(options) as env: - requirement_configuration, interpreter_constraints, targets = try_( - configure_requirements_and_targets(options) - ) - try: resolver_configuration = resolver_options.configure(options) except resolver_options.InvalidConfigurationError as e: die(str(e)) + requirement_configuration, interpreter_constraints, targets = try_( + configure_requirements_and_targets( + options, + pip_configuration=( + resolver_configuration + if isinstance(resolver_configuration, PipConfiguration) + else resolver_configuration.pip_configuration + ), + ) + ) + resolver_configuration = try_( finalize_resolve_config(resolver_configuration, targets, context="PEX building") ) diff --git a/pex/cli/commands/lock.py b/pex/cli/commands/lock.py index d8224d295..2884d2cee 100644 --- a/pex/cli/commands/lock.py +++ b/pex/cli/commands/lock.py @@ -549,7 +549,12 @@ def _add_export_arguments( cls.add_output_option(export_parser, entity="lock") cls._add_target_options(export_parser) resolver_options_parser = cls._create_resolver_options_group(export_parser) - resolver_options.register_network_options(resolver_options_parser) + resolver_options.register( + resolver_options_parser, + include_pex_repository=False, + include_lock=False, + include_pre_resolved=False, + ) @classmethod def _add_export_subset_arguments(cls, export_subset_parser): @@ -652,11 +657,12 @@ def _add_update_arguments(cls, update_parser): cls.add_json_options(update_parser, entity="lock", include_switch=False) cls._add_target_options(update_parser) resolver_options_parser = cls._create_resolver_options_group(update_parser) - resolver_options.register_repos_options(resolver_options_parser) - resolver_options.register_network_options(resolver_options_parser) - resolver_options.register_max_jobs_option(resolver_options_parser) - resolver_options.register_use_pip_config(resolver_options_parser) - resolver_options.register_pip_log(resolver_options_parser) + resolver_options.register( + resolver_options_parser, + include_pex_repository=False, + include_lock=False, + include_pre_resolved=False, + ) @classmethod def add_update_lock_options( @@ -792,7 +798,9 @@ def _resolve_targets( ): # type: (...) -> Union[Targets, Error] - target_config = target_configuration or target_options.configure(self.options) + target_config = target_configuration or target_options.configure( + self.options, pip_configuration=resolver_options.create_pip_configuration(self.options) + ) if style is not LockStyle.UNIVERSAL: return target_config.resolve_targets() @@ -812,7 +820,6 @@ def _resolve_targets( return Targets( platforms=target_config.platforms, complete_platforms=target_config.complete_platforms, - assume_manylinux=target_config.assume_manylinux, ) try: @@ -827,7 +834,6 @@ def _resolve_targets( interpreters=(interpreter,), platforms=target_config.platforms, complete_platforms=target_config.complete_platforms, - assume_manylinux=target_config.assume_manylinux, ) def _gather_requirements( @@ -860,7 +866,11 @@ def _gather_requirements( def _create(self): # type: () -> Result - target_configuration = target_options.configure(self.options) + + pip_configuration = resolver_options.create_pip_configuration(self.options) + target_configuration = target_options.configure( + self.options, pip_configuration=pip_configuration + ) if self.options.style == LockStyle.UNIVERSAL: lock_configuration = LockConfiguration( style=LockStyle.UNIVERSAL, @@ -888,7 +898,7 @@ def _create(self): ) pip_configuration = try_( finalize_resolve_config( - resolver_configuration=resolver_options.create_pip_configuration(self.options), + resolver_configuration=pip_configuration, targets=targets, context="lock creation", ) @@ -953,19 +963,21 @@ def _export(self, requirement_configuration=RequirementConfiguration()): ) lockfile_path, lock_file = self._load_lockfile() - targets = target_options.configure(self.options).resolve_targets() + pip_configuration = resolver_options.create_pip_configuration(self.options) + targets = target_options.configure( + self.options, pip_configuration=pip_configuration + ).resolve_targets() target = targets.require_unique_target( purpose="exporting a lock in the {pip!r} format".format(pip=ExportFormat.PIP) ) - network_configuration = resolver_options.create_network_configuration(self.options) with TRACER.timed("Selecting locks for {target}".format(target=target)): subset_result = try_( subset( targets=targets, lock=lock_file, requirement_configuration=requirement_configuration, - network_configuration=network_configuration, + network_configuration=pip_configuration.network_configuration, build_configuration=lock_file.build_configuration(), transitive=lock_file.transitive, include_all_matches=True, @@ -1079,18 +1091,20 @@ def _create_lock_update_request( ): # type: (...) -> Union[LockUpdateRequest, Error] - network_configuration = resolver_options.create_network_configuration(self.options) + pip_configuration = resolver_options.create_pip_configuration(self.options) lock_updater = LockUpdater.create( lock_file=lock_file, - repos_configuration=resolver_options.create_repos_configuration(self.options), - network_configuration=network_configuration, - max_jobs=resolver_options.get_max_jobs_value(self.options), - use_pip_config=resolver_options.get_use_pip_config_value(self.options), + repos_configuration=pip_configuration.repos_configuration, + network_configuration=pip_configuration.network_configuration, + max_jobs=pip_configuration.max_jobs, + use_pip_config=pip_configuration.use_pip_config, dependency_configuration=dependency_config, pip_log=resolver_options.get_pip_log(self.options), ) - target_configuration = target_options.configure(self.options) + target_configuration = target_options.configure( + self.options, pip_configuration=pip_configuration + ) targets = try_( self._resolve_targets( action="updating", style=lock_file.style, target_configuration=target_configuration @@ -1146,7 +1160,7 @@ def _create_lock_update_request( subset( targets=targets, lock=lock_file, - network_configuration=network_configuration, + network_configuration=pip_configuration.network_configuration, build_configuration=lock_file.build_configuration(), transitive=lock_file.transitive, ) @@ -1498,7 +1512,9 @@ def _sync(self): pip_configuration = resolver_configuration.pip_configuration dependency_config = dependency_configuration.configure(self.options) - target_configuration = target_options.configure(self.options) + target_configuration = target_options.configure( + self.options, pip_configuration=pip_configuration + ) if self.options.style == LockStyle.UNIVERSAL: lock_configuration = LockConfiguration( style=LockStyle.UNIVERSAL, @@ -1613,7 +1629,9 @@ def _sync(self): else PythonInterpreter.from_env(self.options.venv_python) ) else: - targets = target_options.configure(self.options).resolve_targets() + targets = target_options.configure( + self.options, pip_configuration=pip_configuration + ).resolve_targets() interpreters = [ target.get_interpreter() for target in targets.unique_targets() diff --git a/pex/cli/commands/venv.py b/pex/cli/commands/venv.py index 339136f92..2156400ec 100644 --- a/pex/cli/commands/venv.py +++ b/pex/cli/commands/venv.py @@ -21,6 +21,7 @@ from pex.resolve.resolver_configuration import ( LockRepositoryConfiguration, PexRepositoryConfiguration, + PipConfiguration, ) from pex.result import Error, Ok, Result, try_ from pex.targets import LocalInterpreter, Target, Targets @@ -200,7 +201,15 @@ def _inspect(self): def _create(self): # type: () -> Result - targets = target_options.configure(self.options).resolve_targets() + resolver_configuration = resolver_options.configure(self.options) + targets = target_options.configure( + self.options, + pip_configuration=( + resolver_configuration + if isinstance(resolver_configuration, PipConfiguration) + else resolver_configuration.pip_configuration + ), + ).resolve_targets() installer_configuration = installer_options.configure(self.options) dest_dir = ( @@ -266,7 +275,6 @@ def _create(self): ) requirement_configuration = requirement_options.configure(self.options) - resolver_configuration = resolver_options.configure(self.options) with TRACER.timed("Resolving distributions"): resolved = configured_resolve.resolve( targets=targets, diff --git a/pex/interpreter.py b/pex/interpreter.py index 07c9eb925..2df8411b7 100644 --- a/pex/interpreter.py +++ b/pex/interpreter.py @@ -545,9 +545,10 @@ def iter_supported_platforms(self): version=self.version_str, version_info=self.version, abi=self.abi_tag, + supported_tags=self._supported_tags, ) - for tag in self._supported_tags: - yield Platform.from_tag(tag) + for index in range(len(self._supported_tags)): + yield Platform.from_tags(self._supported_tags[index:]) def binary_name(self, version_components=2): # type: (int) -> str diff --git a/pex/pep_425.py b/pex/pep_425.py index a32bf9ec3..ab1c1e1c2 100644 --- a/pex/pep_425.py +++ b/pex/pep_425.py @@ -84,7 +84,7 @@ def from_strings(cls, tags): return cls(tags=tuple(itertools.chain.from_iterable(parse_tag(tag) for tag in tags))) _tags = attr.ib(converter=_prepare_tags) # type: Tuple[Tag, ...] - __rankings = attr.ib(eq=False, factory=dict) # type: MutableMapping[Tag, TagRank] + _rankings = attr.ib(eq=False, factory=dict) # type: MutableMapping[Tag, TagRank] @_tags.validator def _validate_tags( @@ -120,11 +120,11 @@ def to_string_list(self): return [str(tag) for tag in self._tags] @property - def _rankings(self): + def __rankings(self): # type: () -> Mapping[Tag, TagRank] - if not self.__rankings: - self.__rankings.update(TagRank.ranked(self._tags)) - return self.__rankings + if not self._rankings: + self._rankings.update(TagRank.ranked(self._tags)) + return self._rankings @property def lowest_rank(self): @@ -133,7 +133,7 @@ def lowest_rank(self): def rank(self, tag): # type: (Tag) -> Optional[TagRank] - return self._rankings.get(tag) + return self.__rankings.get(tag) def best_match(self, tags): # type: (Iterable[Tag]) -> Optional[RankedTag] @@ -160,18 +160,30 @@ def __getitem__(self, index): # type: (int) -> Tag pass + # MyPy claims this overload collides with the one below even though slice / Tag are disjoint + # input types and CompatibilityTags / TagRank are disjoint return types; thus the ignore[misc]. + @overload + def __getitem__(self, slice_): # type: ignore[misc] + # type: (slice) -> CompatibilityTags + pass + @overload def __getitem__(self, tag): # type: (Tag) -> TagRank pass - def __getitem__(self, index_or_tag): - # type: (Union[int, Tag]) -> Union[Tag, TagRank] + def __getitem__(self, index_or_slice_or_tag): + # type: (Union[int, slice, Tag]) -> Union[Tag, CompatibilityTags, TagRank] """Retrieve tag by its rank or a tags rank. Ranks are 0-based with the 0-rank tag being the most specific (best match). """ - if isinstance(index_or_tag, Tag): - return self._rankings[index_or_tag] + if isinstance(index_or_slice_or_tag, Tag): + return self.__rankings[index_or_slice_or_tag] + elif isinstance(index_or_slice_or_tag, slice): + tags = self._tags[index_or_slice_or_tag] + return CompatibilityTags( + tags=tags, rankings={tag: self.__rankings[tag] for tag in tags} + ) else: - return self._tags[index_or_tag] + return self._tags[index_or_slice_or_tag] diff --git a/pex/pip/foreign_platform/__init__.py b/pex/pip/foreign_platform/__init__.py index afbcee723..69eb9a357 100644 --- a/pex/pip/foreign_platform/__init__.py +++ b/pex/pip/foreign_platform/__init__.py @@ -10,7 +10,7 @@ from pex.interpreter_constraints import iter_compatible_versions from pex.pep_425 import CompatibilityTags from pex.pip.download_observer import DownloadObserver, Patch, PatchSet -from pex.platforms import Platform +from pex.platforms import PlatformSpec from pex.targets import AbbreviatedPlatform, CompletePlatform, Target from pex.tracer import TRACER from pex.typing import TYPE_CHECKING @@ -20,12 +20,12 @@ def iter_platform_args( - platform, # type: Platform + platform_spec, # type: PlatformSpec manylinux=None, # type: Optional[str] ): # type: (...) -> Iterator[str] - plat = platform.platform + platform = platform_spec.platform # N.B.: Pip supports passing multiple --platform and --abi. We pass multiple --platform to # support the following use case 1st surfaced by Twitter in 2018: # @@ -41,21 +41,21 @@ def iter_platform_args( # consume both its own custom-built wheels as well as other manylinux-compliant wheels in # the same application, it needs to advertise that the target machine supports both # `linux_x86_64` wheels and `manylinux2014_x86_64` wheels (for example). - if manylinux and plat.startswith("linux"): + if manylinux and platform.startswith("linux"): yield "--platform" - yield plat.replace("linux", manylinux, 1) + yield platform.replace("linux", manylinux, 1) yield "--platform" - yield plat + yield platform yield "--implementation" - yield platform.impl + yield platform_spec.impl yield "--python-version" - yield platform.version + yield platform_spec.version yield "--abi" - yield platform.abi + yield platform_spec.abi class EvaluationEnvironment(dict): diff --git a/pex/pip/tool.py b/pex/pip/tool.py index dbe925cf7..38874e3da 100644 --- a/pex/pip/tool.py +++ b/pex/pip/tool.py @@ -28,7 +28,7 @@ from pex.pip.log_analyzer import ErrorAnalyzer, ErrorMessage, LogAnalyzer, LogScrapeJob from pex.pip.tailer import Tailer from pex.pip.version import PipVersion, PipVersionValue -from pex.platforms import Platform +from pex.platforms import PlatformSpec from pex.resolve.resolver_configuration import ( BuildConfiguration, ReposConfiguration, @@ -698,7 +698,7 @@ def spawn_build_wheels( def spawn_debug( self, - platform, # type: Platform + platform_spec, # type: PlatformSpec manylinux=None, # type: Optional[str] log=None, # type: Optional[str] ): @@ -713,7 +713,9 @@ def spawn_debug( # only if the Pip command fails, which is what we want. debug_command = ["debug"] - debug_command.extend(foreign_platform.iter_platform_args(platform, manylinux=manylinux)) + debug_command.extend( + foreign_platform.iter_platform_args(platform_spec, manylinux=manylinux) + ) return self._spawn_pip_isolated_job( debug_command, log=log, pip_verbosity=1, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) diff --git a/pex/platforms.py b/pex/platforms.py index 7530f6837..dfd3b4d19 100644 --- a/pex/platforms.py +++ b/pex/platforms.py @@ -3,22 +3,14 @@ from __future__ import absolute_import -import json -import os -import re from textwrap import dedent -from pex import compatibility -from pex.atomic_directory import atomic_directory -from pex.cache.dirs import CacheDir -from pex.common import safe_open, safe_rmtree from pex.pep_425 import CompatibilityTags from pex.third_party.packaging import tags -from pex.tracer import TRACER from pex.typing import TYPE_CHECKING, cast if TYPE_CHECKING: - from typing import Any, Dict, Iterator, List, Optional, Tuple, Union + from typing import Tuple, Union import attr # vendor:skip @@ -33,11 +25,10 @@ def _normalize_platform(platform): @attr.s(frozen=True) -class Platform(object): - """Represents a target platform and it's extended interpreter compatibility tags (e.g. - implementation, version and ABI).""" +class PlatformSpec(object): + """Represents a target Python platform, implementation, version and ABI.""" - class InvalidPlatformError(Exception): + class InvalidSpecError(Exception): """Indicates an invalid platform string.""" @classmethod @@ -92,17 +83,14 @@ def create(cls, platform, cause=None): SEP = "-" @classmethod - def create(cls, platform): - # type: (Union[str, Platform]) -> Platform - if isinstance(platform, Platform): - return platform - - platform_components = platform.rsplit(cls.SEP, 3) + def parse(cls, platform_spec): + # type: (str) -> PlatformSpec + platform_components = platform_spec.rsplit(cls.SEP, 3) try: plat, impl, version, abi = platform_components except ValueError: - raise cls.InvalidPlatformError.create( - platform, + raise cls.InvalidSpecError.create( + platform_spec, cause="There are missing platform fields. Expected 4 but given {count}.".format( count=len(platform_components) ), @@ -112,8 +100,8 @@ def create(cls, platform): if len(version_components) == 1: component = version_components[0] if len(component) < 2: - raise cls.InvalidPlatformError.create( - platform, + raise cls.InvalidSpecError.create( + platform_spec, cause=( "The version field must either be a 2 or more digit digit major/minor " "version or else a component dotted version. " @@ -128,8 +116,8 @@ def create(cls, platform): try: version_info = cast("VersionInfo", tuple(map(int, version_components))) except ValueError: - raise cls.InvalidPlatformError.create( - platform, + raise cls.InvalidSpecError.create( + platform_spec, cause="The version specified had non-integer components. Given: {version!r}".format( version=version ), @@ -139,7 +127,7 @@ def create(cls, platform): @classmethod def from_tag(cls, tag): - # type: (tags.Tag) -> Platform + # type: (tags.Tag) -> PlatformSpec """Creates a platform corresponding to wheel compatibility tags. See: https://www.python.org/dev/peps/pep-0425/#details @@ -151,7 +139,7 @@ def from_tag(cls, tag): try: version_info = cast("VersionInfo", tuple(map(int, components))) except ValueError: - raise cls.InvalidPlatformError.create( + raise cls.InvalidSpecError.create( tag, cause=( "The tag's interpreter field has an non-integer version suffix following the " @@ -179,7 +167,7 @@ def from_tag(cls, tag): @abi.validator def _non_blank(self, attribute, value): if not value: - raise self.InvalidPlatformError.create( + raise self.InvalidSpecError.create( platform=str(self), cause=( "Platform specifiers cannot have blank fields. Given a blank {field}.".format( @@ -207,143 +195,24 @@ def tag(self): # type: () -> tags.Tag return tags.Tag(interpreter=self.interpreter, abi=self.abi, platform=self.platform) - def _calculate_tags( - self, - manylinux=None, # type: Optional[str] - ): - # type: (...) -> Iterator[tags.Tag] - from pex.jobs import SpawnedJob - from pex.pip.installation import get_pip - - def parse_tags(output): - # type: (bytes) -> Iterator[tags.Tag] - count = None # type: Optional[int] - try: - for line in output.decode("utf-8").splitlines(): - if count is None: - match = re.match(r"^Compatible tags: (?P\d+)\s+", line) - if match: - count = int(match.group("count")) - continue - count -= 1 - if count < 0: - raise AssertionError("Expected {} tags but got more.".format(count)) - for tag in tags.parse_tag(line.strip()): - yield tag - finally: - if count != 0: - raise AssertionError("Finished with count {}.".format(count)) - - from pex.resolve.configured_resolver import ConfiguredResolver - - job = SpawnedJob.stdout( - # TODO(John Sirois): Plumb pip_version and the user-configured resolver: - # https://github.com/pex-tool/pex/issues/1894 - job=get_pip(resolver=ConfiguredResolver.default()).spawn_debug( - platform=self, manylinux=manylinux - ), - result_func=parse_tags, - ) - return job.await_result() - - PLAT_INFO_FILE = "PLAT-INFO" - - _SUPPORTED_TAGS_BY_PLATFORM = ( - {} - ) # type: Dict[Tuple[Platform, Optional[str]], CompatibilityTags] - - def supported_tags(self, manylinux=None): - # type: (Optional[str]) -> CompatibilityTags - - # We use a 2 level cache, probing memory first and then a json file on disk in order to - # avoid calculating tags when possible since it's an O(500ms) operation that involves - # spawning Pip. - - # Read level 1. - memory_cache_key = (self, manylinux) - supported_tags = self._SUPPORTED_TAGS_BY_PLATFORM.get(memory_cache_key) - if supported_tags is not None: - return supported_tags - - # Read level 2. - components = [str(self)] - if manylinux: - components.append(manylinux) - disk_cache_key = CacheDir.PLATFORMS.path(self.SEP.join(components)) - with atomic_directory(target_dir=disk_cache_key) as cache_dir: - if not cache_dir.is_finalized(): - # Missed both caches - spawn calculation. - plat_info = attr.asdict(self) - plat_info.update( - supported_tags=[ - (tag.interpreter, tag.abi, tag.platform) - for tag in self._calculate_tags(manylinux=manylinux) - ], - ) - # Write level 2. - with safe_open(os.path.join(cache_dir.work_dir, self.PLAT_INFO_FILE), "w") as fp: - json.dump(plat_info, fp) - - with open(os.path.join(disk_cache_key, self.PLAT_INFO_FILE)) as fp: - try: - data = json.load(fp) - except ValueError as e: - TRACER.log( - "Regenerating the platform info file at {} since it did not contain parsable " - "JSON data: {}".format(fp.name, e) - ) - safe_rmtree(disk_cache_key) - return self.supported_tags(manylinux=manylinux) - - if not isinstance(data, dict): - TRACER.log( - "Regenerating the platform info file at {} since it did not contain a " - "configuration object. Found: {!r}".format(fp.name, data) - ) - safe_rmtree(disk_cache_key) - return self.supported_tags(manylinux=manylinux) - - sup_tags = data.get("supported_tags") - if not isinstance(sup_tags, list): - TRACER.log( - "Regenerating the platform info file at {} since it was missing a valid " - "`supported_tags` list. Found: {!r}".format(fp.name, sup_tags) - ) - safe_rmtree(disk_cache_key) - return self.supported_tags(manylinux=manylinux) - - count = len(sup_tags) - - def parse_tag( - index, # type: int - tag, # type: List[Any] - ): - # type: (...) -> tags.Tag - if len(tag) != 3 or not all( - isinstance(component, compatibility.string) for component in tag - ): - raise ValueError( - "Serialized platform tags should be lists of three strings. Tag {index} of " - "{count} was: {tag!r}.".format(index=index, count=count, tag=tag) - ) - interpreter, abi, platform = tag - return tags.Tag(interpreter=interpreter, abi=abi, platform=platform) - - try: - supported_tags = CompatibilityTags( - tags=[parse_tag(index, tag) for index, tag in enumerate(sup_tags)] - ) - # Write level 1. - self._SUPPORTED_TAGS_BY_PLATFORM[memory_cache_key] = supported_tags - return supported_tags - except ValueError as e: - TRACER.log( - "Regenerating the platform info file at {} since it did not contain parsable " - "tag data: {}".format(fp.name, e) - ) - safe_rmtree(disk_cache_key) - return self.supported_tags(manylinux=manylinux) - def __str__(self): # type: () -> str return cast(str, self.SEP.join((self.platform, self.impl, self.version, self.abi))) + + +@attr.s(frozen=True) +class Platform(PlatformSpec): + @classmethod + def from_tags(cls, compatibility_tags): + # type: (CompatibilityTags) -> Platform + platform_spec = PlatformSpec.from_tag(compatibility_tags[0]) + return Platform( + platform=platform_spec.platform, + impl=platform_spec.impl, + version=platform_spec.version, + version_info=platform_spec.version_info, + abi=platform_spec.abi, + supported_tags=compatibility_tags, + ) + + supported_tags = attr.ib(eq=False) # type: CompatibilityTags diff --git a/pex/resolve/abbreviated_platforms.py b/pex/resolve/abbreviated_platforms.py new file mode 100644 index 000000000..efd08b6a5 --- /dev/null +++ b/pex/resolve/abbreviated_platforms.py @@ -0,0 +1,200 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, print_function + +import json +import os +import re + +from pex import compatibility +from pex.atomic_directory import atomic_directory +from pex.cache.dirs import CacheDir +from pex.common import safe_open, safe_rmtree +from pex.exceptions import production_assert +from pex.jobs import SpawnedJob +from pex.pep_425 import CompatibilityTags +from pex.pip.installation import get_pip +from pex.platforms import Platform, PlatformSpec +from pex.resolve.configured_resolver import ConfiguredResolver +from pex.resolve.resolver_configuration import PipConfiguration +from pex.third_party.packaging import tags +from pex.tracer import TRACER +from pex.typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any, Iterator, List, Optional + + import attr # vendor:skip +else: + from pex.third_party import attr + + +def _calculate_tags( + pip_configuration, # type: PipConfiguration + platform_spec, # type: PlatformSpec + manylinux=None, # type: Optional[str] +): + # type: (...) -> Iterator[tags.Tag] + + def parse_tags(output): + # type: (bytes) -> Iterator[tags.Tag] + count = None # type: Optional[int] + try: + for line in output.decode("utf-8").splitlines(): + if count is None: + match = re.match(r"^Compatible tags: (?P\d+)\s+", line) + if match: + count = int(match.group("count")) + continue + count -= 1 + if count < 0: + raise AssertionError("Expected {} tags but got more.".format(count)) + for tag in tags.parse_tag(line.strip()): + yield tag + finally: + if count != 0: + raise AssertionError("Finished with count {}.".format(count)) + + pip = get_pip( + version=pip_configuration.version, + resolver=ConfiguredResolver(pip_configuration=pip_configuration), + ) + job = SpawnedJob.stdout( + job=pip.spawn_debug( + platform_spec=platform_spec, manylinux=manylinux, log=pip_configuration.log + ), + result_func=parse_tags, + ) + return job.await_result() + + +class CompatibilityTagsParseError(ValueError): + pass + + +def _parse_tags(path): + # type: (str) -> CompatibilityTags + + with open(path) as fp: + try: + data = json.load(fp) + except ValueError as e: + raise CompatibilityTagsParseError( + "Regenerating the platform info file at {} since it did not contain parsable " + "JSON data: {}".format(fp.name, e) + ) + + if not isinstance(data, dict): + raise CompatibilityTagsParseError( + "Regenerating the platform info file at {} since it did not contain a " + "configuration object. Found: {!r}".format(fp.name, data) + ) + + sup_tags = data.get("supported_tags") + if not isinstance(sup_tags, list): + raise CompatibilityTagsParseError( + "Regenerating the platform info file at {} since it was missing a valid " + "`supported_tags` list. Found: {!r}".format(fp.name, sup_tags) + ) + + count = len(sup_tags) + + def parse_tag( + index, # type: int + tag, # type: List[Any] + ): + # type: (...) -> tags.Tag + if len(tag) != 3 or not all( + isinstance(component, compatibility.string) for component in tag + ): + raise CompatibilityTagsParseError( + "Serialized platform tags should be lists of three strings. Tag {index} of " + "{count} was: {tag!r}.".format(index=index, count=count, tag=tag) + ) + interpreter, abi, platform = tag + return tags.Tag(interpreter=interpreter, abi=abi, platform=platform) + + try: + return CompatibilityTags(tags=[parse_tag(index, tag) for index, tag in enumerate(sup_tags)]) + except ValueError as e: + raise CompatibilityTagsParseError( + "Regenerating the platform info file at {} since it did not contain parsable " + "tag data: {}".format(fp.name, e) + ) + + +PLAT_INFO_FILE = "PLAT-INFO" + + +def create( + platform, # type: str + manylinux=None, # type: Optional[str] + pip_configuration=PipConfiguration(), # type: PipConfiguration +): + # type: (...) -> Platform + + platform_spec = PlatformSpec.parse(platform) + components = [str(platform_spec)] + if manylinux: + components.append(manylinux) + disk_cache_key = CacheDir.PLATFORMS.path( + "pip-{version}".format(version=pip_configuration.version), PlatformSpec.SEP.join(components) + ) + + with atomic_directory(target_dir=disk_cache_key) as cache_dir: + if cache_dir.is_finalized(): + cached = True + else: + cached = False + plat_info = attr.asdict(platform_spec) + plat_info.update( + supported_tags=[ + (tag.interpreter, tag.abi, tag.platform) + for tag in _calculate_tags( + pip_configuration, platform_spec, manylinux=manylinux + ) + ], + ) + with safe_open(os.path.join(cache_dir.work_dir, PLAT_INFO_FILE), "w") as fp: + json.dump(plat_info, fp) + + platform_info_file = os.path.join(disk_cache_key, PLAT_INFO_FILE) + try: + compatibility_tags = _parse_tags(path=platform_info_file) + except CompatibilityTagsParseError as e: + production_assert( + cached, + ( + "Unexpectedly generated invalid abbreviated platform compatibility tags from " + "{platform}: {err}".format(platform=platform, err=e) + ), + ) + TRACER.log(str(e)) + safe_rmtree(disk_cache_key) + return create(platform, manylinux=manylinux, pip_configuration=pip_configuration) + + if cached and pip_configuration.log: + # When not cached and the Pip log is being retained, the tags are logged directly by our + # call to `pip -v debug ...` in _calculate_tags above. We do the same for the cached case + # since this can be very useful information when investigating why Pip did not select a + # particular wheel for an abbreviated --platform. + with safe_open(pip_configuration.log, "a") as fp: + print( + "Read {count} compatible tags for abbreviated --platform {platform} from:".format( + count=len(compatibility_tags), platform=platform + ), + file=fp, + ) + print(" {cache_file}".format(cache_file=platform_info_file), file=fp) + for tag in compatibility_tags: + print(tag, file=fp) + + return Platform( + platform=platform_spec.platform, + impl=platform_spec.impl, + version=platform_spec.version, + version_info=platform_spec.version_info, + abi=platform_spec.abi, + supported_tags=compatibility_tags, + ) diff --git a/pex/resolve/config.py b/pex/resolve/config.py index f6ad40d52..bc2ec4eee 100644 --- a/pex/resolve/config.py +++ b/pex/resolve/config.py @@ -9,6 +9,7 @@ LockRepositoryConfiguration, PexRepositoryConfiguration, PipConfiguration, + PreResolvedConfiguration, ) from pex.result import Error, catch, try_ from pex.targets import Targets @@ -19,7 +20,12 @@ import attr # vendor:skip - Configuration = Union[LockRepositoryConfiguration, PexRepositoryConfiguration, PipConfiguration] + Configuration = Union[ + LockRepositoryConfiguration, + PexRepositoryConfiguration, + PreResolvedConfiguration, + PipConfiguration, + ] _C = TypeVar("_C", bound=Configuration) else: @@ -61,6 +67,17 @@ def finalize( ), ) + if isinstance(resolver_configuration, (PexRepositoryConfiguration, PreResolvedConfiguration)): + pip_configuration = try_( + _finalize_pip_configuration( + resolver_configuration.pip_configuration, + targets, + context, + pip_version=pip_version, + ) + ) + return cast("_C", attr.evolve(resolver_configuration, pip_configuration=pip_configuration)) + if isinstance(resolver_configuration, LockRepositoryConfiguration): lock_file = try_(resolver_configuration.parse_lock()) pip_configuration = try_( diff --git a/pex/resolve/lockfile/create.py b/pex/resolve/lockfile/create.py index 38e7371a9..868933cfa 100644 --- a/pex/resolve/lockfile/create.py +++ b/pex/resolve/lockfile/create.py @@ -461,9 +461,7 @@ def create( targets.platforms or targets.complete_platforms ): check_targets = Targets( - platforms=targets.platforms, - complete_platforms=targets.complete_platforms, - assume_manylinux=targets.assume_manylinux, + platforms=targets.platforms, complete_platforms=targets.complete_platforms ) with TRACER.timed( "Checking lock can resolve for platforms: {targets}".format(targets=check_targets) diff --git a/pex/resolve/resolver_configuration.py b/pex/resolve/resolver_configuration.py index 35354b0b7..92df8b0c6 100644 --- a/pex/resolve/resolver_configuration.py +++ b/pex/resolve/resolver_configuration.py @@ -196,13 +196,22 @@ class PipConfiguration(object): @attr.s(frozen=True) class PexRepositoryConfiguration(object): pex_repository = attr.ib() # type: str - network_configuration = attr.ib(default=NetworkConfiguration()) # type: NetworkConfiguration - transitive = attr.ib(default=True) # type: bool + pip_configuration = attr.ib() # type: PipConfiguration @property def repos_configuration(self): # type: () -> ReposConfiguration - return ReposConfiguration() + return self.pip_configuration.repos_configuration + + @property + def network_configuration(self): + # type: () -> NetworkConfiguration + return self.pip_configuration.network_configuration + + @property + def transitive(self): + # type: () -> bool + return self.pip_configuration.transitive @attr.s(frozen=True) diff --git a/pex/resolve/resolver_options.py b/pex/resolve/resolver_options.py index c523088a4..902106e70 100644 --- a/pex/resolve/resolver_options.py +++ b/pex/resolve/resolver_options.py @@ -547,6 +547,8 @@ def configure(options): :raise: :class:`InvalidConfigurationError` if the resolver configuration is invalid. """ + pip_configuration = create_pip_configuration(options) + pex_repository = getattr(options, "pex_repository", None) if pex_repository: if options.indexes or options.find_links: @@ -555,12 +557,9 @@ def configure(options): '"--find-links" options.' ) return PexRepositoryConfiguration( - pex_repository=pex_repository, - network_configuration=create_network_configuration(options), - transitive=options.transitive, + pex_repository=pex_repository, pip_configuration=pip_configuration ) - pip_configuration = create_pip_configuration(options) lock = getattr(options, "lock", None) if lock: return LockRepositoryConfiguration( diff --git a/pex/resolve/target_configuration.py b/pex/resolve/target_configuration.py index 0ccdab265..8e50b692d 100644 --- a/pex/resolve/target_configuration.py +++ b/pex/resolve/target_configuration.py @@ -162,7 +162,6 @@ def interpreter_constraints(self): complete_platforms = attr.ib(default=()) # type: Tuple[CompletePlatform, ...] platforms = attr.ib(default=()) # type: Tuple[Optional[Platform], ...] - assume_manylinux = attr.ib(default="manylinux2014") # type: Optional[str] resolve_local_platforms = attr.ib(default=False) # type: bool def resolve_targets(self): @@ -238,5 +237,4 @@ def resolve_targets(self): interpreters=tuple(interpreters), complete_platforms=tuple(requested_complete_platforms), platforms=tuple(requested_platforms), - assume_manylinux=self.assume_manylinux, ) diff --git a/pex/resolve/target_options.py b/pex/resolve/target_options.py index bb5b40bb4..f77154d7e 100644 --- a/pex/resolve/target_options.py +++ b/pex/resolve/target_options.py @@ -13,10 +13,16 @@ from pex.orderedset import OrderedSet from pex.pep_425 import CompatibilityTags from pex.pep_508 import MarkerEnvironment -from pex.platforms import Platform +from pex.platforms import Platform, PlatformSpec +from pex.resolve import abbreviated_platforms +from pex.resolve.resolver_configuration import PipConfiguration from pex.resolve.resolver_options import _ManylinuxAction from pex.resolve.target_configuration import InterpreterConfiguration, TargetConfiguration from pex.targets import CompletePlatform +from pex.typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Optional def register( @@ -147,14 +153,13 @@ def _register_platform_options( ), ) - default_target_configuration = TargetConfiguration() parser.add_argument( "--manylinux", "--no-manylinux", "--no-use-manylinux", dest="assume_manylinux", type=str, - default=default_target_configuration.assume_manylinux, + default="manylinux2014", action=_ManylinuxAction, help="Whether to allow resolution of manylinux wheels for linux target platforms.", ) @@ -239,23 +244,35 @@ def _create_complete_platform(value): return CompletePlatform.create(marker_environment, supported_tags) -def configure(options): - # type: (Namespace) -> TargetConfiguration +def configure( + options, # type: Namespace + pip_configuration, # type: PipConfiguration +): + # type: (...) -> TargetConfiguration """Creates a target configuration from options via `register(..., include_platforms=True)`. :param options: The target configuration options. + :param pip_configuration: The Pip configuration options. """ interpreter_configuration = configure_interpreters(options) - try: - platforms = tuple( - OrderedSet( - Platform.create(platform) if platform and platform != "current" else None + platforms = OrderedSet() # type: OrderedSet[Optional[Platform]] + if options.platforms: + try: + platforms.update( + ( + abbreviated_platforms.create( + platform, + manylinux=options.assume_manylinux, + pip_configuration=pip_configuration, + ) + if platform and platform != "current" + else None + ) for platform in options.platforms ) - ) - except Platform.InvalidPlatformError as e: - raise ArgumentTypeError(str(e)) + except PlatformSpec.InvalidSpecError as e: + raise ArgumentTypeError(str(e)) complete_platforms = tuple( OrderedSet(_create_complete_platform(value) for value in options.complete_platforms) @@ -263,8 +280,7 @@ def configure(options): return TargetConfiguration( interpreter_configuration=interpreter_configuration, - platforms=platforms, + platforms=tuple(platforms), complete_platforms=complete_platforms, resolve_local_platforms=options.resolve_local_platforms, - assume_manylinux=options.assume_manylinux, ) diff --git a/pex/scie/model.py b/pex/scie/model.py index fb2c0f7a6..50c3da15e 100644 --- a/pex/scie/model.py +++ b/pex/scie/model.py @@ -14,7 +14,7 @@ from pex.finders import get_entry_point_from_console_script from pex.pep_503 import ProjectName from pex.pex import PEX -from pex.platforms import Platform +from pex.platforms import PlatformSpec from pex.targets import Targets from pex.third_party.packaging import tags # noqa from pex.typing import TYPE_CHECKING, cast @@ -384,8 +384,9 @@ def from_tags( tags, # type: Iterable[tags.Tag] ): # type: (...) -> ScieConfiguration - return cls._from_platforms( - options=options, platforms=tuple(Platform.from_tag(tag) for tag in tags) + return cls._from_platform_specs( + options=options, + platform_specs=tuple(PlatformSpec.from_tag(tag) for tag in tags), ) @classmethod @@ -395,16 +396,16 @@ def from_targets( targets, # type: Targets ): # type: (...) -> ScieConfiguration - return cls._from_platforms( + return cls._from_platform_specs( options=options, - platforms=tuple(target.platform for target in targets.unique_targets()), + platform_specs=tuple(target.platform for target in targets.unique_targets()), ) @classmethod - def _from_platforms( + def _from_platform_specs( cls, options, # type: ScieOptions - platforms, # type: Iterable[Platform] + platform_specs, # type: Iterable[PlatformSpec] ): # type: (...) -> ScieConfiguration @@ -412,20 +413,21 @@ def _from_platforms( python_versions_by_platform = defaultdict( set ) # type: DefaultDict[SciePlatform.Value, Set[Union[Tuple[int, int], Tuple[int, int, int]]]] - for plat in platforms: + for platform_spec in platform_specs: if python_version: plat_python_version = python_version - elif len(plat.version_info) < 2: + elif len(platform_spec.version_info) < 2: continue else: # Here were guessing an available PBS CPython version. Since a triple is unlikely to # hit, we just use major / minor. If the user wants control they can specify # options.python_version via `--scie-python-version`. plat_python_version = cast( - "Union[Tuple[int, int], Tuple[int, int, int]]", plat.version_info + "Union[Tuple[int, int], Tuple[int, int, int]]", + platform_spec.version_info, )[:2] - platform_str = plat.platform + platform_str = platform_spec.platform is_aarch64 = "arm64" in platform_str or "aarch64" in platform_str is_x86_64 = "amd64" in platform_str or "x86_64" in platform_str if not is_aarch64 ^ is_x86_64: @@ -446,12 +448,12 @@ def _from_platforms( else: continue - if plat.impl in ("py", "cp"): + if platform_spec.impl in ("py", "cp"): # Python Build Standalone does not support CPython<3.8. if plat_python_version < (3, 8): continue provider = Provider.PythonBuildStandalone - elif "pp" == plat.impl: + elif "pp" == platform_spec.impl: # PyPy distributions for Linux aarch64 start with 3.7 (and PyPy always releases for # 2.7). if ( @@ -460,7 +462,8 @@ def _from_platforms( and plat_python_version < (3, 7) ): continue - # PyPy distributions for Mac arm64 start with 3.8 (and PyPy always releases for 2.7). + # PyPy distributions for Mac arm64 start with 3.8 (and PyPy always releases for + # 2.7). if ( SciePlatform.MACOS_AARCH64 is scie_platform and plat_python_version[0] == 3 diff --git a/pex/targets.py b/pex/targets.py index 49ad956f5..e70323966 100644 --- a/pex/targets.py +++ b/pex/targets.py @@ -251,25 +251,18 @@ def render_description(self): @attr.s(frozen=True) class AbbreviatedPlatform(Target): @classmethod - def create( - cls, - platform, # type: Platform - manylinux=None, # type: Optional[str] - ): - # type: (...) -> AbbreviatedPlatform + def create(cls, platform): + # type: (Platform) -> AbbreviatedPlatform return cls( id=str(platform.tag), marker_environment=MarkerEnvironment.from_platform(platform), platform=platform, - manylinux=manylinux, ) - manylinux = attr.ib() # type: Optional[str] - @property def supported_tags(self): # type: () -> CompatibilityTags - return self.platform.supported_tags(manylinux=self.manylinux) + return self.platform.supported_tags def render_description(self): return "abbreviated platform {platform}".format(platform=self.platform.tag) @@ -298,7 +291,7 @@ def create( ): # type: (...) -> CompletePlatform - platform = Platform.from_tag(supported_tags[0]) + platform = Platform.from_tags(supported_tags) return cls( id=str(platform.tag), marker_environment=marker_environment, @@ -323,7 +316,7 @@ class Targets(object): def from_target(cls, target): # type: (Target) -> Targets if isinstance(target, AbbreviatedPlatform): - return cls(platforms=(target.platform,), assume_manylinux=target.manylinux) + return cls(platforms=(target.platform,)) elif isinstance(target, CompletePlatform): return cls(complete_platforms=(target,)) else: @@ -332,7 +325,6 @@ def from_target(cls, target): interpreters = attr.ib(default=()) # type: Tuple[PythonInterpreter, ...] complete_platforms = attr.ib(default=()) # type: Tuple[CompletePlatform, ...] platforms = attr.ib(default=()) # type: Tuple[Optional[Platform], ...] - assume_manylinux = attr.ib(default=None) # type: Optional[str] @property def interpreter(self): @@ -368,7 +360,7 @@ def iter_targets(): yield current() elif platform is not None: # Build for specific platforms. - yield AbbreviatedPlatform.create(platform, manylinux=self.assume_manylinux) + yield AbbreviatedPlatform.create(platform) for complete_platform in self.complete_platforms: yield complete_platform diff --git a/tests/bin/test_sh_boot.py b/tests/bin/test_sh_boot.py index 46e87220f..5c080e01f 100644 --- a/tests/bin/test_sh_boot.py +++ b/tests/bin/test_sh_boot.py @@ -11,7 +11,7 @@ from pex.orderedset import OrderedSet from pex.pep_425 import CompatibilityTags from pex.pep_508 import MarkerEnvironment -from pex.platforms import Platform +from pex.resolve import abbreviated_platforms from pex.sh_boot import PythonBinaryName from pex.targets import CompletePlatform, Targets from pex.typing import TYPE_CHECKING, cast @@ -120,8 +120,8 @@ def test_calculate_platforms_no_ics(requires_python): ) == calculate_binary_names( Targets( platforms=( - Platform.create("macosx-10.13-x86_64-cp-36-cp36m"), - Platform.create("linux-x86_64-pp-27-pypy_73"), + abbreviated_platforms.create("macosx-10.13-x86_64-cp-36-cp36m"), + abbreviated_platforms.create("linux-x86_64-pp-27-pypy_73"), ) ) ) diff --git a/tests/integration/cli/commands/test_lock.py b/tests/integration/cli/commands/test_lock.py index 6ed2c17b9..add0de3a6 100644 --- a/tests/integration/cli/commands/test_lock.py +++ b/tests/integration/cli/commands/test_lock.py @@ -19,7 +19,7 @@ from pex.pep_440 import Version from pex.pep_503 import ProjectName from pex.pip.version import PipVersion -from pex.platforms import Platform +from pex.resolve import abbreviated_platforms from pex.resolve.locked_resolve import Artifact, LockedRequirement from pex.resolve.lockfile import json_codec from pex.resolve.lockfile.download_manager import DownloadedArtifact @@ -255,7 +255,9 @@ def test_create_universal_platform_check(tmpdir): foreign_platform_310 = ( "macosx_10.9_x86_64-cp-310-cp310" if IS_LINUX else "linux_x86_64-cp-310-cp310" ) - abbreviated_platform_310 = AbbreviatedPlatform.create(Platform.create(foreign_platform_310)) + abbreviated_platform_310 = AbbreviatedPlatform.create( + abbreviated_platforms.create(foreign_platform_310) + ) complete_platform = os.path.join(str(tmpdir), "complete-platform.json") run_pex3("interpreter", "inspect", "--markers", "--tags", "-v", "-i2", "-o", complete_platform) @@ -388,7 +390,9 @@ def test_create_universal_platform_check(tmpdir): "psutil==5.9.1", ) result.assert_failure() - abbreviated_platform_311 = AbbreviatedPlatform.create(Platform.create(foreign_platform_311)) + abbreviated_platform_311 = AbbreviatedPlatform.create( + abbreviated_platforms.create(foreign_platform_311) + ) assert re.search( r"No pre-built wheel was available for psutil 5\.9\.1\.{eol}" r"Successfully built the wheel psutil-5\.9\.1-\S+\.whl from the sdist " diff --git a/tests/integration/cli/commands/test_venv_create.py b/tests/integration/cli/commands/test_venv_create.py index 03686c5df..e79268dc1 100644 --- a/tests/integration/cli/commands/test_venv_create.py +++ b/tests/integration/cli/commands/test_venv_create.py @@ -22,7 +22,7 @@ from pex.pep_440 import Version from pex.pep_503 import ProjectName from pex.pex import PEX -from pex.platforms import Platform +from pex.resolve import abbreviated_platforms from pex.typing import TYPE_CHECKING from pex.venv.virtualenv import Virtualenv from testing import IS_MAC, PY39, PY310, ensure_python_interpreter, make_env, run_pex_command @@ -483,7 +483,7 @@ def test_foreign_target( result.assert_failure() assert ( "Cannot create a local venv for foreign platform {platform}.".format( - platform=Platform.create(foreign_platform) + platform=abbreviated_platforms.create(foreign_platform) ) == result.error.strip() ) @@ -523,7 +523,7 @@ def test_venv_update_target_mismatch( result.assert_failure() assert ( "Cannot update a local venv using a foreign platform. Given: {foreign_platform}.".format( - foreign_platform=Platform.create(foreign_platform) + foreign_platform=abbreviated_platforms.create(foreign_platform) ) == result.error.strip() ), result.error diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 3769243e8..77fd75d09 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -17,6 +17,7 @@ import pexpect # type: ignore[import] # MyPy can't see the types under Python 2.7. import pytest +from pex import targets from pex.cache.dirs import CacheDir from pex.common import is_exe, safe_mkdir, safe_open, safe_rmtree, temporary_dir, touch from pex.compatibility import WINDOWS, commonpath @@ -27,6 +28,7 @@ from pex.network_configuration import NetworkConfiguration from pex.pep_427 import InstallableType from pex.pex_info import PexInfo +from pex.pip.version import PipVersion from pex.requirements import LogicalLine, PyPIRequirement, parse_requirement_file from pex.typing import TYPE_CHECKING, cast from pex.util import named_temporary_file @@ -382,32 +384,51 @@ def do_something(): assert rc == 1 -def test_pex_multi_resolve(): - # type: () -> None +def test_pex_multi_resolve_1(tmpdir): + # type: (Any) -> None """Tests multi-interpreter + multi-platform resolution.""" python38 = ensure_python_interpreter(PY38) python39 = ensure_python_interpreter(PY39) - with temporary_dir() as output_dir: - pex_path = os.path.join(output_dir, "pex.pex") - results = run_pex_command( - [ - "--disable-cache", - "lxml==4.6.1", - "--no-build", - "--platform=linux-x86_64-cp-36-m", - "--platform=macosx-10.9-x86_64-cp-36-m", - "--python={}".format(python38), - "--python={}".format(python39), - "-o", - pex_path, - ] + + pex_path = os.path.join(str(tmpdir), "pex.pex") + + pip_log = os.path.join(str(tmpdir), "pip.log") + + def read_pip_log(): + # type: () -> str + if not os.path.exists(pip_log): + return "Did not find Pip log at {log}.".format(log=pip_log) + with open(pip_log) as fp: + return fp.read() + + result = run_pex_command( + args=[ + "--disable-cache", + "lxml==4.6.1", + "--no-build", + "--platform=linux-x86_64-cp-36-m", + "--platform=macosx-10.9-x86_64-cp-36-m", + "--python={}".format(python38), + "--python={}".format(python39), + "-o", + pex_path, + "--pip-log", + pip_log, + ] + ) + assert 0 == result.return_code, ( + "Failed to resolve lxml for all platforms. Pip download log:\n" + "-----------------------------------------------------------\n" + "{pip_log_text}\n" + "-----------------------------------------------------------\n".format( + pip_log_text=read_pip_log() ) - results.assert_success() + ) - included_dists = get_dep_dist_names_from_pex(pex_path, "lxml") - assert len(included_dists) == 4 - for dist_substr in ("-cp36-", "-cp38-", "-cp39-", "-manylinux1_x86_64", "-macosx_"): - assert any(dist_substr in f for f in included_dists) + included_dists = get_dep_dist_names_from_pex(pex_path, "lxml") + assert len(included_dists) == 4 + for dist_substr in ("-cp36-", "-cp38-", "-cp39-", "-manylinux1_x86_64", "-macosx_"): + assert any(dist_substr in f for f in included_dists) def test_pex_path_arg(): @@ -569,32 +590,49 @@ def inherit_path(inherit_path): assert requests_paths[0] > sys_paths[0] -def test_pex_multi_resolve_2(): - # type: () -> None - """Tests multi-interpreter + multi-platform resolution using extended platform notation.""" - with temporary_dir() as output_dir: - pex_path = os.path.join(output_dir, "pex.pex") - results = run_pex_command( - [ - "--disable-cache", - "lxml==3.8.0", - "--no-build", - "--platform=linux-x86_64-cp-36-m", - "--platform=linux-x86_64-cp-27-m", - "--platform=macosx-10.6-x86_64-cp-36-m", - "--platform=macosx-10.6-x86_64-cp-27-m", - "-o", - pex_path, - ] +def test_pex_multi_resolve_2(tmpdir): + # type: (Any) -> None + + pip_log = os.path.join(str(tmpdir), "pip.log") + + def read_pip_log(): + # type: () -> str + if not os.path.exists(pip_log): + return "Did not find Pip log at {log}.".format(log=pip_log) + with open(pip_log) as fp: + return fp.read() + + pex_path = os.path.join(str(tmpdir), "pex.pex") + result = run_pex_command( + args=[ + "--disable-cache", + "lxml==3.8.0", + "--no-build", + "--platform=linux-x86_64-cp-36-m", + "--platform=linux-x86_64-cp-27-m", + "--platform=macosx-10.6-x86_64-cp-36-m", + "--platform=macosx-10.6-x86_64-cp-27-m", + "-o", + pex_path, + "--pip-log", + pip_log, + ] + ) + assert 0 == result.return_code, ( + "Failed to resolve lxml for all platforms. Pip download log:\n" + "-----------------------------------------------------------\n" + "{pip_log_text}\n" + "-----------------------------------------------------------\n".format( + pip_log_text=read_pip_log() ) - results.assert_success() + ) - included_dists = get_dep_dist_names_from_pex(pex_path, "lxml") - assert len(included_dists) == 4 - for dist_substr in ("-cp27-", "-cp36-", "-manylinux1_x86_64", "-macosx_"): - assert any( - dist_substr in f for f in included_dists - ), "{} was not found in wheel".format(dist_substr) + included_dists = get_dep_dist_names_from_pex(pex_path, "lxml") + assert len(included_dists) == 4 + for dist_substr in ("-cp27-", "-cp36-", "-manylinux1_x86_64", "-macosx_"): + assert any(dist_substr in f for f in included_dists), "{} was not found in wheel".format( + dist_substr + ) if TYPE_CHECKING: @@ -748,24 +786,26 @@ def test_ipython_appnope_env_markers(): res.assert_success() -def test_cross_platform_abi_targeting_behavior_exact(): - # type: () -> None - with temporary_dir() as td: - pex_out_path = os.path.join(td, "pex.pex") - res = run_pex_command( - [ - "--disable-cache", - "--no-pypi", - "--platform=linux-x86_64-cp-27-mu", - "--find-links=tests/example_packages/", - # Since we have no PyPI access, ensure we're using vendored Pip for this test. - "--pip-version=vendored", - "MarkupSafe==1.0", - "-o", - pex_out_path, - ] - ) - res.assert_success() +@pytest.mark.skipif( + not PipVersion.VENDORED.requires_python_applies(targets.current()), + reason="This test needs to use `--pip-version vendored`.", +) +def test_cross_platform_abi_targeting_behavior_exact(tmpdir): + # type: (Any) -> None + pex_out_path = os.path.join(str(tmpdir), "pex.pex") + run_pex_command( + args=[ + "--disable-cache", + "--no-pypi", + "--platform=linux-x86_64-cp-27-mu", + "--find-links=tests/example_packages/", + # Since we have no PyPI access, ensure we're using vendored Pip for this test. + "--pip-version=vendored", + "MarkupSafe==1.0", + "-o", + pex_out_path, + ] + ).assert_success() def test_pex_source_bundling(): @@ -1704,13 +1744,16 @@ def test_seed_verbose( args=args + execution_mode_args + [ + "--pex-root", + pex_root, + "--runtime-pex-root", + pex_root, "--layout", layout.value, get_installable_type_flag(installable_type), "--seed", "verbose", ], - env=make_env(PEX_ROOT=pex_root, PEX_PYTHON_PATH=sys.executable), ) results.assert_success() verbose_info = json.loads(results.output) diff --git a/tests/integration/test_issue_2073.py b/tests/integration/test_issue_2073.py index 6e8350e68..aa016e876 100644 --- a/tests/integration/test_issue_2073.py +++ b/tests/integration/test_issue_2073.py @@ -5,7 +5,7 @@ import re import subprocess -from pex.platforms import Platform +from pex.resolve import abbreviated_platforms from pex.targets import AbbreviatedPlatform from pex.typing import TYPE_CHECKING from testing import IS_LINUX, IntegResults, run_pex_command @@ -18,7 +18,9 @@ FOREIGN_PLATFORM_311 = ( "macosx_10.9_x86_64-cp-311-cp311" if IS_LINUX else "linux_x86_64-cp-311-cp311" ) -ABBREVIATED_FOREIGN_PLATFORM_311 = AbbreviatedPlatform.create(Platform.create(FOREIGN_PLATFORM_311)) +ABBREVIATED_FOREIGN_PLATFORM_311 = AbbreviatedPlatform.create( + abbreviated_platforms.create(FOREIGN_PLATFORM_311) +) def assert_psutil_cross_build_failure(result): diff --git a/tests/integration/test_issue_455.py b/tests/integration/test_issue_455.py index f6ce28547..f221bec95 100644 --- a/tests/integration/test_issue_455.py +++ b/tests/integration/test_issue_455.py @@ -26,8 +26,6 @@ def test_exclude(tmpdir): pex = os.path.join(str(tmpdir), "pex") run_pex_command( args=[ - "--pip-version", - "latest", "--platform", "macosx_10.12-x86_64-cp-310-cp310", "--platform", diff --git a/tests/resolve/test_locked_resolve.py b/tests/resolve/test_locked_resolve.py index bad2d6b2d..b60a612fa 100644 --- a/tests/resolve/test_locked_resolve.py +++ b/tests/resolve/test_locked_resolve.py @@ -12,8 +12,8 @@ from pex.pep_440 import Version from pex.pep_503 import ProjectName from pex.pep_508 import MarkerEnvironment -from pex.platforms import Platform from pex.requirements import VCS +from pex.resolve import abbreviated_platforms from pex.resolve.locked_resolve import ( Artifact, DownloadableArtifact, @@ -272,7 +272,7 @@ def assert_error( def platform(plat): # type: (str) -> AbbreviatedPlatform - return AbbreviatedPlatform.create(Platform.create(plat)) + return AbbreviatedPlatform.create(abbreviated_platforms.create(plat)) def test_platform_resolve(ansicolors_exotic): diff --git a/tests/resolve/test_pex_repository_resolver.py b/tests/resolve/test_pex_repository_resolver.py index 299fd2c5e..9c8f91c00 100644 --- a/tests/resolve/test_pex_repository_resolver.py +++ b/tests/resolve/test_pex_repository_resolver.py @@ -14,6 +14,7 @@ from pex.pex_builder import PEXBuilder from pex.pex_info import PexInfo from pex.platforms import Platform +from pex.resolve import abbreviated_platforms from pex.resolve.configured_resolver import ConfiguredResolver from pex.resolve.pex_repository_resolver import resolve_from_pex from pex.resolve.resolver_configuration import PipConfiguration @@ -33,7 +34,6 @@ def create_pex_repository( requirements=None, # type: Optional[Iterable[str]] requirement_files=None, # type: Optional[Iterable[str]] constraint_files=None, # type: Optional[Iterable[str]] - manylinux=None, # type: Optional[str] result_type=InstallableType.INSTALLED_WHEEL_CHROOT, # type: InstallableType.Value ): # type: (...) -> str @@ -43,7 +43,6 @@ def create_pex_repository( targets=Targets( interpreters=tuple(interpreters) if interpreters else (), platforms=tuple(platforms) if platforms else (), - assume_manylinux=manylinux, ), requirements=requirements, requirement_files=requirement_files, @@ -82,19 +81,13 @@ def py310(): @pytest.fixture(scope="module") def macosx(): # type: () -> Platform - return Platform.create("macosx-10.13-x86_64-cp-36-m") + return abbreviated_platforms.create("macosx-10.13-x86_64-cp-36-m") @pytest.fixture(scope="module") def linux(): # type: () -> Platform - return Platform.create("linux-x86_64-cp-36-m") - - -@pytest.fixture(scope="module") -def manylinux(): - # type: () -> Optional[str] - return None if IS_LINUX else "manylinux2014" + return abbreviated_platforms.create("linux-x86_64-cp-36-m", manylinux="manylinux2014") @pytest.fixture(scope="module") @@ -117,7 +110,6 @@ def pex_repository( py27, # type: PythonInterpreter py310, # type: PythonInterpreter foreign_platform, # type: Platform - manylinux, # type: Optional[str] request, # type: pytest.FixtureRequest ): # type (...) -> str @@ -139,7 +131,6 @@ def pex_repository( platforms=[foreign_platform], requirements=["requests[security,socks]==2.25.1"], constraint_files=[constraints_file], - manylinux=manylinux, result_type=request.param, ) @@ -149,7 +140,6 @@ def test_resolve_from_pex( py27, # type: PythonInterpreter py310, # type: PythonInterpreter foreign_platform, # type: Platform - manylinux, # type: Optional[str] ): # type: (...) -> None pex_info = PexInfo.from_pex(pex_repository) @@ -205,11 +195,7 @@ def assert_resolve_result( resolve_from_pex( pex=pex_repository, requirements=direct_requirements, - targets=Targets( - interpreters=(py27, py310), - platforms=(foreign_platform,), - assume_manylinux=manylinux, - ), + targets=Targets(interpreters=(py27, py310), platforms=(foreign_platform,)), ), expected_result_type=InstallableType.INSTALLED_WHEEL_CHROOT, ) @@ -219,11 +205,7 @@ def assert_resolve_result( resolve_from_pex( pex=pex_repository, requirements=direct_requirements, - targets=Targets( - interpreters=(py27, py310), - platforms=(foreign_platform,), - assume_manylinux=manylinux, - ), + targets=Targets(interpreters=(py27, py310), platforms=(foreign_platform,)), result_type=InstallableType.WHEEL_FILE, ), expected_result_type=InstallableType.WHEEL_FILE, @@ -239,11 +221,7 @@ def assert_resolve_result( resolve_from_pex( pex=pex_repository, requirements=direct_requirements, - targets=Targets( - interpreters=(py27, py310), - platforms=(foreign_platform,), - assume_manylinux=manylinux, - ), + targets=Targets(interpreters=(py27, py310), platforms=(foreign_platform,)), result_type=InstallableType.WHEEL_FILE, ) @@ -251,17 +229,13 @@ def assert_resolve_result( def test_resolve_from_pex_subset( pex_repository, # type: str foreign_platform, # type: Platform - manylinux, # type: Optional[str] ): # type: (...) -> None result = resolve_from_pex( pex=pex_repository, requirements=["cffi"], - targets=Targets( - platforms=(foreign_platform,), - assume_manylinux=manylinux, - ), + targets=Targets(platforms=(foreign_platform,)), ) assert {"cffi", "pycparser"} == { @@ -280,9 +254,7 @@ def test_resolve_from_pex_not_found( resolve_from_pex( pex=pex_repository, requirements=["pex"], - targets=Targets( - interpreters=(py310,), - ), + targets=Targets(interpreters=(py310,)), ) assert "A distribution for pex could not be resolved for {py310_exe}.".format( py310_exe=py310.binary @@ -292,9 +264,7 @@ def test_resolve_from_pex_not_found( resolve_from_pex( pex=pex_repository, requirements=["requests==1.0.0"], - targets=Targets( - interpreters=(py310,), - ), + targets=Targets(interpreters=(py310,)), ) message = str(exec_info.value) assert ( @@ -311,7 +281,6 @@ def test_resolve_from_pex_intransitive( py27, # type: PythonInterpreter py310, # type: PythonInterpreter foreign_platform, # type: Platform - manylinux, # type: Optional[str] ): # type: (...) -> None @@ -319,11 +288,7 @@ def test_resolve_from_pex_intransitive( pex=pex_repository, requirements=["requests"], transitive=False, - targets=Targets( - interpreters=(py27, py310), - platforms=(foreign_platform,), - assume_manylinux=manylinux, - ), + targets=Targets(interpreters=(py27, py310), platforms=(foreign_platform,)), ).distributions assert 3 == len( resolved_distributions @@ -357,9 +322,7 @@ def test_resolve_from_pex_constraints( pex=pex_repository, requirements=["requests"], constraint_files=[create_constraints_file("urllib3==1.26.2")], - targets=Targets( - interpreters=(py27,), - ), + targets=Targets(interpreters=(py27,)), ) message = str(exec_info.value) assert "The following constraints were not satisfied by " in message @@ -378,9 +341,7 @@ def test_resolve_from_pex_ignore_errors( pex=pex_repository, requirements=["requests"], constraint_files=[create_constraints_file("urllib3==1.26.2")], - targets=Targets( - interpreters=(py27,), - ), + targets=Targets(interpreters=(py27,)), ignore_errors=True, ) resolved_distributions_by_key = { diff --git a/tests/resolve/test_target_options.py b/tests/resolve/test_target_options.py index cf1167672..114fc9993 100644 --- a/tests/resolve/test_target_options.py +++ b/tests/resolve/test_target_options.py @@ -14,7 +14,8 @@ from pex.pep_425 import CompatibilityTags from pex.pep_508 import MarkerEnvironment from pex.platforms import Platform -from pex.resolve import target_options +from pex.resolve import abbreviated_platforms, target_options +from pex.resolve.resolver_configuration import PipConfiguration from pex.targets import CompletePlatform, Targets from pex.typing import TYPE_CHECKING from pex.variables import ENV @@ -30,20 +31,18 @@ def compute_target_configuration( ): # type: (...) -> Targets options = parser.parse_args(args=args) - return target_options.configure(options).resolve_targets() + return target_options.configure(options, pip_configuration=PipConfiguration()).resolve_targets() def test_clp_manylinux(parser): # type: (ArgumentParser) -> None target_options.register(parser) - targets = compute_target_configuration(parser, args=[]) - assert targets.assume_manylinux, "The --manylinux option should default to some value." + assert compute_target_configuration(parser, args=[]) is not None def assert_manylinux(value): # type: (str) -> None - rc = compute_target_configuration(parser, args=["--manylinux", value]) - assert value == rc.assume_manylinux + assert compute_target_configuration(parser, args=["--manylinux", value]) is not None # Legacy manylinux standards should be supported. assert_manylinux("manylinux1_x86_64") @@ -54,8 +53,7 @@ def assert_manylinux(value): assert_manylinux("manylinux_2_5_x86_64") assert_manylinux("manylinux_2_33_x86_64") - targets = compute_target_configuration(parser, args=["--no-manylinux"]) - assert targets.assume_manylinux is None + assert compute_target_configuration(parser, args=["--no-manylinux"]) is not None with pytest.raises(ArgumentTypeError): compute_target_configuration(parser, args=["--manylinux", "foo"]) @@ -80,12 +78,16 @@ def assert_platforms( # The special 'current' platform should map to a `None` platform entry. assert_platforms(["current"], None) - assert_platforms(["linux-x86_64-cp-37-cp37m"], Platform.create("linux-x86_64-cp-37-cp37m")) - assert_platforms(["linux-x86_64-cp-37-m"], Platform.create("linux-x86_64-cp-37-cp37m")) + assert_platforms( + ["linux-x86_64-cp-37-cp37m"], abbreviated_platforms.create("linux-x86_64-cp-37-cp37m") + ) + assert_platforms( + ["linux-x86_64-cp-37-m"], abbreviated_platforms.create("linux-x86_64-cp-37-cp37m") + ) assert_platforms( ["linux-x86_64-cp-37-m", "macosx-10.13-x86_64-cp-36-cp36m"], - Platform.create("linux-x86_64-cp-37-cp37m"), - Platform.create("macosx-10.13-x86_64-cp-36-m"), + abbreviated_platforms.create("linux-x86_64-cp-37-cp37m"), + abbreviated_platforms.create("macosx-10.13-x86_64-cp-36-m"), ) @@ -351,7 +353,10 @@ def assert_local_platforms( args.extend(itertools.chain.from_iterable(("--platform", p) for p in platforms)) args.extend(extra_args or ()) targets = compute_target_configuration(parser, args) - assert tuple(Platform.create(ep) for ep in expected_platforms) == targets.platforms + assert ( + tuple(abbreviated_platforms.create(ep) for ep in expected_platforms) + == targets.platforms + ) assert_interpreters_configured(targets, expected_interpreter, expected_interpreters) assert_local_platforms( diff --git a/tests/test_pep_508.py b/tests/test_pep_508.py index e3e82fa4a..38a4caea6 100644 --- a/tests/test_pep_508.py +++ b/tests/test_pep_508.py @@ -4,7 +4,7 @@ import pytest from pex.pep_508 import MarkerEnvironment -from pex.platforms import Platform +from pex.resolve import abbreviated_platforms from pex.third_party.packaging import markers from pex.typing import TYPE_CHECKING, cast @@ -35,7 +35,7 @@ def assert_unknown_marker( def test_platform_marker_environment(): # type: () -> None - platform = Platform.create("linux-x86_64-cp-37-cp37m") + platform = abbreviated_platforms.create("linux-x86_64-cp-37-cp37m") marker_environment = MarkerEnvironment.from_platform(platform) env = marker_environment.as_dict() @@ -57,7 +57,7 @@ def assert_known_marker(expression): def test_extended_platform_marker_environment(): # type: () -> None - platform = Platform.create("linux-x86_64-cp-3.10.1-cp310") + platform = abbreviated_platforms.create("linux-x86_64-cp-3.10.1-cp310") marker_environment = MarkerEnvironment.from_platform(platform) env = marker_environment.as_dict() @@ -84,7 +84,7 @@ def assert_platform_machine( expected, # type: str platform, # type: str ): - marker_environment = MarkerEnvironment.from_platform(Platform.create(platform)) + marker_environment = MarkerEnvironment.from_platform(abbreviated_platforms.create(platform)) assert expected == marker_environment.platform_machine assert_platform_machine("x86_64", "linux-x86_64-cp-37-cp37m") diff --git a/tests/test_pex_binary.py b/tests/test_pex_binary.py index f21a08d07..41c2fe756 100644 --- a/tests/test_pex_binary.py +++ b/tests/test_pex_binary.py @@ -18,6 +18,7 @@ from pex.compatibility import to_bytes from pex.interpreter import PythonInterpreter from pex.resolve import requirement_options, resolver_options, target_options +from pex.resolve.resolver_configuration import PipConfiguration from pex.typing import TYPE_CHECKING from pex.venv.bin_path import BinPath from testing import PY39, built_wheel, ensure_python_interpreter, run_pex_command, run_simple_pex @@ -70,7 +71,7 @@ def test_clp_preamble_file(): requirement_configuration = requirement_options.configure(options) resolver_configuration = resolver_options.configure(options) - target_config = target_options.configure(options) + target_config = target_options.configure(options, pip_configuration=PipConfiguration()) targets = target_config.resolve_targets() pex_builder = build_pex( requirement_configuration=requirement_configuration, @@ -123,7 +124,7 @@ def test_clp_prereleases_resolver(): assert not options.allow_prereleases with pytest.raises(SystemExit): - target_config = target_options.configure(options) + target_config = target_options.configure(options, pip_configuration=PipConfiguration()) build_pex( requirement_configuration=requirement_options.configure(options), resolver_configuration=resolver_options.configure(options), @@ -154,7 +155,7 @@ def test_clp_prereleases_resolver(): # dep==1.2.3b1, dep # # With a correct behavior the assert line is reached and pex_builder object created. - target_config = target_options.configure(options) + target_config = target_options.configure(options, pip_configuration=PipConfiguration()) pex_builder = build_pex( requirement_configuration=requirement_options.configure(options), resolver_configuration=resolver_options.configure(options), diff --git a/tests/test_pip.py b/tests/test_pip.py index 18454b838..079def0b1 100644 --- a/tests/test_pip.py +++ b/tests/test_pip.py @@ -22,7 +22,7 @@ from pex.pip.installation import _PIP, PipInstallation, get_pip from pex.pip.tool import PackageIndexConfiguration, Pip from pex.pip.version import PipVersion, PipVersionValue -from pex.platforms import Platform +from pex.resolve import abbreviated_platforms from pex.resolve.configured_resolver import ConfiguredResolver from pex.resolve.resolver_configuration import ResolverVersion from pex.targets import AbbreviatedPlatform, LocalInterpreter, Target @@ -107,9 +107,7 @@ def test_no_duplicate_constraints_pex_warnings( ): # type: (...) -> None with warnings.catch_warnings(record=True) as events: - pip = create_pip(current_interpreter, version=version) - - pip.spawn_debug(platform=current_interpreter.platform).wait() + create_pip(current_interpreter, version=version) assert 0 == len([event for event in events if "constraints.txt" in str(event)]), ( "Expected no duplicate constraints warnings to be emitted when creating a Pip venv but " @@ -177,7 +175,7 @@ def assert_pyarrow_downloaded( assert_pyarrow_downloaded( "pyarrow-4.0.1-cp38-cp38-manylinux2010_x86_64.whl", target=AbbreviatedPlatform.create( - Platform.create("linux-x86_64-cp-38-cp38"), manylinux="manylinux2010" + abbreviated_platforms.create("linux-x86_64-cp-38-cp38", manylinux="manylinux2010") ), ) @@ -191,7 +189,7 @@ def assert_download_platform_markers_issue_1366( python310_interpreter = PythonInterpreter.from_binary(ensure_python_interpreter(PY310)) pip = create_pip(python310_interpreter, version=version) - python27_platform = Platform.create("manylinux_2_33_x86_64-cp-27-cp27mu") + python27_platform = abbreviated_platforms.create("manylinux_2_33_x86_64-cp-27-cp27mu") download_dir = os.path.join(str(tmpdir), "downloads") pip.spawn_download_distributions( target=AbbreviatedPlatform.create(python27_platform), @@ -242,7 +240,9 @@ def test_download_platform_markers_issue_1366_indeterminate( python310_interpreter = PythonInterpreter.from_binary(ensure_python_interpreter(PY310)) pip = create_pip(python310_interpreter, version=version) - target = AbbreviatedPlatform.create(Platform.create("manylinux_2_33_x86_64-cp-27-cp27mu")) + target = AbbreviatedPlatform.create( + abbreviated_platforms.create("manylinux_2_33_x86_64-cp-27-cp27mu") + ) download_dir = os.path.join(str(tmpdir), "downloads") with pytest.raises(Job.Error) as exc_info: @@ -275,9 +275,11 @@ def test_download_platform_markers_issue_1488( download_dir = os.path.join(str(tmpdir), "downloads") - python39_platform = Platform.create("linux-x86_64-cp-39-cp39") + python39_platform = abbreviated_platforms.create( + "linux-x86_64-cp-39-cp39", manylinux="manylinux2014" + ) create_pip(None, version=version).spawn_download_distributions( - target=AbbreviatedPlatform.create(python39_platform, manylinux="manylinux2014"), + target=AbbreviatedPlatform.create(python39_platform), requirements=["SQLAlchemy==1.4.25"], constraint_files=[constraints_file], download_dir=download_dir, diff --git a/tests/test_platform.py b/tests/test_platform.py index 0fbbcfd86..093df2cd6 100644 --- a/tests/test_platform.py +++ b/tests/test_platform.py @@ -8,7 +8,8 @@ import pytest from pex.pep_425 import CompatibilityTags -from pex.platforms import Platform +from pex.platforms import PlatformSpec +from pex.resolve import abbreviated_platforms from pex.third_party.packaging import tags from testing import data @@ -17,37 +18,38 @@ def test_platform(): # type: () -> None - assert Platform("linux-x86_64", "cp", "27", (2, 7), "mu") == Platform( + assert PlatformSpec("linux-x86_64", "cp", "27", (2, 7), "mu") == PlatformSpec( "linux_x86_64", "cp", "27", (2, 7), "cp27mu" ) - assert Platform("linux-x86_64", "cp", "2.7", (2, 7), "mu") == Platform( + assert PlatformSpec("linux-x86_64", "cp", "2.7", (2, 7), "mu") == PlatformSpec( "linux_x86_64", "cp", "2.7", (2, 7), "cp27mu" ) - assert str(Platform("linux-x86_64", "cp", "27", (2, 7), "m")) == "linux_x86_64-cp-27-cp27m" + assert str(PlatformSpec("linux-x86_64", "cp", "27", (2, 7), "m")) == "linux_x86_64-cp-27-cp27m" assert ( - str(Platform("linux-x86_64", "cp", "310", (3, 10), "cp310")) == "linux_x86_64-cp-310-cp310" + str(PlatformSpec("linux-x86_64", "cp", "310", (3, 10), "cp310")) + == "linux_x86_64-cp-310-cp310" ) assert ( - str(Platform("linux-x86_64", "cp", "3.10", (3, 10), "cp310")) + str(PlatformSpec("linux-x86_64", "cp", "3.10", (3, 10), "cp310")) == "linux_x86_64-cp-3.10-cp310" ) assert ( - str(Platform("linux-x86_64", "cp", "3.10.1", (3, 10, 1), "cp310")) + str(PlatformSpec("linux-x86_64", "cp", "3.10.1", (3, 10, 1), "cp310")) == "linux_x86_64-cp-3.10.1-cp310" ) def test_platform_create(): # type: () -> None - assert Platform.create("linux-x86_64-cp-27-cp27mu") == Platform( + assert PlatformSpec.parse("linux-x86_64-cp-27-cp27mu") == PlatformSpec( "linux_x86_64", "cp", "27", (2, 7), "cp27mu" ) - assert Platform.create("linux-x86_64-cp-27-mu") == Platform( + assert PlatformSpec.parse("linux-x86_64-cp-27-mu") == PlatformSpec( "linux_x86_64", "cp", "27", (2, 7), "cp27mu" ) - assert Platform.create("macosx-10.4-x86_64-cp-27-m") == Platform( + assert PlatformSpec.parse("macosx-10.4-x86_64-cp-27-m") == PlatformSpec( "macosx_10_4_x86_64", "cp", "27", @@ -58,7 +60,7 @@ def test_platform_create(): def assert_raises(platform, expected_cause): with pytest.raises( - Platform.InvalidPlatformError, + PlatformSpec.InvalidSpecError, match=( r".*{literal}.*".format( literal=re.escape( @@ -73,7 +75,7 @@ def assert_raises(platform, expected_cause): ) ), ): - Platform.create(platform) + PlatformSpec.parse(platform) def test_platform_create_bad_platform_missing_fields(): @@ -118,15 +120,9 @@ def test_platform_create_bad_platform_bad_version(): ) -def test_platform_create_noop(): - # type: () -> None - existing = Platform.create("linux-x86_64-cp-27-mu") - assert Platform.create(existing) is existing - - def test_platform_supported_tags(): # type: () -> None - platform = Platform.create("macosx-10.13-x86_64-cp-36-m") + platform = abbreviated_platforms.create("macosx-10.13-x86_64-cp-36-m") # A golden file test. This could break if we upgrade Pip and it upgrades packaging which, from # time to time, corrects omissions in tag sets. @@ -140,15 +136,26 @@ def test_platform_supported_tags(): if not tag.startswith("#") ) ) - == platform.supported_tags() + == platform.supported_tags ) def test_platform_supported_tags_manylinux(): # type: () -> None - platform = Platform.create("linux-x86_64-cp-37-cp37m") - tags = frozenset(platform.supported_tags()) - manylinux1_tags = frozenset(platform.supported_tags(manylinux="manylinux1")) - manylinux2010_tags = frozenset(platform.supported_tags(manylinux="manylinux2010")) - manylinux2014_tags = frozenset(platform.supported_tags(manylinux="manylinux2014")) + tags = frozenset(abbreviated_platforms.create("linux-x86_64-cp-37-cp37m").supported_tags) + manylinux1_tags = frozenset( + abbreviated_platforms.create( + "linux-x86_64-cp-37-cp37m", manylinux="manylinux1" + ).supported_tags + ) + manylinux2010_tags = frozenset( + abbreviated_platforms.create( + "linux-x86_64-cp-37-cp37m", manylinux="manylinux2010" + ).supported_tags + ) + manylinux2014_tags = frozenset( + abbreviated_platforms.create( + "linux-x86_64-cp-37-cp37m", manylinux="manylinux2014" + ).supported_tags + ) assert manylinux2014_tags > manylinux2010_tags > manylinux1_tags > tags diff --git a/tests/test_resolver.py b/tests/test_resolver.py index 8af21d84f..7757a820d 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -17,7 +17,7 @@ from pex.common import safe_copy, safe_mkdtemp, temporary_dir from pex.dist_metadata import Distribution, Requirement from pex.interpreter import PythonInterpreter -from pex.platforms import Platform +from pex.resolve import abbreviated_platforms from pex.resolve.configured_resolver import ConfiguredResolver from pex.resolve.resolver_configuration import BuildConfiguration, ResolverVersion from pex.resolve.resolvers import ResolvedDistribution, ResolveResult, Unsatisfiable @@ -335,7 +335,7 @@ def resolve_current_and_foreign(interpreters=()): # N.B.: None stands in for the "current" platform at higher layers that parse platform # strings to Platform objects. - platforms = (None, Platform.create(foreign_platform)) + platforms = (None, abbreviated_platforms.create(foreign_platform)) return resolve_p537_wheel_names( cache_dir=p537_resolve_cache, targets=Targets(platforms=platforms, interpreters=tuple(interpreters)), @@ -356,14 +356,14 @@ def resolve_current_and_foreign(interpreters=()): assert 3 == len(resolve_current_and_foreign(interpreters=[current_python, other_python])) -def test_resolve_foreign_abi3(): - # type: () -> None +def test_resolve_foreign_abi3(tmpdir): + # type: (Any) -> None # For version 2.8, cryptography publishes the following abi3 wheels for linux and macosx: # cryptography-2.8-cp34-abi3-macosx_10_6_intel.whl # cryptography-2.8-cp34-abi3-manylinux1_x86_64.whl # cryptography-2.8-cp34-abi3-manylinux2010_x86_64.whl - cryptogrpahy_resolve_cache = safe_mkdtemp() + cryptogrpahy_resolve_cache = os.path.join(str(tmpdir), "pex_root") foreign_ver = "37" if PY_VER == (3, 6) else "36" def resolve_cryptography_wheel_names(manylinux): @@ -372,10 +372,13 @@ def resolve_cryptography_wheel_names(manylinux): requirements=["cryptography==2.8"], targets=Targets( platforms=( - Platform.create("linux_x86_64-cp-{}-m".format(foreign_ver)), - Platform.create("macosx_10.11_x86_64-cp-{}-m".format(foreign_ver)), + abbreviated_platforms.create( + "linux_x86_64-cp-{}-m".format(foreign_ver), manylinux=manylinux + ), + abbreviated_platforms.create( + "macosx_10.11_x86_64-cp-{}-m".format(foreign_ver), manylinux=manylinux + ), ), - assume_manylinux=manylinux, ), transitive=False, build_configuration=BuildConfiguration.create(allow_builds=False), diff --git a/tests/test_targets.py b/tests/test_targets.py index de024156a..a26e35fba 100644 --- a/tests/test_targets.py +++ b/tests/test_targets.py @@ -14,6 +14,7 @@ from pex.pep_425 import CompatibilityTags from pex.pep_508 import MarkerEnvironment from pex.platforms import Platform +from pex.resolve import abbreviated_platforms from pex.targets import ( AbbreviatedPlatform, CompletePlatform, @@ -156,7 +157,9 @@ def assert_requires_python( def test_requires_python_abbreviated_platform(): - abbreviated_platform = AbbreviatedPlatform.create(Platform.create("linux-x86_64-cp-37-m")) + abbreviated_platform = AbbreviatedPlatform.create( + abbreviated_platforms.create("linux-x86_64-cp-37-m") + ) def assert_requires_python( expected_result, # type: bool @@ -245,9 +248,7 @@ def test_from_target_local_interpreter(): def test_from_target_abbreviated_platform(current_platform): # type: (Platform) -> None - abbreviated_platform = AbbreviatedPlatform.create( - platform=current_platform, manylinux="manylinux1" - ) + abbreviated_platform = AbbreviatedPlatform.create(platform=current_platform) tgts = Targets.from_target(abbreviated_platform) assert tgts.interpreter is None assert OrderedSet([abbreviated_platform]) == tgts.unique_targets(only_explicit=False) @@ -312,6 +313,7 @@ def test_compatible_shebang( version_info=incompatible_version_info, version=".".join(map(str, incompatible_version_info)), abi=current_platform.abi, + supported_tags=current_platform.supported_tags, ) assert ( Targets(platforms=(current_platform, incompatible_platform_version)).compatible_shebang() @@ -324,6 +326,7 @@ def test_compatible_shebang( version_info=current_platform.version_info, version=current_platform.version, abi=current_platform.abi, + supported_tags=current_platform.supported_tags, ) assert ( Targets(platforms=(current_platform, incompatible_platform_impl)).compatible_shebang()