Skip to content

Commit

Permalink
Plumb --pip-version to Platform tag calculation. (#2538)
Browse files Browse the repository at this point in the history
This fixes a long-standing discrepency between the selected
`--pip-version` and the Pip actually used to determine abbreviated
`--platform` compatibility tags. With the `--pip-log` enhancement
from #2536 the calculated abbreviated platform tags are now always
logged to the Pip log, which is very useful for debugging failed
resolves using abbreviated platforms.

Fixes #1894
  • Loading branch information
jsirois authored Sep 18, 2024
1 parent 315c56a commit 85bdc77
Show file tree
Hide file tree
Showing 32 changed files with 660 additions and 489 deletions.
46 changes: 23 additions & 23 deletions pex/bin/pex.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,9 +53,8 @@
from pex.resolve.resolver_configuration import (
LockRepositoryConfiguration,
PexRepositoryConfiguration,
PreResolvedConfiguration,
PipConfiguration,
)
from pex.resolve.resolver_options import create_pip_configuration
from pex.resolve.resolvers import Unsatisfiable, sorted_requirements
from pex.result import Error, ResultError, catch, try_
from pex.scie import ScieConfiguration
Expand Down Expand Up @@ -1014,20 +1013,11 @@ def build_pex(
DependencyConfiguration.from_pex_info(requirements_pex_info)
)

if isinstance(resolver_configuration, (LockRepositoryConfiguration, PreResolvedConfiguration)):
pip_configuration = resolver_configuration.pip_configuration
elif isinstance(resolver_configuration, PexRepositoryConfiguration):
# TODO(John Sirois): Consider finding a way to support custom --index and --find-links in
# this case. I.E.: I use a corporate index to build a PEX repository and now I want to
# build a --project PEX whose pyproject.toml build-system.requires should be resolved from
# that corporate index.
pip_configuration = try_(
finalize_resolve_config(
create_pip_configuration(options), targets=targets, context="--project building"
)
)
else:
pip_configuration = resolver_configuration
pip_configuration = (
resolver_configuration
if isinstance(resolver_configuration, PipConfiguration)
else resolver_configuration.pip_configuration
)

project_dependencies = OrderedSet() # type: OrderedSet[Requirement]
with TRACER.timed(
Expand Down Expand Up @@ -1157,11 +1147,14 @@ def _compatible_with_current_platform(interpreter, platforms):
return current_platforms.intersection(platforms)


def configure_requirements_and_targets(options):
# type: (Namespace) -> Union[Tuple[RequirementConfiguration, InterpreterConstraints, Targets], Error]
def configure_requirements_and_targets(
options, # type: Namespace
pip_configuration, # type: PipConfiguration
):
# type: (...) -> Union[Tuple[RequirementConfiguration, InterpreterConstraints, Targets], Error]

requirement_configuration = requirement_options.configure(options)
target_config = target_options.configure(options)
target_config = target_options.configure(options, pip_configuration=pip_configuration)
script_metadata = ScriptMetadata()

if options.executable and options.enable_script_metadata:
Expand Down Expand Up @@ -1260,15 +1253,22 @@ def main(args=None):

try:
with global_environment(options) as env:
requirement_configuration, interpreter_constraints, targets = try_(
configure_requirements_and_targets(options)
)

try:
resolver_configuration = resolver_options.configure(options)
except resolver_options.InvalidConfigurationError as e:
die(str(e))

requirement_configuration, interpreter_constraints, targets = try_(
configure_requirements_and_targets(
options,
pip_configuration=(
resolver_configuration
if isinstance(resolver_configuration, PipConfiguration)
else resolver_configuration.pip_configuration
),
)
)

resolver_configuration = try_(
finalize_resolve_config(resolver_configuration, targets, context="PEX building")
)
Expand Down
64 changes: 41 additions & 23 deletions pex/cli/commands/lock.py
Original file line number Diff line number Diff line change
Expand Up @@ -549,7 +549,12 @@ def _add_export_arguments(
cls.add_output_option(export_parser, entity="lock")
cls._add_target_options(export_parser)
resolver_options_parser = cls._create_resolver_options_group(export_parser)
resolver_options.register_network_options(resolver_options_parser)
resolver_options.register(
resolver_options_parser,
include_pex_repository=False,
include_lock=False,
include_pre_resolved=False,
)

@classmethod
def _add_export_subset_arguments(cls, export_subset_parser):
Expand Down Expand Up @@ -652,11 +657,12 @@ def _add_update_arguments(cls, update_parser):
cls.add_json_options(update_parser, entity="lock", include_switch=False)
cls._add_target_options(update_parser)
resolver_options_parser = cls._create_resolver_options_group(update_parser)
resolver_options.register_repos_options(resolver_options_parser)
resolver_options.register_network_options(resolver_options_parser)
resolver_options.register_max_jobs_option(resolver_options_parser)
resolver_options.register_use_pip_config(resolver_options_parser)
resolver_options.register_pip_log(resolver_options_parser)
resolver_options.register(
resolver_options_parser,
include_pex_repository=False,
include_lock=False,
include_pre_resolved=False,
)

@classmethod
def add_update_lock_options(
Expand Down Expand Up @@ -792,7 +798,9 @@ def _resolve_targets(
):
# type: (...) -> Union[Targets, Error]

target_config = target_configuration or target_options.configure(self.options)
target_config = target_configuration or target_options.configure(
self.options, pip_configuration=resolver_options.create_pip_configuration(self.options)
)
if style is not LockStyle.UNIVERSAL:
return target_config.resolve_targets()

Expand All @@ -812,7 +820,6 @@ def _resolve_targets(
return Targets(
platforms=target_config.platforms,
complete_platforms=target_config.complete_platforms,
assume_manylinux=target_config.assume_manylinux,
)

try:
Expand All @@ -827,7 +834,6 @@ def _resolve_targets(
interpreters=(interpreter,),
platforms=target_config.platforms,
complete_platforms=target_config.complete_platforms,
assume_manylinux=target_config.assume_manylinux,
)

def _gather_requirements(
Expand Down Expand Up @@ -860,7 +866,11 @@ def _gather_requirements(

def _create(self):
# type: () -> Result
target_configuration = target_options.configure(self.options)

pip_configuration = resolver_options.create_pip_configuration(self.options)
target_configuration = target_options.configure(
self.options, pip_configuration=pip_configuration
)
if self.options.style == LockStyle.UNIVERSAL:
lock_configuration = LockConfiguration(
style=LockStyle.UNIVERSAL,
Expand Down Expand Up @@ -888,7 +898,7 @@ def _create(self):
)
pip_configuration = try_(
finalize_resolve_config(
resolver_configuration=resolver_options.create_pip_configuration(self.options),
resolver_configuration=pip_configuration,
targets=targets,
context="lock creation",
)
Expand Down Expand Up @@ -953,19 +963,21 @@ def _export(self, requirement_configuration=RequirementConfiguration()):
)

lockfile_path, lock_file = self._load_lockfile()
targets = target_options.configure(self.options).resolve_targets()
pip_configuration = resolver_options.create_pip_configuration(self.options)
targets = target_options.configure(
self.options, pip_configuration=pip_configuration
).resolve_targets()
target = targets.require_unique_target(
purpose="exporting a lock in the {pip!r} format".format(pip=ExportFormat.PIP)
)

network_configuration = resolver_options.create_network_configuration(self.options)
with TRACER.timed("Selecting locks for {target}".format(target=target)):
subset_result = try_(
subset(
targets=targets,
lock=lock_file,
requirement_configuration=requirement_configuration,
network_configuration=network_configuration,
network_configuration=pip_configuration.network_configuration,
build_configuration=lock_file.build_configuration(),
transitive=lock_file.transitive,
include_all_matches=True,
Expand Down Expand Up @@ -1079,18 +1091,20 @@ def _create_lock_update_request(
):
# type: (...) -> Union[LockUpdateRequest, Error]

network_configuration = resolver_options.create_network_configuration(self.options)
pip_configuration = resolver_options.create_pip_configuration(self.options)
lock_updater = LockUpdater.create(
lock_file=lock_file,
repos_configuration=resolver_options.create_repos_configuration(self.options),
network_configuration=network_configuration,
max_jobs=resolver_options.get_max_jobs_value(self.options),
use_pip_config=resolver_options.get_use_pip_config_value(self.options),
repos_configuration=pip_configuration.repos_configuration,
network_configuration=pip_configuration.network_configuration,
max_jobs=pip_configuration.max_jobs,
use_pip_config=pip_configuration.use_pip_config,
dependency_configuration=dependency_config,
pip_log=resolver_options.get_pip_log(self.options),
)

target_configuration = target_options.configure(self.options)
target_configuration = target_options.configure(
self.options, pip_configuration=pip_configuration
)
targets = try_(
self._resolve_targets(
action="updating", style=lock_file.style, target_configuration=target_configuration
Expand Down Expand Up @@ -1146,7 +1160,7 @@ def _create_lock_update_request(
subset(
targets=targets,
lock=lock_file,
network_configuration=network_configuration,
network_configuration=pip_configuration.network_configuration,
build_configuration=lock_file.build_configuration(),
transitive=lock_file.transitive,
)
Expand Down Expand Up @@ -1498,7 +1512,9 @@ def _sync(self):
pip_configuration = resolver_configuration.pip_configuration
dependency_config = dependency_configuration.configure(self.options)

target_configuration = target_options.configure(self.options)
target_configuration = target_options.configure(
self.options, pip_configuration=pip_configuration
)
if self.options.style == LockStyle.UNIVERSAL:
lock_configuration = LockConfiguration(
style=LockStyle.UNIVERSAL,
Expand Down Expand Up @@ -1613,7 +1629,9 @@ def _sync(self):
else PythonInterpreter.from_env(self.options.venv_python)
)
else:
targets = target_options.configure(self.options).resolve_targets()
targets = target_options.configure(
self.options, pip_configuration=pip_configuration
).resolve_targets()
interpreters = [
target.get_interpreter()
for target in targets.unique_targets()
Expand Down
12 changes: 10 additions & 2 deletions pex/cli/commands/venv.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
from pex.resolve.resolver_configuration import (
LockRepositoryConfiguration,
PexRepositoryConfiguration,
PipConfiguration,
)
from pex.result import Error, Ok, Result, try_
from pex.targets import LocalInterpreter, Target, Targets
Expand Down Expand Up @@ -200,7 +201,15 @@ def _inspect(self):
def _create(self):
# type: () -> Result

targets = target_options.configure(self.options).resolve_targets()
resolver_configuration = resolver_options.configure(self.options)
targets = target_options.configure(
self.options,
pip_configuration=(
resolver_configuration
if isinstance(resolver_configuration, PipConfiguration)
else resolver_configuration.pip_configuration
),
).resolve_targets()
installer_configuration = installer_options.configure(self.options)

dest_dir = (
Expand Down Expand Up @@ -266,7 +275,6 @@ def _create(self):
)

requirement_configuration = requirement_options.configure(self.options)
resolver_configuration = resolver_options.configure(self.options)
with TRACER.timed("Resolving distributions"):
resolved = configured_resolve.resolve(
targets=targets,
Expand Down
5 changes: 3 additions & 2 deletions pex/interpreter.py
Original file line number Diff line number Diff line change
Expand Up @@ -545,9 +545,10 @@ def iter_supported_platforms(self):
version=self.version_str,
version_info=self.version,
abi=self.abi_tag,
supported_tags=self._supported_tags,
)
for tag in self._supported_tags:
yield Platform.from_tag(tag)
for index in range(len(self._supported_tags)):
yield Platform.from_tags(self._supported_tags[index:])

def binary_name(self, version_components=2):
# type: (int) -> str
Expand Down
34 changes: 23 additions & 11 deletions pex/pep_425.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ def from_strings(cls, tags):
return cls(tags=tuple(itertools.chain.from_iterable(parse_tag(tag) for tag in tags)))

_tags = attr.ib(converter=_prepare_tags) # type: Tuple[Tag, ...]
__rankings = attr.ib(eq=False, factory=dict) # type: MutableMapping[Tag, TagRank]
_rankings = attr.ib(eq=False, factory=dict) # type: MutableMapping[Tag, TagRank]

@_tags.validator
def _validate_tags(
Expand Down Expand Up @@ -120,11 +120,11 @@ def to_string_list(self):
return [str(tag) for tag in self._tags]

@property
def _rankings(self):
def __rankings(self):
# type: () -> Mapping[Tag, TagRank]
if not self.__rankings:
self.__rankings.update(TagRank.ranked(self._tags))
return self.__rankings
if not self._rankings:
self._rankings.update(TagRank.ranked(self._tags))
return self._rankings

@property
def lowest_rank(self):
Expand All @@ -133,7 +133,7 @@ def lowest_rank(self):

def rank(self, tag):
# type: (Tag) -> Optional[TagRank]
return self._rankings.get(tag)
return self.__rankings.get(tag)

def best_match(self, tags):
# type: (Iterable[Tag]) -> Optional[RankedTag]
Expand All @@ -160,18 +160,30 @@ def __getitem__(self, index):
# type: (int) -> Tag
pass

# MyPy claims this overload collides with the one below even though slice / Tag are disjoint
# input types and CompatibilityTags / TagRank are disjoint return types; thus the ignore[misc].
@overload
def __getitem__(self, slice_): # type: ignore[misc]
# type: (slice) -> CompatibilityTags
pass

@overload
def __getitem__(self, tag):
# type: (Tag) -> TagRank
pass

def __getitem__(self, index_or_tag):
# type: (Union[int, Tag]) -> Union[Tag, TagRank]
def __getitem__(self, index_or_slice_or_tag):
# type: (Union[int, slice, Tag]) -> Union[Tag, CompatibilityTags, TagRank]
"""Retrieve tag by its rank or a tags rank.
Ranks are 0-based with the 0-rank tag being the most specific (best match).
"""
if isinstance(index_or_tag, Tag):
return self._rankings[index_or_tag]
if isinstance(index_or_slice_or_tag, Tag):
return self.__rankings[index_or_slice_or_tag]
elif isinstance(index_or_slice_or_tag, slice):
tags = self._tags[index_or_slice_or_tag]
return CompatibilityTags(
tags=tags, rankings={tag: self.__rankings[tag] for tag in tags}
)
else:
return self._tags[index_or_tag]
return self._tags[index_or_slice_or_tag]
Loading

0 comments on commit 85bdc77

Please sign in to comment.