Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Plumb --pip-version to Platform tag calculation. #2538

Merged
merged 4 commits into from
Sep 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 23 additions & 23 deletions pex/bin/pex.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,9 +53,8 @@
from pex.resolve.resolver_configuration import (
LockRepositoryConfiguration,
PexRepositoryConfiguration,
PreResolvedConfiguration,
PipConfiguration,
)
from pex.resolve.resolver_options import create_pip_configuration
from pex.resolve.resolvers import Unsatisfiable, sorted_requirements
from pex.result import Error, ResultError, catch, try_
from pex.scie import ScieConfiguration
Expand Down Expand Up @@ -1014,20 +1013,11 @@ def build_pex(
DependencyConfiguration.from_pex_info(requirements_pex_info)
)

if isinstance(resolver_configuration, (LockRepositoryConfiguration, PreResolvedConfiguration)):
pip_configuration = resolver_configuration.pip_configuration
elif isinstance(resolver_configuration, PexRepositoryConfiguration):
# TODO(John Sirois): Consider finding a way to support custom --index and --find-links in
# this case. I.E.: I use a corporate index to build a PEX repository and now I want to
# build a --project PEX whose pyproject.toml build-system.requires should be resolved from
# that corporate index.
pip_configuration = try_(
finalize_resolve_config(
create_pip_configuration(options), targets=targets, context="--project building"
)
)
else:
pip_configuration = resolver_configuration
pip_configuration = (
resolver_configuration
if isinstance(resolver_configuration, PipConfiguration)
else resolver_configuration.pip_configuration
)

project_dependencies = OrderedSet() # type: OrderedSet[Requirement]
with TRACER.timed(
Expand Down Expand Up @@ -1157,11 +1147,14 @@ def _compatible_with_current_platform(interpreter, platforms):
return current_platforms.intersection(platforms)


def configure_requirements_and_targets(options):
# type: (Namespace) -> Union[Tuple[RequirementConfiguration, InterpreterConstraints, Targets], Error]
def configure_requirements_and_targets(
options, # type: Namespace
pip_configuration, # type: PipConfiguration
):
# type: (...) -> Union[Tuple[RequirementConfiguration, InterpreterConstraints, Targets], Error]

requirement_configuration = requirement_options.configure(options)
target_config = target_options.configure(options)
target_config = target_options.configure(options, pip_configuration=pip_configuration)
script_metadata = ScriptMetadata()

if options.executable and options.enable_script_metadata:
Expand Down Expand Up @@ -1260,15 +1253,22 @@ def main(args=None):

try:
with global_environment(options) as env:
requirement_configuration, interpreter_constraints, targets = try_(
configure_requirements_and_targets(options)
)

try:
resolver_configuration = resolver_options.configure(options)
except resolver_options.InvalidConfigurationError as e:
die(str(e))

requirement_configuration, interpreter_constraints, targets = try_(
configure_requirements_and_targets(
options,
pip_configuration=(
resolver_configuration
if isinstance(resolver_configuration, PipConfiguration)
else resolver_configuration.pip_configuration
),
)
)

resolver_configuration = try_(
finalize_resolve_config(resolver_configuration, targets, context="PEX building")
)
Expand Down
64 changes: 41 additions & 23 deletions pex/cli/commands/lock.py
Original file line number Diff line number Diff line change
Expand Up @@ -549,7 +549,12 @@ def _add_export_arguments(
cls.add_output_option(export_parser, entity="lock")
cls._add_target_options(export_parser)
resolver_options_parser = cls._create_resolver_options_group(export_parser)
resolver_options.register_network_options(resolver_options_parser)
resolver_options.register(
resolver_options_parser,
include_pex_repository=False,
include_lock=False,
include_pre_resolved=False,
)

@classmethod
def _add_export_subset_arguments(cls, export_subset_parser):
Expand Down Expand Up @@ -652,11 +657,12 @@ def _add_update_arguments(cls, update_parser):
cls.add_json_options(update_parser, entity="lock", include_switch=False)
cls._add_target_options(update_parser)
resolver_options_parser = cls._create_resolver_options_group(update_parser)
resolver_options.register_repos_options(resolver_options_parser)
resolver_options.register_network_options(resolver_options_parser)
resolver_options.register_max_jobs_option(resolver_options_parser)
resolver_options.register_use_pip_config(resolver_options_parser)
resolver_options.register_pip_log(resolver_options_parser)
resolver_options.register(
resolver_options_parser,
include_pex_repository=False,
include_lock=False,
include_pre_resolved=False,
)

@classmethod
def add_update_lock_options(
Expand Down Expand Up @@ -792,7 +798,9 @@ def _resolve_targets(
):
# type: (...) -> Union[Targets, Error]

target_config = target_configuration or target_options.configure(self.options)
target_config = target_configuration or target_options.configure(
self.options, pip_configuration=resolver_options.create_pip_configuration(self.options)
)
if style is not LockStyle.UNIVERSAL:
return target_config.resolve_targets()

Expand All @@ -812,7 +820,6 @@ def _resolve_targets(
return Targets(
platforms=target_config.platforms,
complete_platforms=target_config.complete_platforms,
assume_manylinux=target_config.assume_manylinux,
)

try:
Expand All @@ -827,7 +834,6 @@ def _resolve_targets(
interpreters=(interpreter,),
platforms=target_config.platforms,
complete_platforms=target_config.complete_platforms,
assume_manylinux=target_config.assume_manylinux,
)

def _gather_requirements(
Expand Down Expand Up @@ -860,7 +866,11 @@ def _gather_requirements(

def _create(self):
# type: () -> Result
target_configuration = target_options.configure(self.options)

pip_configuration = resolver_options.create_pip_configuration(self.options)
target_configuration = target_options.configure(
self.options, pip_configuration=pip_configuration
)
if self.options.style == LockStyle.UNIVERSAL:
lock_configuration = LockConfiguration(
style=LockStyle.UNIVERSAL,
Expand Down Expand Up @@ -888,7 +898,7 @@ def _create(self):
)
pip_configuration = try_(
finalize_resolve_config(
resolver_configuration=resolver_options.create_pip_configuration(self.options),
resolver_configuration=pip_configuration,
targets=targets,
context="lock creation",
)
Expand Down Expand Up @@ -953,19 +963,21 @@ def _export(self, requirement_configuration=RequirementConfiguration()):
)

lockfile_path, lock_file = self._load_lockfile()
targets = target_options.configure(self.options).resolve_targets()
pip_configuration = resolver_options.create_pip_configuration(self.options)
targets = target_options.configure(
self.options, pip_configuration=pip_configuration
).resolve_targets()
target = targets.require_unique_target(
purpose="exporting a lock in the {pip!r} format".format(pip=ExportFormat.PIP)
)

network_configuration = resolver_options.create_network_configuration(self.options)
with TRACER.timed("Selecting locks for {target}".format(target=target)):
subset_result = try_(
subset(
targets=targets,
lock=lock_file,
requirement_configuration=requirement_configuration,
network_configuration=network_configuration,
network_configuration=pip_configuration.network_configuration,
build_configuration=lock_file.build_configuration(),
transitive=lock_file.transitive,
include_all_matches=True,
Expand Down Expand Up @@ -1079,18 +1091,20 @@ def _create_lock_update_request(
):
# type: (...) -> Union[LockUpdateRequest, Error]

network_configuration = resolver_options.create_network_configuration(self.options)
pip_configuration = resolver_options.create_pip_configuration(self.options)
lock_updater = LockUpdater.create(
lock_file=lock_file,
repos_configuration=resolver_options.create_repos_configuration(self.options),
network_configuration=network_configuration,
max_jobs=resolver_options.get_max_jobs_value(self.options),
use_pip_config=resolver_options.get_use_pip_config_value(self.options),
repos_configuration=pip_configuration.repos_configuration,
network_configuration=pip_configuration.network_configuration,
max_jobs=pip_configuration.max_jobs,
use_pip_config=pip_configuration.use_pip_config,
dependency_configuration=dependency_config,
pip_log=resolver_options.get_pip_log(self.options),
)

target_configuration = target_options.configure(self.options)
target_configuration = target_options.configure(
self.options, pip_configuration=pip_configuration
)
targets = try_(
self._resolve_targets(
action="updating", style=lock_file.style, target_configuration=target_configuration
Expand Down Expand Up @@ -1146,7 +1160,7 @@ def _create_lock_update_request(
subset(
targets=targets,
lock=lock_file,
network_configuration=network_configuration,
network_configuration=pip_configuration.network_configuration,
build_configuration=lock_file.build_configuration(),
transitive=lock_file.transitive,
)
Expand Down Expand Up @@ -1498,7 +1512,9 @@ def _sync(self):
pip_configuration = resolver_configuration.pip_configuration
dependency_config = dependency_configuration.configure(self.options)

target_configuration = target_options.configure(self.options)
target_configuration = target_options.configure(
self.options, pip_configuration=pip_configuration
)
if self.options.style == LockStyle.UNIVERSAL:
lock_configuration = LockConfiguration(
style=LockStyle.UNIVERSAL,
Expand Down Expand Up @@ -1613,7 +1629,9 @@ def _sync(self):
else PythonInterpreter.from_env(self.options.venv_python)
)
else:
targets = target_options.configure(self.options).resolve_targets()
targets = target_options.configure(
self.options, pip_configuration=pip_configuration
).resolve_targets()
interpreters = [
target.get_interpreter()
for target in targets.unique_targets()
Expand Down
12 changes: 10 additions & 2 deletions pex/cli/commands/venv.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
from pex.resolve.resolver_configuration import (
LockRepositoryConfiguration,
PexRepositoryConfiguration,
PipConfiguration,
)
from pex.result import Error, Ok, Result, try_
from pex.targets import LocalInterpreter, Target, Targets
Expand Down Expand Up @@ -200,7 +201,15 @@ def _inspect(self):
def _create(self):
# type: () -> Result

targets = target_options.configure(self.options).resolve_targets()
resolver_configuration = resolver_options.configure(self.options)
targets = target_options.configure(
self.options,
pip_configuration=(
resolver_configuration
if isinstance(resolver_configuration, PipConfiguration)
else resolver_configuration.pip_configuration
),
).resolve_targets()
installer_configuration = installer_options.configure(self.options)

dest_dir = (
Expand Down Expand Up @@ -266,7 +275,6 @@ def _create(self):
)

requirement_configuration = requirement_options.configure(self.options)
resolver_configuration = resolver_options.configure(self.options)
with TRACER.timed("Resolving distributions"):
resolved = configured_resolve.resolve(
targets=targets,
Expand Down
5 changes: 3 additions & 2 deletions pex/interpreter.py
Original file line number Diff line number Diff line change
Expand Up @@ -545,9 +545,10 @@ def iter_supported_platforms(self):
version=self.version_str,
version_info=self.version,
abi=self.abi_tag,
supported_tags=self._supported_tags,
)
for tag in self._supported_tags:
yield Platform.from_tag(tag)
for index in range(len(self._supported_tags)):
yield Platform.from_tags(self._supported_tags[index:])

def binary_name(self, version_components=2):
# type: (int) -> str
Expand Down
34 changes: 23 additions & 11 deletions pex/pep_425.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ def from_strings(cls, tags):
return cls(tags=tuple(itertools.chain.from_iterable(parse_tag(tag) for tag in tags)))

_tags = attr.ib(converter=_prepare_tags) # type: Tuple[Tag, ...]
__rankings = attr.ib(eq=False, factory=dict) # type: MutableMapping[Tag, TagRank]
_rankings = attr.ib(eq=False, factory=dict) # type: MutableMapping[Tag, TagRank]

@_tags.validator
def _validate_tags(
Expand Down Expand Up @@ -120,11 +120,11 @@ def to_string_list(self):
return [str(tag) for tag in self._tags]

@property
def _rankings(self):
def __rankings(self):
# type: () -> Mapping[Tag, TagRank]
if not self.__rankings:
self.__rankings.update(TagRank.ranked(self._tags))
return self.__rankings
if not self._rankings:
self._rankings.update(TagRank.ranked(self._tags))
return self._rankings

@property
def lowest_rank(self):
Expand All @@ -133,7 +133,7 @@ def lowest_rank(self):

def rank(self, tag):
# type: (Tag) -> Optional[TagRank]
return self._rankings.get(tag)
return self.__rankings.get(tag)

def best_match(self, tags):
# type: (Iterable[Tag]) -> Optional[RankedTag]
Expand All @@ -160,18 +160,30 @@ def __getitem__(self, index):
# type: (int) -> Tag
pass

# MyPy claims this overload collides with the one below even though slice / Tag are disjoint
# input types and CompatibilityTags / TagRank are disjoint return types; thus the ignore[misc].
@overload
def __getitem__(self, slice_): # type: ignore[misc]
# type: (slice) -> CompatibilityTags
pass

@overload
def __getitem__(self, tag):
# type: (Tag) -> TagRank
pass

def __getitem__(self, index_or_tag):
# type: (Union[int, Tag]) -> Union[Tag, TagRank]
def __getitem__(self, index_or_slice_or_tag):
# type: (Union[int, slice, Tag]) -> Union[Tag, CompatibilityTags, TagRank]
"""Retrieve tag by its rank or a tags rank.

Ranks are 0-based with the 0-rank tag being the most specific (best match).
"""
if isinstance(index_or_tag, Tag):
return self._rankings[index_or_tag]
if isinstance(index_or_slice_or_tag, Tag):
return self.__rankings[index_or_slice_or_tag]
elif isinstance(index_or_slice_or_tag, slice):
tags = self._tags[index_or_slice_or_tag]
return CompatibilityTags(
tags=tags, rankings={tag: self.__rankings[tag] for tag in tags}
)
else:
return self._tags[index_or_tag]
return self._tags[index_or_slice_or_tag]
Loading