Skip to content

Commit

Permalink
Factor out configured resolves from pex.
Browse files Browse the repository at this point in the history 
This code as well as selection of a unique target to resolve for will
be re-used by the forthcoming `pex3 venv create` command.

Along the way, clean up confusion noted in pex-tool#2135 in the introduction
of the `pex3 venv` subcommand surrounding `pyvenv.cfg` files generated
by `Virtualenv`.

Work towards pex-tool#1752 and pex-tool#2110.
  • Loading branch information
@jsirois
jsirois committed Apr 28, 2023
1 parent f1646b7 commit 0768b82
Show file tree
Hide file tree
Showing 5 changed files with 144 additions and 102 deletions.
93 changes: 9 additions & 84 deletions pex/bin/pex.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,17 +35,10 @@
from pex.pex_info import PexInfo
from pex.resolve import requirement_options, resolver_options, target_configuration, target_options
from pex.resolve.config import finalize as finalize_resolve_config
from pex.resolve.configured_resolver import ConfiguredResolver
from pex.resolve.lock_resolver import resolve_from_lock
from pex.resolve.pex_repository_resolver import resolve_from_pex
from pex.resolve.configured_resolve import resolve
from pex.resolve.requirement_configuration import RequirementConfiguration
from pex.resolve.resolver_configuration import (
LockRepositoryConfiguration,
PexRepositoryConfiguration,
)
from pex.resolve.resolvers import Unsatisfiable
from pex.resolver import resolve
from pex.result import catch, try_
from pex.result import catch
from pex.targets import Targets
from pex.tracer import TRACER
from pex.typing import TYPE_CHECKING, cast
Expand Down Expand Up @@ -677,81 +670,13 @@ def build_pex(
)
):
try:
if isinstance(resolver_configuration, LockRepositoryConfiguration):
lock = try_(resolver_configuration.parse_lock())
with TRACER.timed(
"Resolving requirements from lock file {lock_file}".format(
lock_file=lock.source
)
):
pip_configuration = resolver_configuration.pip_configuration
result = try_(
resolve_from_lock(
targets=targets,
lock=lock,
resolver=ConfiguredResolver(pip_configuration=pip_configuration),
requirements=requirement_configuration.requirements,
requirement_files=requirement_configuration.requirement_files,
constraint_files=requirement_configuration.constraint_files,
transitive=pip_configuration.transitive,
indexes=pip_configuration.repos_configuration.indexes,
find_links=pip_configuration.repos_configuration.find_links,
resolver_version=pip_configuration.resolver_version,
network_configuration=pip_configuration.network_configuration,
password_entries=pip_configuration.repos_configuration.password_entries,
build=pip_configuration.allow_builds,
use_wheel=pip_configuration.allow_wheels,
prefer_older_binary=pip_configuration.prefer_older_binary,
use_pep517=pip_configuration.use_pep517,
build_isolation=pip_configuration.build_isolation,
compile=options.compile,
max_parallel_jobs=pip_configuration.max_jobs,
pip_version=lock.pip_version,
)
)
elif isinstance(resolver_configuration, PexRepositoryConfiguration):
with TRACER.timed(
"Resolving requirements from PEX {pex_repository}.".format(
pex_repository=resolver_configuration.pex_repository
)
):
result = resolve_from_pex(
targets=targets,
pex=resolver_configuration.pex_repository,
requirements=requirement_configuration.requirements,
requirement_files=requirement_configuration.requirement_files,
constraint_files=requirement_configuration.constraint_files,
network_configuration=resolver_configuration.network_configuration,
transitive=resolver_configuration.transitive,
ignore_errors=options.ignore_errors,
)
else:
with TRACER.timed("Resolving requirements."):
result = resolve(
targets=targets,
requirements=requirement_configuration.requirements,
requirement_files=requirement_configuration.requirement_files,
constraint_files=requirement_configuration.constraint_files,
allow_prereleases=resolver_configuration.allow_prereleases,
transitive=resolver_configuration.transitive,
indexes=resolver_configuration.repos_configuration.indexes,
find_links=resolver_configuration.repos_configuration.find_links,
resolver_version=resolver_configuration.resolver_version,
network_configuration=resolver_configuration.network_configuration,
password_entries=resolver_configuration.repos_configuration.password_entries,
build=resolver_configuration.allow_builds,
use_wheel=resolver_configuration.allow_wheels,
prefer_older_binary=resolver_configuration.prefer_older_binary,
use_pep517=resolver_configuration.use_pep517,
build_isolation=resolver_configuration.build_isolation,
compile=options.compile,
max_parallel_jobs=resolver_configuration.max_jobs,
ignore_errors=options.ignore_errors,
preserve_log=resolver_configuration.preserve_log,
pip_version=resolver_configuration.version,
resolver=ConfiguredResolver(pip_configuration=resolver_configuration),
)

result = resolve(
targets=targets,
requirement_configuration=requirement_configuration,
resolver_configuration=resolver_configuration,
compile_pyc=options.compile,
ignore_errors=options.ignore_errors,
)
for installed_dist in result.installed_distributions:
pex_builder.add_distribution(
installed_dist.distribution, fingerprint=installed_dist.fingerprint
Expand Down
18 changes: 3 additions & 15 deletions pex/cli/commands/lock.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -499,21 +499,9 @@ def _export(self):

lockfile_path, lock_file = self._load_lockfile()
targets = target_options.configure(self.options).resolve_targets()
resolved_targets = targets.unique_targets()
if len(resolved_targets) > 1:
return Error(
"A lock can only be exported for a single target in the {pip!r} format.\n"
"There were {count} targets selected:\n"
"{targets}".format(
pip=ExportFormat.PIP,
count=len(resolved_targets),
targets="\n".join(
"{index}. {target}".format(index=index, target=target)
for index, target in enumerate(resolved_targets, start=1)
),
)
)
target = next(iter(resolved_targets))
target = targets.require_unique_target(
purpose="exporting a lock in the {pip!r} format".format(pip=ExportFormat.PIP)
)

network_configuration = resolver_options.create_network_configuration(self.options)
with TRACER.timed("Selecting locks for {target}".format(target=target)):
Expand Down
104 changes: 104 additions & 0 deletions pex/resolve/configured_resolve.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import absolute_import

from pex.resolve.configured_resolver import ConfiguredResolver
from pex.resolve.lock_resolver import resolve_from_lock
from pex.resolve.pex_repository_resolver import resolve_from_pex
from pex.resolve.requirement_configuration import RequirementConfiguration
from pex.resolve.resolver_configuration import (
LockRepositoryConfiguration,
PexRepositoryConfiguration,
)
from pex.resolve.resolvers import Installed
from pex.resolver import resolve as resolve_via_pip
from pex.result import try_
from pex.targets import Targets
from pex.tracer import TRACER
from pex.typing import TYPE_CHECKING

if TYPE_CHECKING:
from pex.resolve.resolver_options import ResolverConfiguration


def resolve(
targets, # type: Targets
requirement_configuration, # type: RequirementConfiguration
resolver_configuration, # type: ResolverConfiguration
compile_pyc=False, # type: bool
ignore_errors=False, # type: bool
):
# type: (...) -> Installed
if isinstance(resolver_configuration, LockRepositoryConfiguration):
lock = try_(resolver_configuration.parse_lock())
with TRACER.timed(
"Resolving requirements from lock file {lock_file}".format(lock_file=lock.source)
):
pip_configuration = resolver_configuration.pip_configuration
return try_(
resolve_from_lock(
targets=targets,
lock=lock,
resolver=ConfiguredResolver(pip_configuration=pip_configuration),
requirements=requirement_configuration.requirements,
requirement_files=requirement_configuration.requirement_files,
constraint_files=requirement_configuration.constraint_files,
transitive=pip_configuration.transitive,
indexes=pip_configuration.repos_configuration.indexes,
find_links=pip_configuration.repos_configuration.find_links,
resolver_version=pip_configuration.resolver_version,
network_configuration=pip_configuration.network_configuration,
password_entries=pip_configuration.repos_configuration.password_entries,
build=pip_configuration.allow_builds,
use_wheel=pip_configuration.allow_wheels,
prefer_older_binary=pip_configuration.prefer_older_binary,
use_pep517=pip_configuration.use_pep517,
build_isolation=pip_configuration.build_isolation,
compile=compile_pyc,
max_parallel_jobs=pip_configuration.max_jobs,
pip_version=lock.pip_version,
)
)
elif isinstance(resolver_configuration, PexRepositoryConfiguration):
with TRACER.timed(
"Resolving requirements from PEX {pex_repository}.".format(
pex_repository=resolver_configuration.pex_repository
)
):
return resolve_from_pex(
targets=targets,
pex=resolver_configuration.pex_repository,
requirements=requirement_configuration.requirements,
requirement_files=requirement_configuration.requirement_files,
constraint_files=requirement_configuration.constraint_files,
network_configuration=resolver_configuration.network_configuration,
transitive=resolver_configuration.transitive,
ignore_errors=ignore_errors,
)
else:
with TRACER.timed("Resolving requirements."):
return resolve_via_pip(
targets=targets,
requirements=requirement_configuration.requirements,
requirement_files=requirement_configuration.requirement_files,
constraint_files=requirement_configuration.constraint_files,
allow_prereleases=resolver_configuration.allow_prereleases,
transitive=resolver_configuration.transitive,
indexes=resolver_configuration.repos_configuration.indexes,
find_links=resolver_configuration.repos_configuration.find_links,
resolver_version=resolver_configuration.resolver_version,
network_configuration=resolver_configuration.network_configuration,
password_entries=resolver_configuration.repos_configuration.password_entries,
build=resolver_configuration.allow_builds,
use_wheel=resolver_configuration.allow_wheels,
prefer_older_binary=resolver_configuration.prefer_older_binary,
use_pep517=resolver_configuration.use_pep517,
build_isolation=resolver_configuration.build_isolation,
compile=compile_pyc,
max_parallel_jobs=resolver_configuration.max_jobs,
ignore_errors=ignore_errors,
preserve_log=resolver_configuration.preserve_log,
pip_version=resolver_configuration.version,
resolver=ConfiguredResolver(pip_configuration=resolver_configuration),
)
22 changes: 21 additions & 1 deletion pex/targets.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,12 @@
from pex.pep_508 import MarkerEnvironment
from pex.platforms import Platform
from pex.third_party.packaging.specifiers import SpecifierSet

from pex.result import Error
from pex.typing import TYPE_CHECKING, cast

if TYPE_CHECKING:
from typing import Iterable, Iterator, Optional, Tuple
from typing import Iterable, Iterator, Optional, Tuple, Union

import attr # vendor:skip
else:
Expand Down Expand Up @@ -337,3 +339,21 @@ def iter_targets():
yield complete_platform

return OrderedSet(iter_targets())

def require_unique_target(self, purpose):
# type: (str) -> Union[Target, Error]
resolved_targets = self.unique_targets()
if len(resolved_targets) != 1:
return Error(
"A single target is required for {purpose}.\n"
"There were {count} targets selected:\n"
"{targets}".format(
purpose=purpose,
count=len(resolved_targets),
targets="\n".join(
"{index}. {target}".format(index=index, target=target)
for index, target in enumerate(resolved_targets, start=1)
),
)
)
return cast(Target, next(iter(resolved_targets)))
9 changes: 7 additions & 2 deletions pex/venv/virtualenv.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -194,7 +194,12 @@ def create(
custom_prompt = prompt
interpreter.execute(args=args, env=env)
# Modern virtualenv provides a pyvenv.cfg; so we provide one on 16.7.12's behalf
# since users might expect one.
# since users might expect one. To ward off any confusion for readers of the emitted
# pyvenv.cfg file, we add a bespoke created-by field to help make it clear that Pex
# created the pyvenv.cfg file on virtualenv 16.7.12's behalf.
# N.B.: This bespoke created-by "note" field is not related to the
# Virtualenv.created_by property which reflects the underlying venv technology.
# In this case it will report "virtualenv 16.7.12".
with open(os.path.join(venv_dir, "pyvenv.cfg"), "w") as fp:
fp.write(
dedent(
Expand All @@ -203,7 +208,7 @@ def create(
include-system-site-packages = {include_system_site_packages}
virtualenv = {virtualenv_version}
version = {python_version}
created_by = pex {pex_version}
created-by = pex {pex_version}
"""
).format(
home=os.path.dirname(interpreter.binary),
Expand Down

0 comments on commit 0768b82

Please sign in to comment.