Skip to content

Commit

Permalink
Support environment markers during pex activation. (#582)
Browse files Browse the repository at this point in the history
We've had support for environment markers on the resolve side for a
while and with just a little plumbing we can now support multi-python
pexes with environment-specific requirements.

Fixes #456
  • Loading branch information
jsirois authored Oct 8, 2018
1 parent 55e6482 commit 5f1f00f
Show file tree
Hide file tree
Showing 7 changed files with 116 additions and 114 deletions.
9 changes: 5 additions & 4 deletions pex/bin/pex.py
Original file line number Diff line number Diff line change
Expand Up @@ -661,10 +661,11 @@ def walk_and_do(fn, src_dir):
allow_prereleases=resolver_option_builder.prereleases_allowed,
use_manylinux=options.use_manylinux)

for dist in resolveds:
log(' %s' % dist, v=options.verbosity)
pex_builder.add_distribution(dist)
pex_builder.add_requirement(dist.as_requirement())
for resolved_dist in resolveds:
log(' %s -> %s' % (resolved_dist.requirement, resolved_dist.distribution),
v=options.verbosity)
pex_builder.add_distribution(resolved_dist.distribution)
pex_builder.add_requirement(resolved_dist.requirement)
except Unsatisfiable as e:
die(e)

Expand Down
3 changes: 3 additions & 0 deletions pex/environment.py
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,9 @@ def _resolve(self, working_set, reqs):
# Resolve them one at a time so that we can figure out which ones we need to elide should
# there be an interpreter incompatibility.
for req in reqs:
if req.marker and not req.marker.evaluate():
TRACER.log('Skipping activation of `%s` due to environment marker de-selection' % req)
continue
with TRACER.timed('Resolving %s' % req, V=2):
try:
resolveds.update(working_set.resolve([req], env=self))
Expand Down
60 changes: 21 additions & 39 deletions pex/resolver.py
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,10 @@ def map_packages(resolved_packages):
return _ResolvableSet([map_packages(rp) for rp in self.__tuples])


class ResolvedDistribution(namedtuple('ResolvedDistribution', 'requirement distribution')):
"""A requirement and the resolved distribution that satisfies it."""


class Resolver(object):
"""Interface for resolving resolvable entities into python packages."""

Expand Down Expand Up @@ -212,12 +216,10 @@ def expand_platform():
# platform.
return expand_platform()

def __init__(self, allow_prereleases=None, interpreter=None, platform=None,
pkg_blacklist=None, use_manylinux=None):
def __init__(self, allow_prereleases=None, interpreter=None, platform=None, use_manylinux=None):
self._interpreter = interpreter or PythonInterpreter.get()
self._platform = self._maybe_expand_platform(self._interpreter, platform)
self._allow_prereleases = allow_prereleases
self._blacklist = pkg_blacklist.copy() if pkg_blacklist else {}
self._supported_tags = self._platform.supported_tags(
self._interpreter,
use_manylinux
Expand Down Expand Up @@ -257,12 +259,6 @@ def build(self, package, options):
'Could not get distribution for %s on platform %s.' % (package, self._platform))
return dist

def _resolvable_is_blacklisted(self, resolvable_name):
return (
resolvable_name in self._blacklist and
self._interpreter.identity.matches(self._blacklist[resolvable_name])
)

def resolve(self, resolvables, resolvable_set=None):
resolvables = [(resolvable, None) for resolvable in resolvables]
resolvable_set = resolvable_set or _ResolvableSet()
Expand All @@ -277,10 +273,7 @@ def resolve(self, resolvables, resolvable_set=None):
continue
packages = self.package_iterator(resolvable, existing=resolvable_set.get(resolvable.name))

# TODO: Remove blacklist strategy in favor of smart requirement handling
# https://github.com/pantsbuild/pex/issues/456
if not self._resolvable_is_blacklisted(resolvable.name):
resolvable_set.merge(resolvable, packages, parent)
resolvable_set.merge(resolvable, packages, parent)
processed_resolvables.add(resolvable)

built_packages = {}
Expand Down Expand Up @@ -327,7 +320,13 @@ def resolve(self, resolvables, resolvable_set=None):
continue
assert len(packages) > 0, 'ResolvableSet.packages(%s) should not be empty' % resolvable
package = next(iter(packages))
dists.append(distributions[package])
distribution = distributions[package]
if isinstance(resolvable, ResolvableRequirement):
requirement = resolvable.requirement
else:
requirement = distribution.as_requirement()
dists.append(ResolvedDistribution(requirement=requirement,
distribution=distribution))
return dists


Expand Down Expand Up @@ -404,7 +403,6 @@ def resolve(requirements,
cache=None,
cache_ttl=None,
allow_prereleases=None,
pkg_blacklist=None,
use_manylinux=None):
"""Produce all distributions needed to (recursively) meet `requirements`
Expand Down Expand Up @@ -440,16 +438,8 @@ def resolve(requirements,
``context``.
:keyword allow_prereleases: (optional) Include pre-release and development versions. If
unspecified only stable versions will be resolved, unless explicitly included.
:keyword pkg_blacklist: (optional) A blacklist dict (str->str) that maps package name to
an interpreter constraint. If a package name is in the blacklist and its interpreter
constraint matches the target interpreter, skip the requirement. This is needed to ensure
that universal requirement resolves for a target interpreter version do not error out on
interpreter specific requirements such as backport libs like `functools32`.
For example, a valid blacklist is {'functools32': 'CPython>3'}.
NOTE: this keyword is a temporary fix and will be reverted in favor of a long term solution
tracked by: https://github.com/pantsbuild/pex/issues/456
:keyword use_manylinux: (optional) Whether or not to use manylinux for linux resolves.
:returns: List of :class:`pkg_resources.Distribution` instances meeting ``requirements``.
:returns: List of :class:`ResolvedDistribution` instances meeting ``requirements``.
:raises Unsatisfiable: If ``requirements`` is not transitively satisfiable.
:raises Untranslateable: If no compatible distributions could be acquired for
a particular requirement.
Expand All @@ -475,6 +465,10 @@ def resolve(requirements,
.. versionchanged:: 1.0
``resolver`` is now just a wrapper around the :class:`Resolver` and :class:`CachingResolver`
classes.
.. versionchanged:: 1.5.0
The ``pkg_blacklist`` has been removed and the return type change to a list of
:class:`ResolvedDistribution`.
"""

builder = ResolverOptionsBuilder(fetchers=fetchers,
Expand All @@ -489,14 +483,12 @@ def resolve(requirements,
allow_prereleases=allow_prereleases,
use_manylinux=use_manylinux,
interpreter=interpreter,
platform=platform,
pkg_blacklist=pkg_blacklist)
platform=platform)
else:
resolver = Resolver(allow_prereleases=allow_prereleases,
use_manylinux=use_manylinux,
interpreter=interpreter,
platform=platform,
pkg_blacklist=pkg_blacklist)
platform=platform)

return resolver.resolve(resolvables_from_iterable(requirements, builder))

Expand All @@ -510,7 +502,6 @@ def resolve_multi(requirements,
cache=None,
cache_ttl=None,
allow_prereleases=None,
pkg_blacklist=None,
use_manylinux=None):
"""A generator function that produces all distributions needed to meet `requirements`
for multiple interpreters and/or platforms.
Expand Down Expand Up @@ -542,15 +533,7 @@ def resolve_multi(requirements,
``context``.
:keyword allow_prereleases: (optional) Include pre-release and development versions. If
unspecified only stable versions will be resolved, unless explicitly included.
:keyword pkg_blacklist: (optional) A blacklist dict (str->str) that maps package name to
an interpreter constraint. If a package name is in the blacklist and its interpreter
constraint matches the target interpreter, skip the requirement. This is needed to ensure
that universal requirement resolves for a target interpreter version do not error out on
interpreter specific requirements such as backport libs like `functools32`.
For example, a valid blacklist is {'functools32': 'CPython>3'}.
NOTE: this keyword is a temporary fix and will be reverted in favor of a long term solution
tracked by: https://github.com/pantsbuild/pex/issues/456
:yields: All :class:`pkg_resources.Distribution` instances meeting ``requirements``.
:yields: All :class:`ResolvedDistribution` instances meeting ``requirements``.
:raises Unsatisfiable: If ``requirements`` is not transitively satisfiable.
:raises Untranslateable: If no compatible distributions could be acquired for
a particular requirement.
Expand All @@ -571,7 +554,6 @@ def resolve_multi(requirements,
cache,
cache_ttl,
allow_prereleases,
pkg_blacklist=pkg_blacklist,
use_manylinux=use_manylinux):
if resolvable not in seen:
seen.add(resolvable)
Expand Down
21 changes: 11 additions & 10 deletions tests/test_environment.py
Original file line number Diff line number Diff line change
Expand Up @@ -127,20 +127,21 @@ def bad_interpreter(include_site_extras=True):
# We need to run the bad interpreter with a modern, non-Apple-Extras setuptools in order to
# successfully install psutil.
for requirement in (SETUPTOOLS_REQUIREMENT, WHEEL_REQUIREMENT):
for dist in resolver.resolve([requirement],
cache=cache,
# We can't use wheels since we're bootstrapping them.
precedence=(SourcePackage, EggPackage),
interpreter=interpreter):
for resolved_dist in resolver.resolve([requirement],
cache=cache,
# We can't use wheels since we're bootstrapping them.
precedence=(SourcePackage, EggPackage),
interpreter=interpreter):
dist = resolved_dist.distribution
interpreter = interpreter.with_extra(dist.key, dist.version, dist.location)

with nested(yield_pex_builder(installer_impl=WheelInstaller, interpreter=interpreter),
temporary_filename()) as (pb, pex_file):
for dist in resolver.resolve(['psutil==5.4.3'],
cache=cache,
precedence=(SourcePackage, WheelPackage),
interpreter=interpreter):
pb.add_dist_location(dist.location)
for resolved_dist in resolver.resolve(['psutil==5.4.3'],
cache=cache,
precedence=(SourcePackage, WheelPackage),
interpreter=interpreter):
pb.add_dist_location(resolved_dist.distribution.location)
pb.build(pex_file)

# NB: We want PEX to find the bare bad interpreter at runtime.
Expand Down
38 changes: 37 additions & 1 deletion tests/test_integration.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
import pytest
from twitter.common.contextutil import environment_as, temporary_dir

from pex.compatibility import WINDOWS
from pex.compatibility import WINDOWS, to_bytes
from pex.installer import EggInstaller
from pex.pex_bootstrapper import get_pex_info
from pex.testing import (
Expand Down Expand Up @@ -1107,3 +1107,39 @@ def test_setup_interpreter_constraint():
'-o', pex])
results.assert_success()
subprocess.check_call([pex, '-c', 'import jsonschema'])


@pytest.mark.skipif(IS_PYPY,
reason='Our pyenv interpreter setup fails under pypy: '
'https://github.com/pantsbuild/pex/issues/477')
def test_setup_python_multiple():
py27_interpreter = ensure_python_interpreter(PY27)
py36_interpreter = ensure_python_interpreter(PY36)
with temporary_dir() as out:
pex = os.path.join(out, 'pex.pex')
results = run_pex_command(['jsonschema==2.6.0',
'--disable-cache',
'--python-shebang=#!/usr/bin/env python',
'--python={}'.format(py27_interpreter),
'--python={}'.format(py36_interpreter),
'-o', pex])
results.assert_success()

pex_program = [pex, '-c']
py2_only_program = pex_program + ['import functools32']
both_program = pex_program + [
'import jsonschema, os, sys; print(os.path.realpath(sys.executable))'
]

with environment_as(PATH=os.path.dirname(py27_interpreter)):
subprocess.check_call(py2_only_program)

stdout = subprocess.check_output(both_program)
assert to_bytes(os.path.realpath(py27_interpreter)) == stdout.strip()

with environment_as(PATH=os.path.dirname(py36_interpreter)):
with pytest.raises(subprocess.CalledProcessError):
subprocess.check_call(py2_only_program)

stdout = subprocess.check_output(both_program)
assert to_bytes(os.path.realpath(py36_interpreter)) == stdout.strip()
9 changes: 6 additions & 3 deletions tests/test_pex.py
Original file line number Diff line number Diff line change
Expand Up @@ -419,7 +419,8 @@ def test_execute_interpreter_file_program():

def test_pex_run_custom_setuptools_useable():
with temporary_dir() as resolve_cache:
dists = resolve(['setuptools==36.2.7'], cache=resolve_cache)
dists = [resolved_dist.distribution
for resolved_dist in resolve(['setuptools==36.2.7'], cache=resolve_cache)]
with temporary_dir() as temp_dir:
pex = write_simple_pex(
temp_dir,
Expand All @@ -440,11 +441,13 @@ def test_pex_run_conflicting_custom_setuptools_useable():
# > pkg_resources/py31compat.py
# > pkg_resources/_vendor/appdirs.py
with temporary_dir() as resolve_cache:
dists = resolve(['setuptools==20.3.1'], cache=resolve_cache)
dists = [resolved_dist.distribution
for resolved_dist in resolve(['setuptools==20.3.1'], cache=resolve_cache)]
interpreter = PythonInterpreter.from_binary(sys.executable,
path_extras=[dist.location for dist in dists],
include_site_extras=False)
dists = resolve(['setuptools==40.4.3'], cache=resolve_cache)
dists = [resolved_dist.distribution
for resolved_dist in resolve(['setuptools==40.4.3'], cache=resolve_cache)]
with temporary_dir() as temp_dir:
pex = write_simple_pex(
temp_dir,
Expand Down
Loading

0 comments on commit 5f1f00f

Please sign in to comment.