Support environment markers during pex activation. (pex-tool#582)

We've had support for environment markers on the resolve side for a
while and with just a little plumbing we can now support multi-python
pexes with environment-specific requirements.

Fixes pex-tool#456

pex/bin/pex.py

@@ -661,10 +661,11 @@ def walk_and_do(fn, src_dir):
                                 allow_prereleases=resolver_option_builder.prereleases_allowed,
                                 use_manylinux=options.use_manylinux)
 
-      for dist in resolveds:
-        log('  %s' % dist, v=options.verbosity)
-        pex_builder.add_distribution(dist)
-        pex_builder.add_requirement(dist.as_requirement())
+      for resolved_dist in resolveds:
+        log('  %s -> %s' % (resolved_dist.requirement, resolved_dist.distribution),
+            v=options.verbosity)
+        pex_builder.add_distribution(resolved_dist.distribution)
+        pex_builder.add_requirement(resolved_dist.requirement)
     except Unsatisfiable as e:
       die(e)
 

pex/environment.py

@@ -161,6 +161,9 @@ def _resolve(self, working_set, reqs):
     # Resolve them one at a time so that we can figure out which ones we need to elide should
     # there be an interpreter incompatibility.
     for req in reqs:
+      if req.marker and not req.marker.evaluate():
+        TRACER.log('Skipping activation of `%s` due to environment marker de-selection' % req)
+        continue
       with TRACER.timed('Resolving %s' % req, V=2):
         try:
           resolveds.update(working_set.resolve([req], env=self))

pex/resolver.py

@@ -159,6 +159,10 @@ def map_packages(resolved_packages):
     return _ResolvableSet([map_packages(rp) for rp in self.__tuples])
 
 
+class ResolvedDistribution(namedtuple('ResolvedDistribution', 'requirement distribution')):
+  """A requirement and the resolved distribution that satisfies it."""
+
+
 class Resolver(object):
   """Interface for resolving resolvable entities into python packages."""
 
@@ -212,12 +216,10 @@ def expand_platform():
         # platform.
         return expand_platform()
 
-  def __init__(self, allow_prereleases=None, interpreter=None, platform=None,
-               pkg_blacklist=None, use_manylinux=None):
+  def __init__(self, allow_prereleases=None, interpreter=None, platform=None, use_manylinux=None):
     self._interpreter = interpreter or PythonInterpreter.get()
     self._platform = self._maybe_expand_platform(self._interpreter, platform)
     self._allow_prereleases = allow_prereleases
-    self._blacklist = pkg_blacklist.copy() if pkg_blacklist else {}
     self._supported_tags = self._platform.supported_tags(
       self._interpreter,
       use_manylinux
@@ -257,12 +259,6 @@ def build(self, package, options):
         'Could not get distribution for %s on platform %s.' % (package, self._platform))
     return dist
 
-  def _resolvable_is_blacklisted(self, resolvable_name):
-    return (
-      resolvable_name in self._blacklist and
-      self._interpreter.identity.matches(self._blacklist[resolvable_name])
-    )
-
   def resolve(self, resolvables, resolvable_set=None):
     resolvables = [(resolvable, None) for resolvable in resolvables]
     resolvable_set = resolvable_set or _ResolvableSet()
@@ -277,10 +273,7 @@ def resolve(self, resolvables, resolvable_set=None):
           continue
         packages = self.package_iterator(resolvable, existing=resolvable_set.get(resolvable.name))
 
-        # TODO: Remove blacklist strategy in favor of smart requirement handling
-        # https://github.com/pantsbuild/pex/issues/456
-        if not self._resolvable_is_blacklisted(resolvable.name):
-          resolvable_set.merge(resolvable, packages, parent)
+        resolvable_set.merge(resolvable, packages, parent)
         processed_resolvables.add(resolvable)
 
       built_packages = {}
@@ -327,7 +320,13 @@ def resolve(self, resolvables, resolvable_set=None):
         continue
       assert len(packages) > 0, 'ResolvableSet.packages(%s) should not be empty' % resolvable
       package = next(iter(packages))
-      dists.append(distributions[package])
+      distribution = distributions[package]
+      if isinstance(resolvable, ResolvableRequirement):
+        requirement = resolvable.requirement
+      else:
+        requirement = distribution.as_requirement()
+      dists.append(ResolvedDistribution(requirement=requirement,
+                                        distribution=distribution))
     return dists
 
 
@@ -404,7 +403,6 @@ def resolve(requirements,
             cache=None,
             cache_ttl=None,
             allow_prereleases=None,
-            pkg_blacklist=None,
             use_manylinux=None):
   """Produce all distributions needed to (recursively) meet `requirements`
 
@@ -440,16 +438,8 @@ def resolve(requirements,
     ``context``.
   :keyword allow_prereleases: (optional) Include pre-release and development versions.  If
     unspecified only stable versions will be resolved, unless explicitly included.
-  :keyword pkg_blacklist: (optional) A blacklist dict (str->str) that maps package name to
-    an interpreter constraint. If a package name is in the blacklist and its interpreter
-    constraint matches the target interpreter, skip the requirement. This is needed to ensure
-    that universal requirement resolves for a target interpreter version do not error out on
-    interpreter specific requirements such as backport libs like `functools32`.
-    For example, a valid blacklist is {'functools32': 'CPython>3'}.
-    NOTE: this keyword is a temporary fix and will be reverted in favor of a long term solution
-    tracked by: https://github.com/pantsbuild/pex/issues/456
   :keyword use_manylinux: (optional) Whether or not to use manylinux for linux resolves.
-  :returns: List of :class:`pkg_resources.Distribution` instances meeting ``requirements``.
+  :returns: List of :class:`ResolvedDistribution` instances meeting ``requirements``.
   :raises Unsatisfiable: If ``requirements`` is not transitively satisfiable.
   :raises Untranslateable: If no compatible distributions could be acquired for
     a particular requirement.
@@ -475,6 +465,10 @@ def resolve(requirements,
   .. versionchanged:: 1.0
     ``resolver`` is now just a wrapper around the :class:`Resolver` and :class:`CachingResolver`
     classes.
+
+  .. versionchanged:: 1.5.0
+    The ``pkg_blacklist``  has been removed and the return type change to a list of
+    :class:`ResolvedDistribution`.
   """
 
   builder = ResolverOptionsBuilder(fetchers=fetchers,
@@ -489,14 +483,12 @@ def resolve(requirements,
                                allow_prereleases=allow_prereleases,
                                use_manylinux=use_manylinux,
                                interpreter=interpreter,
-                               platform=platform,
-                               pkg_blacklist=pkg_blacklist)
+                               platform=platform)
   else:
     resolver = Resolver(allow_prereleases=allow_prereleases,
                         use_manylinux=use_manylinux,
                         interpreter=interpreter,
-                        platform=platform,
-                        pkg_blacklist=pkg_blacklist)
+                        platform=platform)
 
   return resolver.resolve(resolvables_from_iterable(requirements, builder))
 
@@ -510,7 +502,6 @@ def resolve_multi(requirements,
                   cache=None,
                   cache_ttl=None,
                   allow_prereleases=None,
-                  pkg_blacklist=None,
                   use_manylinux=None):
   """A generator function that produces all distributions needed to meet `requirements`
   for multiple interpreters and/or platforms.
@@ -542,15 +533,7 @@ def resolve_multi(requirements,
     ``context``.
   :keyword allow_prereleases: (optional) Include pre-release and development versions.  If
     unspecified only stable versions will be resolved, unless explicitly included.
-  :keyword pkg_blacklist: (optional) A blacklist dict (str->str) that maps package name to
-    an interpreter constraint. If a package name is in the blacklist and its interpreter
-    constraint matches the target interpreter, skip the requirement. This is needed to ensure
-    that universal requirement resolves for a target interpreter version do not error out on
-    interpreter specific requirements such as backport libs like `functools32`.
-    For example, a valid blacklist is {'functools32': 'CPython>3'}.
-    NOTE: this keyword is a temporary fix and will be reverted in favor of a long term solution
-    tracked by: https://github.com/pantsbuild/pex/issues/456
-  :yields: All :class:`pkg_resources.Distribution` instances meeting ``requirements``.
+  :yields: All :class:`ResolvedDistribution` instances meeting ``requirements``.
   :raises Unsatisfiable: If ``requirements`` is not transitively satisfiable.
   :raises Untranslateable: If no compatible distributions could be acquired for
     a particular requirement.
@@ -571,7 +554,6 @@ def resolve_multi(requirements,
                                 cache,
                                 cache_ttl,
                                 allow_prereleases,
-                                pkg_blacklist=pkg_blacklist,
                                 use_manylinux=use_manylinux):
         if resolvable not in seen:
           seen.add(resolvable)

tests/test_environment.py

@@ -127,20 +127,21 @@ def bad_interpreter(include_site_extras=True):
     # We need to run the bad interpreter with a modern, non-Apple-Extras setuptools in order to
     # successfully install psutil.
     for requirement in (SETUPTOOLS_REQUIREMENT, WHEEL_REQUIREMENT):
-      for dist in resolver.resolve([requirement],
-                                   cache=cache,
-                                   # We can't use wheels since we're bootstrapping them.
-                                   precedence=(SourcePackage, EggPackage),
-                                   interpreter=interpreter):
+      for resolved_dist in resolver.resolve([requirement],
+                                            cache=cache,
+                                            # We can't use wheels since we're bootstrapping them.
+                                            precedence=(SourcePackage, EggPackage),
+                                            interpreter=interpreter):
+        dist = resolved_dist.distribution
         interpreter = interpreter.with_extra(dist.key, dist.version, dist.location)
 
     with nested(yield_pex_builder(installer_impl=WheelInstaller, interpreter=interpreter),
                 temporary_filename()) as (pb, pex_file):
-      for dist in resolver.resolve(['psutil==5.4.3'],
-                                   cache=cache,
-                                   precedence=(SourcePackage, WheelPackage),
-                                   interpreter=interpreter):
-        pb.add_dist_location(dist.location)
+      for resolved_dist in resolver.resolve(['psutil==5.4.3'],
+                                            cache=cache,
+                                            precedence=(SourcePackage, WheelPackage),
+                                            interpreter=interpreter):
+        pb.add_dist_location(resolved_dist.distribution.location)
       pb.build(pex_file)
 
       # NB: We want PEX to find the bare bad interpreter at runtime.

tests/test_integration.py

@@ -12,7 +12,7 @@
 import pytest
 from twitter.common.contextutil import environment_as, temporary_dir
 
-from pex.compatibility import WINDOWS
+from pex.compatibility import WINDOWS, to_bytes
 from pex.installer import EggInstaller
 from pex.pex_bootstrapper import get_pex_info
 from pex.testing import (
@@ -1107,3 +1107,39 @@ def test_setup_interpreter_constraint():
                                  '-o', pex])
       results.assert_success()
       subprocess.check_call([pex, '-c', 'import jsonschema'])
+
+
+@pytest.mark.skipif(IS_PYPY,
+                    reason='Our pyenv interpreter setup fails under pypy: '
+                           'https://github.com/pantsbuild/pex/issues/477')
+def test_setup_python_multiple():
+  py27_interpreter = ensure_python_interpreter(PY27)
+  py36_interpreter = ensure_python_interpreter(PY36)
+  with temporary_dir() as out:
+    pex = os.path.join(out, 'pex.pex')
+    results = run_pex_command(['jsonschema==2.6.0',
+                               '--disable-cache',
+                               '--python-shebang=#!/usr/bin/env python',
+                               '--python={}'.format(py27_interpreter),
+                               '--python={}'.format(py36_interpreter),
+                               '-o', pex])
+    results.assert_success()
+
+    pex_program = [pex, '-c']
+    py2_only_program = pex_program + ['import functools32']
+    both_program = pex_program + [
+      'import jsonschema, os, sys; print(os.path.realpath(sys.executable))'
+    ]
+
+    with environment_as(PATH=os.path.dirname(py27_interpreter)):
+      subprocess.check_call(py2_only_program)
+
+      stdout = subprocess.check_output(both_program)
+      assert to_bytes(os.path.realpath(py27_interpreter)) == stdout.strip()
+
+    with environment_as(PATH=os.path.dirname(py36_interpreter)):
+      with pytest.raises(subprocess.CalledProcessError):
+        subprocess.check_call(py2_only_program)
+
+      stdout = subprocess.check_output(both_program)
+      assert to_bytes(os.path.realpath(py36_interpreter)) == stdout.strip()

tests/test_pex.py

@@ -419,7 +419,8 @@ def test_execute_interpreter_file_program():
 
 def test_pex_run_custom_setuptools_useable():
   with temporary_dir() as resolve_cache:
-    dists = resolve(['setuptools==36.2.7'], cache=resolve_cache)
+    dists = [resolved_dist.distribution
+             for resolved_dist in resolve(['setuptools==36.2.7'], cache=resolve_cache)]
     with temporary_dir() as temp_dir:
       pex = write_simple_pex(
         temp_dir,
@@ -440,11 +441,13 @@ def test_pex_run_conflicting_custom_setuptools_useable():
   # > pkg_resources/py31compat.py
   # > pkg_resources/_vendor/appdirs.py
   with temporary_dir() as resolve_cache:
-    dists = resolve(['setuptools==20.3.1'], cache=resolve_cache)
+    dists = [resolved_dist.distribution
+             for resolved_dist in resolve(['setuptools==20.3.1'], cache=resolve_cache)]
     interpreter = PythonInterpreter.from_binary(sys.executable,
                                                 path_extras=[dist.location for dist in dists],
                                                 include_site_extras=False)
-    dists = resolve(['setuptools==40.4.3'], cache=resolve_cache)
+    dists = [resolved_dist.distribution
+             for resolved_dist in resolve(['setuptools==40.4.3'], cache=resolve_cache)]
     with temporary_dir() as temp_dir:
       pex = write_simple_pex(
         temp_dir,