# This is a combination of 8 commits.

# The first commit's message is:
pex-tool#572: Allow import of ctypes to be skipped if use_manylinux is false

# This is the 2nd commit message:

Narrow the env marker test. (pex-tool#578)

The jupyter dist is just a meta-dist with fully unconstrained deps on ~6
other dists. This test was added to test environment marker support in
pex, which ipython - not jupyter - leverages heavily.

# This is the 3rd commit message:

Fix resolve regressions introduced by the 1.4.8. (pex-tool#580)

PR pex-tool#571 regressed the half-broken state of having
`--interpreter_constraint` selected interpreters not setup to also
having `--python` selected interpreters also not setup. In addition,
PR pex-tool#568 incorrectly classified the current Platform passed by
`resolve_multi` as a user-specified extended platform specification
breaking custom interpreter resolution. Fix both and add tests that
failed prior to this combination of fixes.

A more comprehensive fix is tracked in part by pex-tool#579.

# This is the 4th commit message:

Cleanup `PexInfo` and `PythonInterpreter`. (pex-tool#581)

Kill an unused type in `PexInfo` as well as our last remaining use of
`pkg_resources.get_platform`. Also kill unused `COMPATIBLE_SETUPTOOLS`
constants in `PythonInterpreter`.

# This is the 5th commit message:

Support environment markers during pex activation. (pex-tool#582)

We've had support for environment markers on the resolve side for a
while and with just a little plumbing we can now support multi-python
pexes with environment-specific requirements.

Fixes pex-tool#456

# This is the 6th commit message:

Revert "Support environment markers during pex activation. (pex-tool#582)"

This reverts commit 5f1f00f.

We want to do a 1.4.9 bugfix release before this ~API change.

# This is the 7th commit message:

Prepare the 1.4.9 release. (pex-tool#588)

Work towards pex-tool#583

# This is the 8th commit message:

Revert "Revert "Support environment markers during pex activation. (pex-tool#582)""

This reverts commit 44ff463.

This restores pex-tool#582 for the 1.5.0 release tracked by pex-tool#585.

CHANGES.rst

@@ -1,6 +1,24 @@
 Release Notes
 =============
 
+1.4.9
+-----
+
+This is a hotfix release for 1.4.8 that fixes a regression in interpreter setup that could lead to
+resolved distributions failing to build or install.
+
+* Cleanup `PexInfo` and `PythonInterpreter`. (#581)
+  `PR #581 <https://github.com/pantsbuild/pex/pull/581>`_
+
+* Fix resolve regressions introduced by the 1.4.8. (#580)
+  `PR #580 <https://github.com/pantsbuild/pex/pull/580>`_
+
+* Narrow the env marker test. (#578)
+  `PR #578 <https://github.com/pantsbuild/pex/pull/578>`_
+
+* Documentation for #569 (#574)
+  `PR #574 <https://github.com/pantsbuild/pex/pull/574>`_
+
 1.4.8
 -----
 

pex/bin/pex.py

@@ -654,17 +654,18 @@ def walk_and_do(fn, src_dir):
   with TRACER.timed('Resolving distributions'):
     try:
       resolveds = resolve_multi(resolvables,
-                                interpreters=interpreters,
+                                interpreters=setup_interpreters,
                                 platforms=options.platforms,
                                 cache=options.cache_dir,
                                 cache_ttl=options.cache_ttl,
                                 allow_prereleases=resolver_option_builder.prereleases_allowed,
                                 use_manylinux=options.use_manylinux)
 
-      for dist in resolveds:
-        log('  %s' % dist, v=options.verbosity)
-        pex_builder.add_distribution(dist)
-        pex_builder.add_requirement(dist.as_requirement())
+      for resolved_dist in resolveds:
+        log('  %s -> %s' % (resolved_dist.requirement, resolved_dist.distribution),
+            v=options.verbosity)
+        pex_builder.add_distribution(resolved_dist.distribution)
+        pex_builder.add_requirement(resolved_dist.requirement)
     except Unsatisfiable as e:
       die(e)
 

pex/environment.py

@@ -161,6 +161,9 @@ def _resolve(self, working_set, reqs):
     # Resolve them one at a time so that we can figure out which ones we need to elide should
     # there be an interpreter incompatibility.
     for req in reqs:
+      if req.marker and not req.marker.evaluate():
+        TRACER.log('Skipping activation of `%s` due to environment marker de-selection' % req)
+        continue
       with TRACER.timed('Resolving %s' % req, V=2):
         try:
           resolveds.update(working_set.resolve([req], env=self))

pex/glibc.py

@@ -1,14 +1,30 @@
 # This file was copied from the pip project master branch on 2016/12/05
 from __future__ import absolute_import
 
-import ctypes
+try:
+  import ctypes
+except ImportError:
+  # If this is being pulled in from the bootstrapper, let it decide whether this
+  # is an error. `ctypes` isn't needed if we aren't using manylinux, this
+  # allows PEX files to be run in environments where ctypes isn't available.
+  from traceback import extract_stack
+  if any(trace for trace in extract_stack()
+         if any('pex_bootstrapper.py' in str(field) for field in trace)):
+    CTYPES_UNDEF = True
+  else:
+    raise
 import platform
 import re
 import warnings
 
 
 def glibc_version_string():
   "Returns glibc version string, or None if not using glibc."
+  try:
+    CTYPES_UNDEF
+    return None
+  except NameError:
+    pass
 
   # ctypes.CDLL(None) internally calls dlopen(NULL), and as the dlopen
   # manpage says, "If filename is NULL, then the returned handle is for the

pex/interpreter.py

@@ -293,14 +293,6 @@ class PythonInterpreter(object):
 
   CACHE = {}  # memoize executable => PythonInterpreter
 
-  try:
-    # Versions of distribute prior to the setuptools merge would automatically replace
-    # 'setuptools' requirements with 'distribute'.  It provided the 'replacement' kwarg
-    # to toggle this, but it was removed post-merge.
-    COMPATIBLE_SETUPTOOLS = Requirement.parse('setuptools>=1.0', replacement=False)
-  except TypeError:
-    COMPATIBLE_SETUPTOOLS = Requirement.parse('setuptools>=1.0')
-
   class Error(Exception): pass
   class IdentificationError(Error): pass
   class InterpreterNotFound(Error): pass

pex/pex_bootstrapper.py

@@ -166,6 +166,16 @@ def bootstrap_pex(entry_point):
   pex_info = get_pex_info(entry_point)
   maybe_reexec_pex(pex_info.interpreter_constraints)
 
+  # If use_manylinux is set and ctypes is unavailable, fail
+  if pex_info.use_manylinux:
+    try:
+      from . import glibc
+      glibc.CTYPES_UNDEF
+      raise ImportError(
+          "use_manylinux set in PEX-INFO, but ctypes could not be imported.")
+    except NameError:
+      pass
+
   from . import pex
   pex.PEX(entry_point).execute()
 

pex/pex_info.py

@@ -6,7 +6,6 @@
 import json
 import os
 import warnings
-from collections import namedtuple
 
 from .common import open_zip
 from .compatibility import PY2
@@ -15,8 +14,6 @@
 from .util import merge_split
 from .variables import ENV
 
-PexPlatform = namedtuple('PexPlatform', 'interpreter version strict')
-
 
 # TODO(wickman) Split this into a PexInfoBuilder/PexInfo to ensure immutability.
 # Issue #92.
@@ -53,13 +50,15 @@ class PexInfo(object):
   @classmethod
   def make_build_properties(cls, interpreter=None):
     from .interpreter import PythonInterpreter
-    from pkg_resources import get_platform
+    from .platforms import Platform
 
     pi = interpreter or PythonInterpreter.get()
+    plat = Platform.current()
+    platform_name = plat.platform
     return {
       'class': pi.identity.interpreter,
       'version': pi.identity.version,
-      'platform': get_platform(),
+      'platform': platform_name,
     }
 
   @classmethod

pex/resolver.py

@@ -159,6 +159,10 @@ def map_packages(resolved_packages):
     return _ResolvableSet([map_packages(rp) for rp in self.__tuples])
 
 
+class ResolvedDistribution(namedtuple('ResolvedDistribution', 'requirement distribution')):
+  """A requirement and the resolved distribution that satisfies it."""
+
+
 class Resolver(object):
   """Interface for resolving resolvable entities into python packages."""
 
@@ -212,12 +216,10 @@ def expand_platform():
         # platform.
         return expand_platform()
 
-  def __init__(self, allow_prereleases=None, interpreter=None, platform=None,
-               pkg_blacklist=None, use_manylinux=None):
+  def __init__(self, allow_prereleases=None, interpreter=None, platform=None, use_manylinux=None):
     self._interpreter = interpreter or PythonInterpreter.get()
     self._platform = self._maybe_expand_platform(self._interpreter, platform)
     self._allow_prereleases = allow_prereleases
-    self._blacklist = pkg_blacklist.copy() if pkg_blacklist else {}
     self._supported_tags = self._platform.supported_tags(
       self._interpreter,
       use_manylinux
@@ -257,12 +259,6 @@ def build(self, package, options):
         'Could not get distribution for %s on platform %s.' % (package, self._platform))
     return dist
 
-  def _resolvable_is_blacklisted(self, resolvable_name):
-    return (
-      resolvable_name in self._blacklist and
-      self._interpreter.identity.matches(self._blacklist[resolvable_name])
-    )
-
   def resolve(self, resolvables, resolvable_set=None):
     resolvables = [(resolvable, None) for resolvable in resolvables]
     resolvable_set = resolvable_set or _ResolvableSet()
@@ -277,10 +273,7 @@ def resolve(self, resolvables, resolvable_set=None):
           continue
         packages = self.package_iterator(resolvable, existing=resolvable_set.get(resolvable.name))
 
-        # TODO: Remove blacklist strategy in favor of smart requirement handling
-        # https://github.com/pantsbuild/pex/issues/456
-        if not self._resolvable_is_blacklisted(resolvable.name):
-          resolvable_set.merge(resolvable, packages, parent)
+        resolvable_set.merge(resolvable, packages, parent)
         processed_resolvables.add(resolvable)
 
       built_packages = {}
@@ -327,7 +320,13 @@ def resolve(self, resolvables, resolvable_set=None):
         continue
       assert len(packages) > 0, 'ResolvableSet.packages(%s) should not be empty' % resolvable
       package = next(iter(packages))
-      dists.append(distributions[package])
+      distribution = distributions[package]
+      if isinstance(resolvable, ResolvableRequirement):
+        requirement = resolvable.requirement
+      else:
+        requirement = distribution.as_requirement()
+      dists.append(ResolvedDistribution(requirement=requirement,
+                                        distribution=distribution))
     return dists
 
 
@@ -404,7 +403,6 @@ def resolve(requirements,
             cache=None,
             cache_ttl=None,
             allow_prereleases=None,
-            pkg_blacklist=None,
             use_manylinux=None):
   """Produce all distributions needed to (recursively) meet `requirements`
 
@@ -440,16 +438,8 @@ def resolve(requirements,
     ``context``.
   :keyword allow_prereleases: (optional) Include pre-release and development versions.  If
     unspecified only stable versions will be resolved, unless explicitly included.
-  :keyword pkg_blacklist: (optional) A blacklist dict (str->str) that maps package name to
-    an interpreter constraint. If a package name is in the blacklist and its interpreter
-    constraint matches the target interpreter, skip the requirement. This is needed to ensure
-    that universal requirement resolves for a target interpreter version do not error out on
-    interpreter specific requirements such as backport libs like `functools32`.
-    For example, a valid blacklist is {'functools32': 'CPython>3'}.
-    NOTE: this keyword is a temporary fix and will be reverted in favor of a long term solution
-    tracked by: https://github.com/pantsbuild/pex/issues/456
   :keyword use_manylinux: (optional) Whether or not to use manylinux for linux resolves.
-  :returns: List of :class:`pkg_resources.Distribution` instances meeting ``requirements``.
+  :returns: List of :class:`ResolvedDistribution` instances meeting ``requirements``.
   :raises Unsatisfiable: If ``requirements`` is not transitively satisfiable.
   :raises Untranslateable: If no compatible distributions could be acquired for
     a particular requirement.
@@ -475,6 +465,10 @@ def resolve(requirements,
   .. versionchanged:: 1.0
     ``resolver`` is now just a wrapper around the :class:`Resolver` and :class:`CachingResolver`
     classes.
+
+  .. versionchanged:: 1.5.0
+    The ``pkg_blacklist``  has been removed and the return type change to a list of
+    :class:`ResolvedDistribution`.
   """
 
   builder = ResolverOptionsBuilder(fetchers=fetchers,
@@ -489,14 +483,12 @@ def resolve(requirements,
                                allow_prereleases=allow_prereleases,
                                use_manylinux=use_manylinux,
                                interpreter=interpreter,
-                               platform=platform,
-                               pkg_blacklist=pkg_blacklist)
+                               platform=platform)
   else:
     resolver = Resolver(allow_prereleases=allow_prereleases,
                         use_manylinux=use_manylinux,
                         interpreter=interpreter,
-                        platform=platform,
-                        pkg_blacklist=pkg_blacklist)
+                        platform=platform)
 
   return resolver.resolve(resolvables_from_iterable(requirements, builder))
 
@@ -510,7 +502,6 @@ def resolve_multi(requirements,
                   cache=None,
                   cache_ttl=None,
                   allow_prereleases=None,
-                  pkg_blacklist=None,
                   use_manylinux=None):
   """A generator function that produces all distributions needed to meet `requirements`
   for multiple interpreters and/or platforms.
@@ -542,22 +533,14 @@ def resolve_multi(requirements,
     ``context``.
   :keyword allow_prereleases: (optional) Include pre-release and development versions.  If
     unspecified only stable versions will be resolved, unless explicitly included.
-  :keyword pkg_blacklist: (optional) A blacklist dict (str->str) that maps package name to
-    an interpreter constraint. If a package name is in the blacklist and its interpreter
-    constraint matches the target interpreter, skip the requirement. This is needed to ensure
-    that universal requirement resolves for a target interpreter version do not error out on
-    interpreter specific requirements such as backport libs like `functools32`.
-    For example, a valid blacklist is {'functools32': 'CPython>3'}.
-    NOTE: this keyword is a temporary fix and will be reverted in favor of a long term solution
-    tracked by: https://github.com/pantsbuild/pex/issues/456
-  :yields: All :class:`pkg_resources.Distribution` instances meeting ``requirements``.
+  :yields: All :class:`ResolvedDistribution` instances meeting ``requirements``.
   :raises Unsatisfiable: If ``requirements`` is not transitively satisfiable.
   :raises Untranslateable: If no compatible distributions could be acquired for
     a particular requirement.
   """
 
   interpreters = interpreters or [PythonInterpreter.get()]
-  platforms = platforms or [Platform.current()]
+  platforms = platforms or ['current']
 
   seen = set()
   for interpreter in interpreters:
@@ -571,7 +554,6 @@ def resolve_multi(requirements,
                                 cache,
                                 cache_ttl,
                                 allow_prereleases,
-                                pkg_blacklist=pkg_blacklist,
                                 use_manylinux=use_manylinux):
         if resolvable not in seen:
           seen.add(resolvable)

pex/version.py

@@ -1,7 +1,7 @@
 # Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-__version__ = '1.4.8'
+__version__ = '1.4.9'
 
 # Versions 34.0.0 through 35.0.2 (last pre-36.0.0) de-vendored dependencies which causes problems
 # for pex code so we exclude that range.

tests/test_environment.py

@@ -127,20 +127,21 @@ def bad_interpreter(include_site_extras=True):
     # We need to run the bad interpreter with a modern, non-Apple-Extras setuptools in order to
     # successfully install psutil.
     for requirement in (SETUPTOOLS_REQUIREMENT, WHEEL_REQUIREMENT):
-      for dist in resolver.resolve([requirement],
-                                   cache=cache,
-                                   # We can't use wheels since we're bootstrapping them.
-                                   precedence=(SourcePackage, EggPackage),
-                                   interpreter=interpreter):
+      for resolved_dist in resolver.resolve([requirement],
+                                            cache=cache,
+                                            # We can't use wheels since we're bootstrapping them.
+                                            precedence=(SourcePackage, EggPackage),
+                                            interpreter=interpreter):
+        dist = resolved_dist.distribution
         interpreter = interpreter.with_extra(dist.key, dist.version, dist.location)
 
     with nested(yield_pex_builder(installer_impl=WheelInstaller, interpreter=interpreter),
                 temporary_filename()) as (pb, pex_file):
-      for dist in resolver.resolve(['psutil==5.4.3'],
-                                   cache=cache,
-                                   precedence=(SourcePackage, WheelPackage),
-                                   interpreter=interpreter):
-        pb.add_dist_location(dist.location)
+      for resolved_dist in resolver.resolve(['psutil==5.4.3'],
+                                            cache=cache,
+                                            precedence=(SourcePackage, WheelPackage),
+                                            interpreter=interpreter):
+        pb.add_dist_location(resolved_dist.distribution.location)
       pb.build(pex_file)
 
       # NB: We want PEX to find the bare bad interpreter at runtime.