Use a single setuptools shim

for all setup.py invocations
bdist_wheel will now use tokenize in Python 3 just like for install
fixes pypa#2042

pip/download.py

@@ -28,7 +28,8 @@
 from pip.models import PyPI
 from pip.utils import (splitext, rmtree, format_size, display_path,
                        backup_dir, ask_path_exists, unpack_file,
-                       ARCHIVE_EXTENSIONS, consume, call_subprocess)
+                       ARCHIVE_EXTENSIONS, consume, call_subprocess,
+                       SETUPTOOLS_SHIM)
 from pip.utils.filesystem import check_path_owner
 from pip.utils.logging import indent_log
 from pip.utils.ui import DownloadProgressBar, DownloadProgressSpinner
@@ -725,10 +726,7 @@ def _copy_dist_from_dir(link_path, location):
     setup_py = 'setup.py'
     sdist_args = [sys.executable]
     sdist_args.append('-c')
-    sdist_args.append(
-        "import setuptools, tokenize;__file__=%r;"
-        "exec(compile(getattr(tokenize, 'open', open)(__file__).read()"
-        ".replace('\\r\\n', '\\n'), __file__, 'exec'))" % setup_py)
+    sdist_args.append(SETUPTOOLS_SHIM % setup_py)
     sdist_args.append('sdist')
     sdist_args += ['--dist-dir', location]
     logger.info('Running setup.py sdist for %s', link_path)

pip/req/req_install.py

@@ -32,7 +32,7 @@
     display_path, rmtree, ask_path_exists, backup_dir, is_installable_dir,
     dist_in_usersite, dist_in_site_packages, egg_link_path,
     call_subprocess, read_text_file, FakeFile, _make_build_dir, ensure_dir,
-    get_installed_version, canonicalize_name
+    get_installed_version, canonicalize_name, SETUPTOOLS_SHIM
 )
 from pip.utils.hashes import Hashes
 from pip.utils.logging import indent_log
@@ -407,9 +407,7 @@ def run_egg_info(self):
             )
 
         with indent_log():
-            script = self._run_setup_py
-            script = script.replace('__SETUP_PY__', repr(self.setup_py))
-            script = script.replace('__PKG_NAME__', repr(self.name))
+            script = self._run_setup_py % self.setup_py
             base_cmd = [sys.executable, '-c', script]
             if self.isolated:
                 base_cmd += ["--no-user-cfg"]
@@ -460,11 +458,9 @@ def run_egg_info(self):
     # FIXME: This is a lame hack, entirely for PasteScript which has
     # a self-provided entry point that causes this awkwardness
     _run_setup_py = """
-__file__ = __SETUP_PY__
 from setuptools.command import egg_info
 import pkg_resources
 import os
-import tokenize
 def replacement_run(self):
     self.mkpath(self.egg_info)
     installer = self.distribution.fetch_build_egg
@@ -475,12 +471,7 @@ def replacement_run(self):
             writer(self, ep.name, os.path.join(self.egg_info,ep.name))
     self.find_sources()
 egg_info.egg_info.run = replacement_run
-exec(compile(
-    getattr(tokenize, 'open', open)(__file__).read().replace('\\r\\n', '\\n'),
-    __file__,
-    'exec'
-))
-"""
+""" + SETUPTOOLS_SHIM
 
     def egg_info_data(self, filename):
         if self.satisfied_by is not None:
@@ -852,11 +843,7 @@ def install(self, install_options, global_options=[], root=None):
         try:
             install_args = [sys.executable, "-u"]
             install_args.append('-c')
-            install_args.append(
-                "import setuptools, tokenize;__file__=%r;"
-                "exec(compile(getattr(tokenize, 'open', open)(__file__).read()"
-                ".replace('\\r\\n', '\\n'), __file__, 'exec'))" % self.setup_py
-            )
+            install_args.append(SETUPTOOLS_SHIM % self.setup_py)
             install_args += list(global_options) + \
                 ['install', '--record', record_filename]
 
@@ -976,9 +963,7 @@ def install_editable(self, install_options, global_options=()):
                 [
                     sys.executable,
                     '-c',
-                    "import setuptools, tokenize; __file__=%r; exec(compile("
-                    "getattr(tokenize, 'open', open)(__file__).read().replace"
-                    "('\\r\\n', '\\n'), __file__, 'exec'))" % self.setup_py
+                    SETUPTOOLS_SHIM % self.setup_py
                 ] +
                 list(global_options) +
                 ['develop', '--no-deps'] +

pip/utils/__init__.py

@@ -815,3 +815,11 @@ def canonicalize_name(name):
 def consume(iterator):
     """Consume an iterable at C speed."""
     deque(iterator, maxlen=0)
+
+
+# Shim to wrap setup.py invocation with setuptools
+SETUPTOOLS_SHIM = (
+    "import setuptools, tokenize;__file__=%r;"
+    "exec(compile(getattr(tokenize, 'open', open)(__file__).read()"
+    ".replace('\\r\\n', '\\n'), __file__, 'exec'))"
+)

pip/wheel.py

@@ -32,7 +32,7 @@
 from pip import pep425tags
 from pip.utils import (
     call_subprocess, ensure_dir, captured_stdout, rmtree, canonicalize_name,
-    read_chunks)
+    read_chunks, SETUPTOOLS_SHIM)
 from pip.utils.ui import open_spinner
 from pip.utils.logging import indent_log
 from pip._vendor.distlib.scripts import ScriptMaker
@@ -687,9 +687,7 @@ def _build_one(self, req, output_dir, python_tag=None):
     def _base_setup_args(self, req):
         return [
             sys.executable, "-u", '-c',
-            "import setuptools;__file__=%r;"
-            "exec(compile(open(__file__).read().replace('\\r\\n', '\\n'), "
-            "__file__, 'exec'))" % req.setup_py
+            SETUPTOOLS_SHIM % req.setup_py
         ] + list(self.global_options)
 
     def __build_one(self, req, tempd, python_tag=None):