support for c-extension builds within virtualenv (#1503)

* test include folders

- add test to check if it works

Signed-off-by: Bernat Gabor <[email protected]>

* pypy add lib on Linux

Signed-off-by: Bernat Gabor <[email protected]>

* fix Windows

* fix

Signed-off-by: Bernat Gabor <[email protected]>

* debug macos

Signed-off-by: Bernat Gabor <[email protected]>

* try fix pypy windows

Signed-off-by: Bernat Gabor <[email protected]>

* fix Windows

* fix

* fix

Signed-off-by: Bernat Gabor <[email protected]>

* Windows PyPy just does not understand non-ascii PATHS :-(

* allow pypy3 to fail

Signed-off-by: Bernat Gabor <[email protected]>

azure-pipelines.yml

@@ -37,13 +37,13 @@ jobs:
   parameters:
     jobs:
       py38:
-        image: [linux, windows, macOs]
+        image: [linux, vs2017-win2016, macOs]
       py37:
-        image: [linux, windows, macOs]
+        image: [linux, vs2017-win2016, macOs]
       py36:
-        image: [linux, windows, macOs]
+        image: [linux, vs2017-win2016, macOs]
       py35:
-        image: [linux, windows, macOs]
+        image: [linux, vs2017-win2016, macOs]
       py27:
         image: [linux, windows, macOs]
       pypy:
@@ -88,6 +88,9 @@ jobs:
         displayName: provision pypy 2
         inputs:
           versionSpec: 'pypy2'
+      - script: choco install vcpython27 --yes -f
+        condition: and(succeeded(), eq(variables['image_name'], 'windows'), in(variables['TOXENV'], 'py27', 'pypy'))
+        displayName: Install Visual C++ for Python 2.7
     coverage:
       with_toxenv: 'coverage' # generate .tox/.coverage, .tox/coverage.xml after test run
       for_envs: [py38, py37, py36, py35, py27, pypy, pypy3]

src/virtualenv/activation/batch/__init__.py

@@ -17,7 +17,7 @@ def templates(self):
         yield Path("deactivate.bat")
         yield Path("pydoc.bat")
 
-    def instantiate_template(self, replacements, template):
+    def instantiate_template(self, replacements, template, creator):
         # ensure the text has all newlines as \r\n - required by batch
-        base = super(BatchActivator, self).instantiate_template(replacements, template)
+        base = super(BatchActivator, self).instantiate_template(replacements, template, creator)
         return base.replace(os.linesep, "\n").replace("\n", os.linesep)

src/virtualenv/activation/python/__init__.py

@@ -1,8 +1,11 @@
 from __future__ import absolute_import, unicode_literals
 
-import json
 import os
+from collections import OrderedDict
 
+import six
+
+from virtualenv.info import WIN_CPYTHON_2
 from virtualenv.util.path import Path
 
 from ..via_template import ViaTemplateActivator
@@ -14,6 +17,18 @@ def templates(self):
 
     def replacements(self, creator, dest_folder):
         replacements = super(PythonActivator, self).replacements(creator, dest_folder)
-        site_dump = json.dumps(list({os.path.relpath(str(i), str(dest_folder)) for i in creator.libs}), indent=2)
-        replacements.update({"__SITE_PACKAGES__": site_dump})
+        lib_folders = OrderedDict((os.path.relpath(str(i), str(dest_folder)), None) for i in creator.libs)
+        replacements.update(
+            {
+                "__LIB_FOLDERS__": six.ensure_text(os.pathsep.join(lib_folders.keys())),
+                "__DECODE_PATH__": ("yes" if WIN_CPYTHON_2 else ""),
+            }
+        )
         return replacements
+
+    @staticmethod
+    def _repr_unicode(creator, value):
+        py2 = creator.interpreter.version_info.major == 2
+        if py2:  # on Python 2 we need to encode this into explicit utf-8, py3 supports unicode literals
+            value = six.ensure_text(repr(value.encode("utf-8"))[1:-1])
+        return value

src/virtualenv/activation/python/activate_this.py

@@ -5,51 +5,28 @@
 
 This can be used when you must use an existing Python interpreter, not the virtualenv bin/python.
 """
-import json
 import os
 import site
 import sys
 
 try:
-    __file__
+    abs_file = os.path.abspath(__file__)
 except NameError:
     raise AssertionError("You must use exec(open(this_file).read(), {'__file__': this_file}))")
 
-
-def set_env(key, value, encoding):
-    if sys.version_info[0] == 2:
-        value = value.encode(encoding)
-    os.environ[key] = value
-
+bin_dir = os.path.dirname(abs_file)
+base = bin_dir[: -len("__BIN_NAME__") - 1]  # strip away the bin part from the __file__, plus the path separator
 
 # prepend bin to PATH (this file is inside the bin directory)
-bin_dir = os.path.dirname(os.path.abspath(__file__))
-set_env("PATH", os.pathsep.join([bin_dir] + os.environ.get("PATH", "").split(os.pathsep)), sys.getfilesystemencoding())
-
-base = os.path.dirname(bin_dir)
-
-# virtual env is right above bin directory
-set_env("VIRTUAL_ENV", base, sys.getfilesystemencoding())
-
-# add the virtual environments site-packages to the host python import mechanism
-prev = set(sys.path)
-
-site_packages = r"""
-__SITE_PACKAGES__
-"""
-
-for site_package in json.loads(site_packages):
-    if sys.version_info[0] == 2:
-        site_package = site_package.encode("utf-8").decode(sys.getfilesystemencoding())
-    path = os.path.realpath(os.path.join(os.path.dirname(__file__), site_package))
-    if sys.version_info[0] == 2:
-        path = path.encode(sys.getfilesystemencoding())
-    site.addsitedir(path)
+os.environ["PATH"] = os.pathsep.join([bin_dir] + os.environ.get("PATH", "").split(os.pathsep))
+os.environ["VIRTUAL_ENV"] = base  # virtual env is right above bin directory
 
+# add the virtual environments libraries to the host python import mechanism
+prev_length = len(sys.path)
+for lib in "__LIB_FOLDERS__".split(os.pathsep):
+    path = os.path.realpath(os.path.join(bin_dir, lib))
+    site.addsitedir(path.decode("utf-8") if "__DECODE_PATH__" else path)
+sys.path[:] = sys.path[prev_length:] + sys.path[0:prev_length]
 
 sys.real_prefix = sys.prefix
 sys.prefix = base
-
-# Move the added items to the front of the path, in place
-new = list(sys.path)
-sys.path[:] = [i for i in new if i not in prev] + [i for i in new if i in prev]

src/virtualenv/activation/via_template.py

@@ -22,7 +22,8 @@ def templates(self):
 
     def generate(self, creator):
         dest_folder = creator.bin_dir
-        self._generate(self.replacements(creator, dest_folder), self.templates(), dest_folder)
+        replacements = self.replacements(creator, dest_folder)
+        self._generate(replacements, self.templates(), dest_folder, creator)
         if self.flag_prompt is not None:
             creator.pyenv_cfg["prompt"] = self.flag_prompt
 
@@ -32,17 +33,23 @@ def replacements(self, creator, dest_folder):
             "__VIRTUAL_ENV__": six.ensure_text(str(creator.dest)),
             "__VIRTUAL_NAME__": creator.env_name,
             "__BIN_NAME__": six.ensure_text(str(creator.bin_dir.relative_to(creator.dest))),
-            "__PATH_SEP__": os.pathsep,
+            "__PATH_SEP__": six.ensure_text(os.pathsep),
         }
 
-    def _generate(self, replacements, templates, to_folder):
+    def _generate(self, replacements, templates, to_folder, creator):
         for template in templates:
-            text = self.instantiate_template(replacements, template)
+            text = self.instantiate_template(replacements, template, creator)
             (to_folder / template).write_text(text, encoding="utf-8")
 
-    def instantiate_template(self, replacements, template):
+    def instantiate_template(self, replacements, template, creator):
         # read text and do replacements
         text = read_text(self.__module__, str(template), encoding="utf-8", errors="strict")
-        for start, end in replacements.items():
-            text = text.replace(start, end)
+        for key, value in replacements.items():
+            value = self._repr_unicode(creator, value)
+            text = text.replace(key, value)
         return text
+
+    @staticmethod
+    def _repr_unicode(creator, value):
+        # by default we just let it be unicode
+        return value

src/virtualenv/create/creator.py

@@ -15,7 +15,7 @@
 from six import add_metaclass
 
 from virtualenv.discovery.py_info import Cmd
-from virtualenv.info import IS_ZIPAPP
+from virtualenv.info import IS_PYPY, IS_ZIPAPP
 from virtualenv.pyenv_cfg import PyEnvCfg
 from virtualenv.util.path import Path
 from virtualenv.util.subprocess import run_cmd
@@ -173,8 +173,10 @@ def debug_script(self):
 def get_env_debug_info(env_exe, debug_script):
     if IS_ZIPAPP:
         debug_script = extract_to_app_data(debug_script)
-    cmd = [six.ensure_text(str(env_exe)), six.ensure_text(str(debug_script))]
-    logging.debug("debug via %r", Cmd(cmd))
+    cmd = [str(env_exe), str(debug_script)]
+    if not IS_PYPY and six.PY2:
+        cmd = [six.ensure_text(i) for i in cmd]
+    logging.debug(str("debug via %r"), Cmd(cmd))
     env = os.environ.copy()
     env.pop(str("PYTHONPATH"), None)
     code, out, err = run_cmd(cmd)

src/virtualenv/create/debug.py

@@ -1,14 +1,19 @@
 """Inspect a target Python interpreter virtual environment wise"""
 import sys  # built-in
 
+PYPY2_WIN = hasattr(sys, "pypy_version_info") and sys.platform != "win32" and sys.version_info[0] == 2
+
 
 def encode_path(value):
     if value is None:
         return None
-    if isinstance(value, bytes):
-        return value.decode(sys.getfilesystemencoding())
-    elif not isinstance(value, str):
-        return repr(value if isinstance(value, type) else type(value))
+    if not isinstance(value, (str, bytes)):
+        if isinstance(value, type):
+            value = repr(value)
+        else:
+            value = repr(type(value))
+    if isinstance(value, bytes) and not PYPY2_WIN:
+        value = value.decode(sys.getfilesystemencoding())
     return value
 
 

src/virtualenv/create/describe.py

@@ -77,6 +77,9 @@ def exe_stem(cls):
         """executable name without suffix - there seems to be no standard way to get this without creating it"""
         raise NotImplementedError
 
+    def script(self, name):
+        return self.script_dir / "{}{}".format(name, self.suffix)
+
 
 @add_metaclass(ABCMeta)
 class Python2Supports(Describe):

src/virtualenv/create/via_global_ref/builtin/cpython/cpython2.py

@@ -1,10 +1,11 @@
 from __future__ import absolute_import, unicode_literals
 
 import abc
+import logging
 
 import six
 
-from virtualenv.create.via_global_ref.builtin.ref import RefToDest
+from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest
 from virtualenv.util.path import Path
 
 from ..python2.python2 import Python2
@@ -15,23 +16,50 @@
 class CPython2(CPython, Python2):
     """Create a CPython version 2  virtual environment"""
 
+    @classmethod
+    def sources(cls, interpreter):
+        for src in super(CPython2, cls).sources(interpreter):
+            yield src
+        # include folder needed on Python 2 as we don't have pyenv.cfg
+        host_include_marker = cls.host_include_marker(interpreter)
+        if host_include_marker.exists():
+            yield PathRefToDest(host_include_marker.parent, dest=lambda self, _: self.include)
+
+    @classmethod
+    def host_include_marker(cls, interpreter):
+        return Path(interpreter.system_include) / "Python.h"
+
+    @property
+    def include(self):
+        # the pattern include the distribution name too at the end, remove that via the parent call
+        return (self.dest / self.interpreter.distutils_install["headers"]).parent
+
     @classmethod
     def modules(cls):
         return [
             "os",  # landmark to set sys.prefix
         ]
 
+    def ensure_directories(self):
+        dirs = super(CPython2, self).ensure_directories()
+        host_include_marker = self.host_include_marker(self.interpreter)
+        if host_include_marker.exists():
+            dirs.add(self.include.parent)
+        else:
+            logging.debug("no include folders as can't find include marker %s", host_include_marker)
+        return dirs
+
 
 class CPython2Posix(CPython2, CPythonPosix):
     """CPython 2 on POSIX"""
 
     @classmethod
     def sources(cls, interpreter):
-        for src in super(CPythonPosix, cls).sources(interpreter):
+        for src in super(CPython2Posix, cls).sources(interpreter):
             yield src
         # landmark for exec_prefix
         name = "lib-dynload"
-        yield RefToDest(Path(interpreter.system_stdlib) / name, dest=cls.to_stdlib)
+        yield PathRefToDest(Path(interpreter.system_stdlib) / name, dest=cls.to_stdlib)
 
 
 class CPython2Windows(CPython2, CPythonWindows):
@@ -43,4 +71,8 @@ def sources(cls, interpreter):
             yield src
         py27_dll = Path(interpreter.system_executable).parent / "python27.dll"
         if py27_dll.exists():  # this might be global in the Windows folder in which case it's alright to be missing
-            yield RefToDest(py27_dll, dest=cls.to_bin)
+            yield PathRefToDest(py27_dll, dest=cls.to_bin)
+
+        libs = Path(interpreter.system_prefix) / "libs"
+        if libs.exists():
+            yield PathRefToDest(libs, dest=lambda self, s: self.dest / s.name)

src/virtualenv/create/via_global_ref/builtin/cpython/cpython3.py

@@ -5,7 +5,7 @@
 import six
 
 from virtualenv.create.describe import Python3Supports
-from virtualenv.create.via_global_ref.builtin.ref import RefToDest
+from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest
 from virtualenv.util.path import Path
 
 from .common import CPython, CPythonPosix, CPythonWindows
@@ -37,7 +37,7 @@ def include_dll_and_pyd(cls, interpreter):
         for folder in [host_exe_folder, dll_folder]:
             for file in folder.iterdir():
                 if file.suffix in (".pyd", ".dll"):
-                    yield RefToDest(file, dest=cls.to_dll_and_pyd)
+                    yield PathRefToDest(file, dest=cls.to_dll_and_pyd)
 
     def to_dll_and_pyd(self, src):
         return self.bin_dir / src.name

src/virtualenv/create/via_global_ref/builtin/pypy/common.py

@@ -4,7 +4,7 @@
 
 import six
 
-from virtualenv.create.via_global_ref.builtin.ref import RefToDest
+from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest
 from virtualenv.util.path import Path
 
 from ..via_global_self_do import ViaGlobalRefVirtualenvBuiltin
@@ -35,7 +35,7 @@ def sources(cls, interpreter):
         for src in super(PyPy, cls).sources(interpreter):
             yield src
         for host in cls._add_shared_libs(interpreter):
-            yield RefToDest(host, dest=cls.to_shared_lib)
+            yield PathRefToDest(host, dest=lambda self, s: self.bin_dir / s.name)
 
     @classmethod
     def _add_shared_libs(cls, interpreter):
@@ -46,9 +46,6 @@ def _add_shared_libs(cls, interpreter):
             if src.exists():
                 yield src
 
-    def to_shared_lib(self, src):
-        return [self.bin_dir]
-
     @classmethod
     def _shared_libs(cls):
         raise NotImplementedError