Skip to content

Commit

Permalink
Windows embedable support (#2353)
Browse files Browse the repository at this point in the history
* Bump pip and setuptools (#2348)

Signed-off-by: Bernát Gábor <[email protected]>

* Use shlex.quote instead of deprecated pipes.quote (#2351)

* Embeds the "python<VERSION>.zip" for Windows.

For example, for Python 3.10 the embeddable file name would be
"python310.zip". If this file would be found in `sys.path`, the
virtualenv should copy it into the "<venv>\Scripts\python310.zip".

* For Windows CPython3: *.dll/*.pyd -> to_bin

* Fixture for a Python interpreter info.

Helps to test virtualenv creator classes.

* Creators tests: path_mock as separate module.

* Clarifies tests, separates testing tools.

* Tests for CPython3Windows sources.

* Tests for the embedded Python std lib for Windows.

* Add news entry.

* Replaces `yield from` for backward compability.

* FIX: Path mocking in pypy tests.

* Wrap `sys` `Path` with `str` for importlib.

The importlib accepts a Path-like objects from Python 3.6

* Makes PathMock ABC compatible with Python 2

* Does not collect tests for Python3 under Python 2

It is possible to make pass CPython3 tests under Python 2,
but it's better to disable it instead of decreasing the
readability and performance of Python 3 style.

* Allows empty `Path()` in Windows with Python 2

* Allows to load fixture files with PY2 Windows Path

* Skips one PY3 POSIX test in PY2 Windows

Co-authored-by: Bernát Gábor <[email protected]>
Co-authored-by: Lumír 'Frenzy' Balhar <[email protected]>
  • Loading branch information
3 people authored Jun 25, 2022
1 parent b01515b commit 805dcff
Show file tree
Hide file tree
Showing 14 changed files with 438 additions and 102 deletions.
2 changes: 2 additions & 0 deletions docs/changelog/1774.feature.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Support for Windows embeddable Python package: includes ``python<VERSION>.zip``
in the creator sources.
49 changes: 40 additions & 9 deletions src/virtualenv/create/via_global_ref/builtin/cpython/cpython3.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
from __future__ import absolute_import, unicode_literals

import abc
import fnmatch
from itertools import chain
from operator import methodcaller as method
from textwrap import dedent

from six import add_metaclass
Expand Down Expand Up @@ -53,11 +56,20 @@ def setup_meta(cls, interpreter):

@classmethod
def sources(cls, interpreter):
for src in super(CPython3Windows, cls).sources(interpreter):
yield src
if not cls.has_shim(interpreter):
for src in cls.include_dll_and_pyd(interpreter):
yield src
if cls.has_shim(interpreter):
refs = cls.executables(interpreter)
else:
refs = chain(
cls.executables(interpreter),
cls.dll_and_pyd(interpreter),
cls.python_zip(interpreter),
)
for ref in refs:
yield ref

@classmethod
def executables(cls, interpreter):
return super(CPython3Windows, cls).sources(interpreter)

@classmethod
def has_shim(cls, interpreter):
Expand All @@ -79,13 +91,32 @@ def host_python(cls, interpreter):
return super(CPython3Windows, cls).host_python(interpreter)

@classmethod
def include_dll_and_pyd(cls, interpreter):
def dll_and_pyd(cls, interpreter):
dll_folder = Path(interpreter.system_prefix) / "DLLs"
host_exe_folder = Path(interpreter.system_executable).parent
for folder in [host_exe_folder, dll_folder]:
for file in folder.iterdir():
if file.suffix in (".pyd", ".dll"):
yield PathRefToDest(file, dest=cls.to_dll_and_pyd)
yield PathRefToDest(file, cls.to_bin)

def to_dll_and_pyd(self, src):
return self.bin_dir / src.name
@classmethod
def python_zip(cls, interpreter):
"""
"python{VERSION}.zip" contains compiled *.pyc std lib packages, where
"VERSION" is `py_version_nodot` var from the `sysconfig` module.
:see: https://docs.python.org/3/using/windows.html#the-embeddable-package
:see: `discovery.py_info.PythonInfo` class (interpreter).
:see: `python -m sysconfig` output.
:note: The embeddable Python distribution for Windows includes
"python{VERSION}.zip" and "python{VERSION}._pth" files. User can
move/rename *zip* file and edit `sys.path` by editing *_pth* file.
Here the `pattern` is used only for the default *zip* file name!
"""
pattern = "*python{}.zip".format(interpreter.version_nodot)
matches = fnmatch.filter(interpreter.path, pattern)
matched_paths = map(Path, matches)
existing_paths = filter(method("exists"), matched_paths)
path = next(existing_paths, None)
if path is not None:
yield PathRefToDest(path, cls.to_bin)
4 changes: 4 additions & 0 deletions src/virtualenv/discovery/py_info.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,10 @@ def abs_path(v):
self.version_info = VersionInfo(*list(u(i) for i in sys.version_info))
self.architecture = 64 if sys.maxsize > 2**32 else 32

# Used to determine some file names.
# See `CPython3Windows.python_zip()`.
self.version_nodot = sysconfig.get_config_var("py_version_nodot")

self.version = u(sys.version)
self.os = u(os.name)

Expand Down
Binary file not shown.
11 changes: 10 additions & 1 deletion src/virtualenv/util/path/_pathlib/via_os_path.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from __future__ import absolute_import, unicode_literals

import fnmatch
import os
import platform
from contextlib import contextmanager
Expand All @@ -10,7 +11,7 @@


class Path(object):
def __init__(self, path):
def __init__(self, path=""):
if isinstance(path, Path):
_path = path._path
else:
Expand Down Expand Up @@ -147,5 +148,13 @@ def chmod(self, mode):
def absolute(self):
return Path(os.path.abspath(self._path))

def rglob(self, pattern):
"""
Rough emulation of the origin method. Just for searching fixture files.
"""
for root, _dirs, files in os.walk(self._path):
for filename in fnmatch.filter(files, pattern):
yield Path(os.path.join(root, filename))


__all__ = ("Path",)
10 changes: 9 additions & 1 deletion tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
from virtualenv.app_data import AppDataDiskFolder
from virtualenv.discovery.builtin import get_interpreter
from virtualenv.discovery.py_info import PythonInfo
from virtualenv.info import IS_PYPY, IS_WIN, fs_supports_symlink
from virtualenv.info import IS_PYPY, IS_WIN, PY2, fs_supports_symlink
from virtualenv.report import LOGGER
from virtualenv.util.path import Path
from virtualenv.util.six import ensure_str, ensure_text
Expand Down Expand Up @@ -388,3 +388,11 @@ def skip_if_test_in_system(session_app_data):
current = PythonInfo.current(session_app_data)
if current.system_executable is not None:
pytest.skip("test not valid if run under system")


def pytest_ignore_collect(path):
"""
We can't just skip these tests due to syntax errors that occurs during
collecting tests under a Python 2 host.
"""
return PY2 and str(path).endswith("test_cpython3_win.py")
25 changes: 25 additions & 0 deletions tests/unit/create/via_global_ref/builtin/conftest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
import sys

import pytest
from testing import path
from testing.py_info import read_fixture

from virtualenv.util.path import Path

# Allows to import from `testing` into test submodules.
sys.path.append(str(Path(__file__).parent))


@pytest.fixture
def py_info(py_info_name):
return read_fixture(py_info_name)


@pytest.fixture
def mock_files(mocker):
return lambda paths, files: path.mock_files(mocker, paths, files)


@pytest.fixture
def mock_pypy_libs(mocker):
return lambda pypy, libs: path.mock_pypy_libs(mocker, pypy, libs)
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
{
"platform": "win32",
"implementation": "CPython",
"version_info": {
"major": 3,
"minor": 10,
"micro": 4,
"releaselevel": "final",
"serial": 0
},
"architecture": 64,
"version_nodot": "310",
"version": "3.10.4 (tags/v3.10.4:9d38120, Mar 23 2022, 23:13:41) [MSC v.1929 64 bit (AMD64)]",
"os": "nt",
"prefix": "c:\\path\\to\\python",
"base_prefix": "c:\\path\\to\\python",
"real_prefix": null,
"base_exec_prefix": "c:\\path\\to\\python",
"exec_prefix": "c:\\path\\to\\python",
"executable": "c:\\path\\to\\python\\python.exe",
"original_executable": "c:\\path\\to\\python\\python.exe",
"system_executable": "c:\\path\\to\\python\\python.exe",
"has_venv": false,
"path": [
"c:\\path\\to\\python\\Scripts\\virtualenv.exe",
"c:\\path\\to\\python\\python310.zip",
"c:\\path\\to\\python",
"c:\\path\\to\\python\\Lib\\site-packages"
],
"file_system_encoding": "utf-8",
"stdout_encoding": "utf-8",
"sysconfig_scheme": null,
"sysconfig_paths": {
"stdlib": "{installed_base}/Lib",
"platstdlib": "{base}/Lib",
"purelib": "{base}/Lib/site-packages",
"platlib": "{base}/Lib/site-packages",
"include": "{installed_base}/Include",
"scripts": "{base}/Scripts",
"data": "{base}"
},
"distutils_install": {
"purelib": "Lib\\site-packages",
"platlib": "Lib\\site-packages",
"headers": "Include\\UNKNOWN",
"scripts": "Scripts",
"data": ""
},
"sysconfig": {
"makefile_filename": "c:\\path\\to\\python\\Lib\\config\\Makefile"
},
"sysconfig_vars": {
"PYTHONFRAMEWORK": "",
"installed_base": "c:\\path\\to\\python",
"base": "c:\\path\\to\\python"
},
"system_stdlib": "c:\\path\\to\\python\\Lib",
"system_stdlib_platform": "c:\\path\\to\\python\\Lib",
"max_size": 9223372036854775807,
"_creators": null
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
import pytest
from testing.helpers import contains_exe, contains_ref
from testing.path import join as path

from virtualenv.create.via_global_ref.builtin.cpython.cpython3 import CPython3Windows

CPYTHON3_PATH = (
"virtualenv.create.via_global_ref.builtin.cpython.common.Path",
"virtualenv.create.via_global_ref.builtin.cpython.cpython3.Path",
)


@pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"])
def test_2_exe_on_default_py_host(py_info, mock_files):
mock_files(CPYTHON3_PATH, [py_info.system_executable])
sources = tuple(CPython3Windows.sources(interpreter=py_info))
# Default Python exe.
assert contains_exe(sources, py_info.system_executable)
# Should always exist.
assert contains_exe(sources, path(py_info.prefix, "pythonw.exe"))


@pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"])
def test_3_exe_on_not_default_py_host(py_info, mock_files):
# Not default python host.
py_info.system_executable = path(py_info.prefix, "python666.exe")
mock_files(CPYTHON3_PATH, [py_info.system_executable])
sources = tuple(CPython3Windows.sources(interpreter=py_info))
# Not default Python exe linked to both the default name and origin.
assert contains_exe(sources, py_info.system_executable, "python.exe")
assert contains_exe(sources, py_info.system_executable, "python666.exe")
# Should always exist.
assert contains_exe(sources, path(py_info.prefix, "pythonw.exe"))


@pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"])
def test_only_shim(py_info, mock_files):
shim = path(py_info.system_stdlib, "venv\\scripts\\nt\\python.exe")
py_files = (
path(py_info.prefix, "libcrypto-1_1.dll"),
path(py_info.prefix, "libffi-7.dll"),
path(py_info.prefix, "_asyncio.pyd"),
path(py_info.prefix, "_bz2.pyd"),
)
mock_files(CPYTHON3_PATH, [shim, *py_files])
sources = tuple(CPython3Windows.sources(interpreter=py_info))
assert CPython3Windows.has_shim(interpreter=py_info)
assert contains_exe(sources, shim)
assert not contains_exe(sources, py_info.system_executable)
for file in py_files:
assert not contains_ref(sources, file)


@pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"])
def test_exe_dll_pyd_without_shim(py_info, mock_files):
py_files = (
path(py_info.prefix, "libcrypto-1_1.dll"),
path(py_info.prefix, "libffi-7.dll"),
path(py_info.prefix, "_asyncio.pyd"),
path(py_info.prefix, "_bz2.pyd"),
)
mock_files(CPYTHON3_PATH, py_files)
sources = tuple(CPython3Windows.sources(interpreter=py_info))
assert not CPython3Windows.has_shim(interpreter=py_info)
assert contains_exe(sources, py_info.system_executable)
for file in py_files:
assert contains_ref(sources, file)


@pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"])
def test_python_zip_if_exists_and_set_in_path(py_info, mock_files):
python_zip_name = "python{}.zip".format(py_info.version_nodot)
python_zip = path(py_info.prefix, python_zip_name)
mock_files(CPYTHON3_PATH, [python_zip])
sources = tuple(CPython3Windows.sources(interpreter=py_info))
assert python_zip in py_info.path
assert contains_ref(sources, python_zip)


@pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"])
def test_no_python_zip_if_exists_and_not_set_in_path(py_info, mock_files):
python_zip_name = "python{}.zip".format(py_info.version_nodot)
python_zip = path(py_info.prefix, python_zip_name)
py_info.path.remove(python_zip)
mock_files(CPYTHON3_PATH, [python_zip])
sources = tuple(CPython3Windows.sources(interpreter=py_info))
assert python_zip not in py_info.path
assert not contains_ref(sources, python_zip)


@pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"])
def test_no_python_zip_if_not_exists(py_info, mock_files):
python_zip_name = "python{}.zip".format(py_info.version_nodot)
python_zip = path(py_info.prefix, python_zip_name)
# No `python_zip`, just python.exe file.
mock_files(CPYTHON3_PATH, [py_info.system_executable])
sources = tuple(CPython3Windows.sources(interpreter=py_info))
assert python_zip in py_info.path
assert not contains_ref(sources, python_zip)
Loading

0 comments on commit 805dcff

Please sign in to comment.