From 47e1d35e614d972c6f4d5b1c980191e00a70fa8d Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Wed, 20 May 2020 10:36:23 +0200 Subject: [PATCH] rpm wheels --- src/virtualenv/seed/embed/pip_invoke.py | 10 ++-- src/virtualenv/seed/embed/wheels/__init__.py | 59 +++++++++++++++++++ src/virtualenv/seed/embed/wheels/acquire.py | 20 ++++--- .../seed/via_app_data/via_app_data.py | 2 +- 4 files changed, 76 insertions(+), 15 deletions(-) diff --git a/src/virtualenv/seed/embed/pip_invoke.py b/src/virtualenv/seed/embed/pip_invoke.py index 74c96f1bb..25cc7e87a 100644 --- a/src/virtualenv/seed/embed/pip_invoke.py +++ b/src/virtualenv/seed/embed/pip_invoke.py @@ -23,8 +23,8 @@ def __init__(self, options): def run(self, creator): if not self.enabled: return - with self.get_pip_install_cmd(creator.exe, creator.interpreter.version_release_str) as cmd: - with pip_wheel_env_run(creator.interpreter.version_release_str, self.app_data) as env: + with self.get_pip_install_cmd(creator) as cmd: + with pip_wheel_env_run(creator, self.app_data) as env: logging.debug("pip seed by running: %s", LogCmd(cmd, env)) process = Popen(cmd, env=env) process.communicate() @@ -32,8 +32,8 @@ def run(self, creator): raise RuntimeError("failed seed with code {}".format(process.returncode)) @contextmanager - def get_pip_install_cmd(self, exe, version): - cmd = [str(exe), "-m", "pip", "-q", "install", "--only-binary", ":all:"] + def get_pip_install_cmd(self, creator): + cmd = [str(creator.exe), "-m", "pip", "-q", "install", "--only-binary", ":all:"] if not self.download: cmd.append("--no-index") pkg_versions = self.package_version() @@ -41,7 +41,7 @@ def get_pip_install_cmd(self, exe, version): cmd.append("{}{}".format(key, "=={}".format(ver) if ver is not None else "")) with ExitStack() as stack: folders = set() - for context in (ensure_file_on_disk(get_bundled_wheel(p, version), self.app_data) for p in pkg_versions): + for context in (ensure_file_on_disk(get_bundled_wheel(p, creator), self.app_data) for p in pkg_versions): folders.add(stack.enter_context(context).parent) for folder in folders: cmd.extend(["--find-links", str(folder)]) diff --git a/src/virtualenv/seed/embed/wheels/__init__.py b/src/virtualenv/seed/embed/wheels/__init__.py index 826272b13..6db311646 100644 --- a/src/virtualenv/seed/embed/wheels/__init__.py +++ b/src/virtualenv/seed/embed/wheels/__init__.py @@ -1,4 +1,7 @@ from __future__ import absolute_import, unicode_literals +from subprocess import check_output, CalledProcessError + +from virtualenv.util.path import Path BUNDLE_SUPPORT = { "3.9": { @@ -38,3 +41,59 @@ }, } MAX = "3.9" + + +# The mapping above is overwritten by the code below this comment. +# It's intentionaly left here so the patch will stay the same even +# for future changes of the versions. +class SystemWheels: + def __getitem__(self, creator): + bundled = ["pip", "setuptools", "wheel"] + # creator might be an instance of the internal Creator object + # or str/Path with a path to Python executable + if isinstance(creator, str): + executable = creator + else: + executable = creator.exe + paths = [] + result = {} + + # ensurepip path + # We need subprocess here to check ensurepip with the Python we are creating + # a new virtual environment for + try: + ensurepip_path = check_output((executable, "-u", "-c", + 'import ensurepip; print(ensurepip.__path__[0])'), + universal_newlines=True) + ensurepip_path = Path(ensurepip_path.strip()) / "_bundled" + except CalledProcessError: + pass + else: + if ensurepip_path.is_dir(): + paths.append(ensurepip_path) + + # Standard wheels path + wheels_dir = Path("/usr/share/python-wheels") + if wheels_dir.exists(): + paths.append(wheels_dir) + + # Find and use the first wheel for all bundled packages + # ensurepip takes precedence (if exists) + for package in bundled: + result[package] = None + for path in paths: + wheels = list(path.glob(package + "-*.whl")) + if wheels: + result[package] = wheels[0] + break + + return result + + def get(self, key, default=None): + return self.__getitem__(key) + + +BUNDLE_SUPPORT = SystemWheels() + +# We should never ever need this but it has to stay importable +MAX = None diff --git a/src/virtualenv/seed/embed/wheels/acquire.py b/src/virtualenv/seed/embed/wheels/acquire.py index 28ac6a770..a19cf2b05 100644 --- a/src/virtualenv/seed/embed/wheels/acquire.py +++ b/src/virtualenv/seed/embed/wheels/acquire.py @@ -30,11 +30,12 @@ def __init__(self, packages, for_py_version, exit_code, out, err): self.err = err.strip() -def get_wheels(for_py_version, wheel_cache_dir, extra_search_dir, packages, app_data, download): +def get_wheels(creator, wheel_cache_dir, extra_search_dir, packages, app_data, download): + for_py_version = creator.interpreter.version_release_str # not all wheels are compatible with all python versions, so we need to py version qualify it processed = copy(packages) # 1. acquire from bundle - acquire_from_bundle(processed, for_py_version, wheel_cache_dir) + acquire_from_bundle(processed, creator, wheel_cache_dir) # 2. acquire from extra search dir acquire_from_dir(processed, for_py_version, wheel_cache_dir, extra_search_dir) # 3. download from the internet @@ -46,9 +47,9 @@ def get_wheels(for_py_version, wheel_cache_dir, extra_search_dir, packages, app_ return {p: next(iter(ver_to_files))[1] for p, ver_to_files in wheels.items()} -def acquire_from_bundle(packages, for_py_version, to_folder): +def acquire_from_bundle(packages, creator, to_folder): for pkg, version in list(packages.items()): - bundle = get_bundled_wheel(pkg, for_py_version) + bundle = get_bundled_wheel(pkg, creator) if bundle is not None: pkg_version = bundle.stem.split("-")[1] exact_version_match = version == pkg_version @@ -66,8 +67,8 @@ def acquire_from_bundle(packages, for_py_version, to_folder): copy2(str(bundle), str(bundled_wheel_file)) -def get_bundled_wheel(package, version_release): - return BUNDLE_FOLDER / (BUNDLE_SUPPORT.get(version_release, {}) or BUNDLE_SUPPORT[MAX]).get(package) +def get_bundled_wheel(package, creator): + return BUNDLE_FOLDER / (BUNDLE_SUPPORT.get(creator, {}) or BUNDLE_SUPPORT[MAX]).get(package) def acquire_from_dir(packages, for_py_version, to_folder, extra_search_dir): @@ -156,7 +157,7 @@ def download_wheel(packages, for_py_version, to_folder, app_data): cmd.extend(to_download) # pip has no interface in python - must be a new sub-process - with pip_wheel_env_run("{}.{}".format(*sys.version_info[0:2]), app_data) as env: + with pip_wheel_env_run(sys.executable, app_data) as env: process = Popen(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) out, err = process.communicate() if process.returncode != 0: @@ -164,7 +165,7 @@ def download_wheel(packages, for_py_version, to_folder, app_data): @contextmanager -def pip_wheel_env_run(version, app_data): +def pip_wheel_env_run(creator, app_data): env = os.environ.copy() env.update( { @@ -172,7 +173,8 @@ def pip_wheel_env_run(version, app_data): for k, v in {"PIP_USE_WHEEL": "1", "PIP_USER": "0", "PIP_NO_INPUT": "1"}.items() } ) - with ensure_file_on_disk(get_bundled_wheel("pip", version), app_data) as pip_wheel_path: + print("Calling ge_bundled_wheel with", sys.executable, creator) + with ensure_file_on_disk(get_bundled_wheel("pip", creator), app_data) as pip_wheel_path: # put the bundled wheel onto the path, and use it to do the bootstrap operation env[str("PYTHONPATH")] = str(pip_wheel_path) yield env diff --git a/src/virtualenv/seed/via_app_data/via_app_data.py b/src/virtualenv/seed/via_app_data/via_app_data.py index 3d4b7c720..14ae4384f 100644 --- a/src/virtualenv/seed/via_app_data/via_app_data.py +++ b/src/virtualenv/seed/via_app_data/via_app_data.py @@ -69,7 +69,7 @@ def _get_seed_wheels(self, creator, base_cache): def _get(package, version): wheel_loader = partial( get_wheels, - creator.interpreter.version_release_str, + creator, wheels_to, self.extra_search_dir, {package: version},