From 0ec9320608177852c317e590fdfbcd02fbe5d9b7 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Sat, 20 Apr 2024 11:52:06 -0400 Subject: [PATCH 1/4] Add support for embedded Python on Windows --- .github/workflows/ci.yml | 34 ++++++++++ crates/uv-virtualenv/src/bare.rs | 110 +++++++++++++++++++++++++------ scripts/check_embedded_python.py | 88 +++++++++++++++++++++++++ 3 files changed, 212 insertions(+), 20 deletions(-) create mode 100755 scripts/check_embedded_python.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 57b8e1c41996..effee9bc388a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -689,6 +689,40 @@ jobs: - name: "Validate global Python install" run: py -3.13 ./scripts/check_system_python.py --uv ./uv.exe + system-test-windows-embedded-python-310: + needs: build-binary-windows + name: "check system | embedded python3.10 on windows" + runs-on: windows-latest + env: + # Avoid debug build stack overflows. + UV_STACK_SIZE: 2000000 # 2 megabyte, double the default on windows + steps: + - uses: actions/checkout@v4 + + - name: "Download binary" + uses: actions/download-artifact@v4 + with: + name: uv-windows-${{ github.sha }} + + # Download embedded Python. + - name: "Download embedded Python" + run: curl -LsSf https://www.python.org/ftp/python/3.11.8/python-3.11.8-embed-amd64.zip -o python-3.11.8-embed-amd64.zip + + - name: "Unzip embedded Python" + run: 7z x python-3.11.8-embed-amd64.zip -oembedded-python + + - name: "Show embedded Python contents" + run: ls embedded-python + + - name: "Set PATH" + run: echo "${{ github.workspace }}\embedded-python" >> $env:GITHUB_PATH + + - name: "Print Python path" + run: echo $(which python) + + - name: "Validate global Python install" + run: python ./scripts/check_embedded_python.py --uv ./uv.exe + system-test-choco: needs: build-binary-windows name: "check system | python3.12 via chocolatey" diff --git a/crates/uv-virtualenv/src/bare.rs b/crates/uv-virtualenv/src/bare.rs index a5090fdeca31..bad9c9dd24e6 100644 --- a/crates/uv-virtualenv/src/bare.rs +++ b/crates/uv-virtualenv/src/bare.rs @@ -139,6 +139,14 @@ pub fn create_bare_venv( // Create a `.gitignore` file to ignore all files in the venv. fs::write(location.join(".gitignore"), "*")?; + // Per PEP 405, the Python `home` is the parent directory of the interpreter. + let python_home = base_python.parent().ok_or_else(|| { + io::Error::new( + io::ErrorKind::NotFound, + "The Python interpreter needs to have a parent directory", + ) + })?; + // Different names for the python interpreter fs::create_dir(&scripts)?; let executable = scripts.join(format!("python{EXE_SUFFIX}")); @@ -167,8 +175,10 @@ pub fn create_bare_venv( { // https://github.com/python/cpython/blob/d457345bbc6414db0443819290b04a9a4333313d/Lib/venv/__init__.py#L261-L267 // https://github.com/pypa/virtualenv/blob/d9fdf48d69f0d0ca56140cf0381edbb5d6fe09f5/src/virtualenv/create/via_global_ref/builtin/cpython/cpython3.py#L78-L83 - // There's two kinds of applications on windows: Those that allocate a console (python.exe) and those that - // don't because they use window(s) (pythonw.exe). + // There's two kinds of applications on windows: Those that allocate a console (python.exe) + // and those that don't because they use window(s) (pythonw.exe). + + // First priority: the `python.exe` and `pythonw.exe` shims. for python_exe in ["python.exe", "pythonw.exe"] { let shim = interpreter .stdlib() @@ -179,13 +189,14 @@ pub fn create_bare_venv( match fs_err::copy(shim, scripts.join(python_exe)) { Ok(_) => {} Err(err) if err.kind() == io::ErrorKind::NotFound => { + // Second priority: the `venvlauncher.exe` and `venvwlauncher.exe` shims. + // These are equivalent to the `python.exe` and `pythonw.exe` shims, which were + // renamed in Python 3.13. let launcher = match python_exe { "python.exe" => "venvlauncher.exe", "pythonw.exe" => "venvwlauncher.exe", _ => unreachable!(), }; - - // If `python.exe` doesn't exist, try the `venvlauncher.exe` shim. let shim = interpreter .stdlib() .join("venv") @@ -193,13 +204,81 @@ pub fn create_bare_venv( .join("nt") .join(launcher); - // If the `venvlauncher.exe` shim doesn't exist, then on Conda at least, we - // can look for it next to the Python executable itself. match fs_err::copy(shim, scripts.join(python_exe)) { Ok(_) => {} Err(err) if err.kind() == io::ErrorKind::NotFound => { + // Third priority: on Conda at least, we can look for the launcher shim next to + // the Python executable itself. let shim = base_python.with_file_name(launcher); - fs_err::copy(shim, scripts.join(python_exe))?; + match fs_err::copy(shim, scripts.join(python_exe)) { + Ok(_) => {} + Err(err) if err.kind() == io::ErrorKind::NotFound => { + // Fourth priority: if the launcher shim doesn't exist, assume this is + // an embedded Python. Copy the Python executable itself, along with + // the DLLs, `.pyd` files, and `.zip` files in the same directory. + fs_err::copy( + base_python.with_file_name(python_exe), + scripts.join(python_exe), + )?; + + // Copy `.dll` and `.pyd` files from the top-level, and from the + // `DLLs` subdirectory (if it exists). + for directory in [ + python_home, + interpreter.base_prefix().join("DLLs").as_path(), + ] { + let entries = match fs_err::read_dir(directory) { + Ok(read_dir) => read_dir, + Err(err) if err.kind() == io::ErrorKind::NotFound => { + continue; + } + Err(err) => { + return Err(err.into()); + } + }; + for entry in entries { + let entry = entry?; + let path = entry.path(); + if let Some(ext) = path.extension() { + if ext == "dll" || ext == "pyd" { + if let Some(file_name) = path.file_name() { + fs_err::copy( + &path, + scripts.join(file_name), + )?; + } + } + } + } + } + + // Copy `.zip` files from the top-level. + let entries = match fs_err::read_dir(python_home) { + Ok(read_dir) => read_dir, + Err(err) if err.kind() == io::ErrorKind::NotFound => { + continue; + } + Err(err) => { + return Err(err.into()); + } + }; + + for entry in entries { + let entry = entry?; + let path = entry.path(); + if let Some(ext) = path.extension() { + if ext == "zip" { + if let Some(file_name) = path.file_name() { + fs_err::copy(&path, scripts.join(file_name))?; + } + } + } + } + } + Err(err) => { + return Err(err.into()); + } + } } Err(err) => { return Err(err.into()); @@ -242,18 +321,6 @@ pub fn create_bare_venv( fs::write(scripts.join(name), activator)?; } - // Per PEP 405, the Python `home` is the parent directory of the interpreter. - let python_home = base_python - .parent() - .ok_or_else(|| { - io::Error::new( - io::ErrorKind::NotFound, - "The Python interpreter needs to have a parent directory", - ) - })? - .simplified_display() - .to_string(); - // Validate extra_cfg let reserved_keys = [ "home", @@ -272,7 +339,10 @@ pub fn create_bare_venv( } let mut pyvenv_cfg_data: Vec<(String, String)> = vec![ - ("home".to_string(), python_home), + ( + "home".to_string(), + python_home.simplified_display().to_string(), + ), ( "implementation".to_string(), interpreter.markers().platform_python_implementation.clone(), diff --git a/scripts/check_embedded_python.py b/scripts/check_embedded_python.py new file mode 100755 index 000000000000..81b3f6c6c42d --- /dev/null +++ b/scripts/check_embedded_python.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 + +"""Install `pylint` and `numpy` into an embedded Python.""" + +import argparse +import logging +import os +import subprocess +import sys +import tempfile + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") + + parser = argparse.ArgumentParser(description="Check a Python interpreter.") + parser.add_argument("--uv", help="Path to a uv binary.") + args = parser.parse_args() + + uv: str = os.path.abspath(args.uv) if args.uv else "uv" + + # Create a temporary directory. + with tempfile.TemporaryDirectory() as temp_dir: + # Create a virtual environment with `uv`. + logging.info("Creating virtual environment with `uv`...") + subprocess.run( + [uv, "venv", ".venv", "--seed", "--python", sys.executable], + cwd=temp_dir, + check=True, + ) + + if os.name == "nt": + executable = os.path.join(temp_dir, ".venv", "Scripts", "python.exe") + else: + executable = os.path.join(temp_dir, ".venv", "bin", "python") + + logging.info("Querying virtual environment...") + subprocess.run( + [executable, "--version"], + cwd=temp_dir, + check=True, + ) + + logging.info("Installing into `uv` virtual environment...") + + # Disable the `CONDA_PREFIX` and `VIRTUAL_ENV` environment variables, so that + # we only rely on virtual environment discovery via the `.venv` directory. + # Our "system Python" here might itself be a Conda environment! + env = os.environ.copy() + env["CONDA_PREFIX"] = "" + env["VIRTUAL_ENV"] = "" + subprocess.run( + [uv, "pip", "install", "pylint", "--verbose"], + cwd=temp_dir, + check=True, + env=env, + ) + + # Ensure that the package (`pylint`) is installed in the virtual environment. + logging.info("Checking that `pylint` is installed.") + code = subprocess.run( + [executable, "-c", "import pylint"], + cwd=temp_dir, + ) + if code.returncode != 0: + raise Exception( + "The package `pylint` isn't installed in the virtual environment." + ) + + # Uninstall the package (`pylint`). + logging.info("Uninstalling the package `pylint`.") + subprocess.run( + [uv, "pip", "uninstall", "pylint", "--verbose"], + cwd=temp_dir, + check=True, + env=env, + ) + + # Ensure that the package (`pylint`) isn't installed in the virtual environment. + logging.info("Checking that `pylint` isn't installed.") + code = subprocess.run( + [executable, "-m", "pip", "show", "pylint"], + cwd=temp_dir, + ) + if code.returncode == 0: + raise Exception( + "The package `pylint` is installed in the virtual environment (but shouldn't be)." + ) From 08c219386c58906c9657230d5aa12eac4d281c03 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Sat, 20 Apr 2024 17:40:17 -0400 Subject: [PATCH 2/4] Reverts --- .github/workflows/ci.yml | 68 +++++++++---------- crates/uv-virtualenv/src/bare.rs | 110 ++++++------------------------- 2 files changed, 54 insertions(+), 124 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index effee9bc388a..f0de2809a370 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -689,40 +689,6 @@ jobs: - name: "Validate global Python install" run: py -3.13 ./scripts/check_system_python.py --uv ./uv.exe - system-test-windows-embedded-python-310: - needs: build-binary-windows - name: "check system | embedded python3.10 on windows" - runs-on: windows-latest - env: - # Avoid debug build stack overflows. - UV_STACK_SIZE: 2000000 # 2 megabyte, double the default on windows - steps: - - uses: actions/checkout@v4 - - - name: "Download binary" - uses: actions/download-artifact@v4 - with: - name: uv-windows-${{ github.sha }} - - # Download embedded Python. - - name: "Download embedded Python" - run: curl -LsSf https://www.python.org/ftp/python/3.11.8/python-3.11.8-embed-amd64.zip -o python-3.11.8-embed-amd64.zip - - - name: "Unzip embedded Python" - run: 7z x python-3.11.8-embed-amd64.zip -oembedded-python - - - name: "Show embedded Python contents" - run: ls embedded-python - - - name: "Set PATH" - run: echo "${{ github.workspace }}\embedded-python" >> $env:GITHUB_PATH - - - name: "Print Python path" - run: echo $(which python) - - - name: "Validate global Python install" - run: python ./scripts/check_embedded_python.py --uv ./uv.exe - system-test-choco: needs: build-binary-windows name: "check system | python3.12 via chocolatey" @@ -848,3 +814,37 @@ jobs: - name: "Validate global Python install" run: python3 scripts/check_system_python.py --uv ./uv + + system-test-windows-embedded-python-310: + needs: build-binary-windows + name: "check system | embedded python3.10 on windows" + runs-on: windows-latest + env: + # Avoid debug build stack overflows. + UV_STACK_SIZE: 2000000 # 2 megabyte, double the default on windows + steps: + - uses: actions/checkout@v4 + + - name: "Download binary" + uses: actions/download-artifact@v4 + with: + name: uv-windows-${{ github.sha }} + + # Download embedded Python. + - name: "Download embedded Python" + run: curl -LsSf https://www.python.org/ftp/python/3.11.8/python-3.11.8-embed-amd64.zip -o python-3.11.8-embed-amd64.zip + + - name: "Unzip embedded Python" + run: 7z x python-3.11.8-embed-amd64.zip -oembedded-python + + - name: "Show embedded Python contents" + run: ls embedded-python + + - name: "Set PATH" + run: echo "${{ github.workspace }}\embedded-python" >> $env:GITHUB_PATH + + - name: "Print Python path" + run: echo $(which python) + + - name: "Validate embedded Python install" + run: python ./scripts/check_embedded_python.py --uv ./uv.exe diff --git a/crates/uv-virtualenv/src/bare.rs b/crates/uv-virtualenv/src/bare.rs index bad9c9dd24e6..a5090fdeca31 100644 --- a/crates/uv-virtualenv/src/bare.rs +++ b/crates/uv-virtualenv/src/bare.rs @@ -139,14 +139,6 @@ pub fn create_bare_venv( // Create a `.gitignore` file to ignore all files in the venv. fs::write(location.join(".gitignore"), "*")?; - // Per PEP 405, the Python `home` is the parent directory of the interpreter. - let python_home = base_python.parent().ok_or_else(|| { - io::Error::new( - io::ErrorKind::NotFound, - "The Python interpreter needs to have a parent directory", - ) - })?; - // Different names for the python interpreter fs::create_dir(&scripts)?; let executable = scripts.join(format!("python{EXE_SUFFIX}")); @@ -175,10 +167,8 @@ pub fn create_bare_venv( { // https://github.com/python/cpython/blob/d457345bbc6414db0443819290b04a9a4333313d/Lib/venv/__init__.py#L261-L267 // https://github.com/pypa/virtualenv/blob/d9fdf48d69f0d0ca56140cf0381edbb5d6fe09f5/src/virtualenv/create/via_global_ref/builtin/cpython/cpython3.py#L78-L83 - // There's two kinds of applications on windows: Those that allocate a console (python.exe) - // and those that don't because they use window(s) (pythonw.exe). - - // First priority: the `python.exe` and `pythonw.exe` shims. + // There's two kinds of applications on windows: Those that allocate a console (python.exe) and those that + // don't because they use window(s) (pythonw.exe). for python_exe in ["python.exe", "pythonw.exe"] { let shim = interpreter .stdlib() @@ -189,14 +179,13 @@ pub fn create_bare_venv( match fs_err::copy(shim, scripts.join(python_exe)) { Ok(_) => {} Err(err) if err.kind() == io::ErrorKind::NotFound => { - // Second priority: the `venvlauncher.exe` and `venvwlauncher.exe` shims. - // These are equivalent to the `python.exe` and `pythonw.exe` shims, which were - // renamed in Python 3.13. let launcher = match python_exe { "python.exe" => "venvlauncher.exe", "pythonw.exe" => "venvwlauncher.exe", _ => unreachable!(), }; + + // If `python.exe` doesn't exist, try the `venvlauncher.exe` shim. let shim = interpreter .stdlib() .join("venv") @@ -204,81 +193,13 @@ pub fn create_bare_venv( .join("nt") .join(launcher); + // If the `venvlauncher.exe` shim doesn't exist, then on Conda at least, we + // can look for it next to the Python executable itself. match fs_err::copy(shim, scripts.join(python_exe)) { Ok(_) => {} Err(err) if err.kind() == io::ErrorKind::NotFound => { - // Third priority: on Conda at least, we can look for the launcher shim next to - // the Python executable itself. let shim = base_python.with_file_name(launcher); - match fs_err::copy(shim, scripts.join(python_exe)) { - Ok(_) => {} - Err(err) if err.kind() == io::ErrorKind::NotFound => { - // Fourth priority: if the launcher shim doesn't exist, assume this is - // an embedded Python. Copy the Python executable itself, along with - // the DLLs, `.pyd` files, and `.zip` files in the same directory. - fs_err::copy( - base_python.with_file_name(python_exe), - scripts.join(python_exe), - )?; - - // Copy `.dll` and `.pyd` files from the top-level, and from the - // `DLLs` subdirectory (if it exists). - for directory in [ - python_home, - interpreter.base_prefix().join("DLLs").as_path(), - ] { - let entries = match fs_err::read_dir(directory) { - Ok(read_dir) => read_dir, - Err(err) if err.kind() == io::ErrorKind::NotFound => { - continue; - } - Err(err) => { - return Err(err.into()); - } - }; - for entry in entries { - let entry = entry?; - let path = entry.path(); - if let Some(ext) = path.extension() { - if ext == "dll" || ext == "pyd" { - if let Some(file_name) = path.file_name() { - fs_err::copy( - &path, - scripts.join(file_name), - )?; - } - } - } - } - } - - // Copy `.zip` files from the top-level. - let entries = match fs_err::read_dir(python_home) { - Ok(read_dir) => read_dir, - Err(err) if err.kind() == io::ErrorKind::NotFound => { - continue; - } - Err(err) => { - return Err(err.into()); - } - }; - - for entry in entries { - let entry = entry?; - let path = entry.path(); - if let Some(ext) = path.extension() { - if ext == "zip" { - if let Some(file_name) = path.file_name() { - fs_err::copy(&path, scripts.join(file_name))?; - } - } - } - } - } - Err(err) => { - return Err(err.into()); - } - } + fs_err::copy(shim, scripts.join(python_exe))?; } Err(err) => { return Err(err.into()); @@ -321,6 +242,18 @@ pub fn create_bare_venv( fs::write(scripts.join(name), activator)?; } + // Per PEP 405, the Python `home` is the parent directory of the interpreter. + let python_home = base_python + .parent() + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::NotFound, + "The Python interpreter needs to have a parent directory", + ) + })? + .simplified_display() + .to_string(); + // Validate extra_cfg let reserved_keys = [ "home", @@ -339,10 +272,7 @@ pub fn create_bare_venv( } let mut pyvenv_cfg_data: Vec<(String, String)> = vec![ - ( - "home".to_string(), - python_home.simplified_display().to_string(), - ), + ("home".to_string(), python_home), ( "implementation".to_string(), interpreter.markers().platform_python_implementation.clone(), From 6368cc6fa8f28e64f5febda81305b3f3bdffba86 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Sat, 20 Apr 2024 17:46:14 -0400 Subject: [PATCH 3/4] Re-add --- crates/uv-virtualenv/src/bare.rs | 140 ++++++++++++++++++++++++++----- scripts/check_embedded_python.py | 75 +++++++++-------- 2 files changed, 162 insertions(+), 53 deletions(-) diff --git a/crates/uv-virtualenv/src/bare.rs b/crates/uv-virtualenv/src/bare.rs index a5090fdeca31..a05ae4493ce3 100644 --- a/crates/uv-virtualenv/src/bare.rs +++ b/crates/uv-virtualenv/src/bare.rs @@ -139,6 +139,14 @@ pub fn create_bare_venv( // Create a `.gitignore` file to ignore all files in the venv. fs::write(location.join(".gitignore"), "*")?; + // Per PEP 405, the Python `home` is the parent directory of the interpreter. + let python_home = base_python.parent().ok_or_else(|| { + io::Error::new( + io::ErrorKind::NotFound, + "The Python interpreter needs to have a parent directory", + ) + })?; + // Different names for the python interpreter fs::create_dir(&scripts)?; let executable = scripts.join(format!("python{EXE_SUFFIX}")); @@ -167,8 +175,10 @@ pub fn create_bare_venv( { // https://github.com/python/cpython/blob/d457345bbc6414db0443819290b04a9a4333313d/Lib/venv/__init__.py#L261-L267 // https://github.com/pypa/virtualenv/blob/d9fdf48d69f0d0ca56140cf0381edbb5d6fe09f5/src/virtualenv/create/via_global_ref/builtin/cpython/cpython3.py#L78-L83 - // There's two kinds of applications on windows: Those that allocate a console (python.exe) and those that - // don't because they use window(s) (pythonw.exe). + // There's two kinds of applications on windows: Those that allocate a console (python.exe) + // and those that don't because they use window(s) (pythonw.exe). + + // First priority: the `python.exe` and `pythonw.exe` shims. for python_exe in ["python.exe", "pythonw.exe"] { let shim = interpreter .stdlib() @@ -179,13 +189,14 @@ pub fn create_bare_venv( match fs_err::copy(shim, scripts.join(python_exe)) { Ok(_) => {} Err(err) if err.kind() == io::ErrorKind::NotFound => { + // Second priority: the `venvlauncher.exe` and `venvwlauncher.exe` shims. + // These are equivalent to the `python.exe` and `pythonw.exe` shims, which were + // renamed in Python 3.13. let launcher = match python_exe { "python.exe" => "venvlauncher.exe", "pythonw.exe" => "venvwlauncher.exe", _ => unreachable!(), }; - - // If `python.exe` doesn't exist, try the `venvlauncher.exe` shim. let shim = interpreter .stdlib() .join("venv") @@ -193,13 +204,111 @@ pub fn create_bare_venv( .join("nt") .join(launcher); - // If the `venvlauncher.exe` shim doesn't exist, then on Conda at least, we - // can look for it next to the Python executable itself. match fs_err::copy(shim, scripts.join(python_exe)) { Ok(_) => {} Err(err) if err.kind() == io::ErrorKind::NotFound => { + // Third priority: on Conda at least, we can look for the launcher shim next to + // the Python executable itself. let shim = base_python.with_file_name(launcher); - fs_err::copy(shim, scripts.join(python_exe))?; + match fs_err::copy(shim, scripts.join(python_exe)) { + Ok(_) => {} + Err(err) if err.kind() == io::ErrorKind::NotFound => { + // Fourth priority: if the launcher shim doesn't exist, assume this is + // an embedded Python. Copy the Python executable itself, along with + // the DLLs, `.pyd` files, and `.zip` files in the same directory. + match fs_err::copy( + base_python.with_file_name(python_exe), + scripts.join(python_exe), + ) { + Ok(_) => {} + Err(err) if err.kind() == io::ErrorKind::NotFound => { + return Err(Error::IO(io::Error::new( + io::ErrorKind::NotFound, + format!( + "Could not find a suitable Python executable for the virtual environment (tried: {}, {}, {}, {})", + interpreter + .stdlib() + .join("venv") + .join("scripts") + .join("nt") + .join(python_exe) + .user_display(), + interpreter + .stdlib() + .join("venv") + .join("scripts") + .join("nt") + .join(launcher) + .user_display(), + base_python.with_file_name(launcher).user_display(), + base_python + .with_file_name(python_exe) + .user_display(), + ) + ))); + } + Err(err) => { + return Err(err.into()); + } + } + + // Copy `.dll` and `.pyd` files from the top-level, and from the + // `DLLs` subdirectory (if it exists). + for directory in [ + python_home, + interpreter.base_prefix().join("DLLs").as_path(), + ] { + let entries = match fs_err::read_dir(directory) { + Ok(read_dir) => read_dir, + Err(err) if err.kind() == io::ErrorKind::NotFound => { + continue; + } + Err(err) => { + return Err(err.into()); + } + }; + for entry in entries { + let entry = entry?; + let path = entry.path(); + if path.extension().is_some_and(|ext| { + ext.eq_ignore_ascii_case("dll") + || ext.eq_ignore_ascii_case("pyd") + }) { + if let Some(file_name) = path.file_name() { + fs_err::copy(&path, scripts.join(file_name))?; + } + } + } + } + + // Copy `.zip` files from the top-level. + let entries = match fs_err::read_dir(python_home) { + Ok(read_dir) => read_dir, + Err(err) if err.kind() == io::ErrorKind::NotFound => { + continue; + } + Err(err) => { + return Err(err.into()); + } + }; + + for entry in entries { + let entry = entry?; + let path = entry.path(); + if path + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("zip")) + { + if let Some(file_name) = path.file_name() { + fs_err::copy(&path, scripts.join(file_name))?; + } + } + } + } + Err(err) => { + return Err(err.into()); + } + } } Err(err) => { return Err(err.into()); @@ -242,18 +351,6 @@ pub fn create_bare_venv( fs::write(scripts.join(name), activator)?; } - // Per PEP 405, the Python `home` is the parent directory of the interpreter. - let python_home = base_python - .parent() - .ok_or_else(|| { - io::Error::new( - io::ErrorKind::NotFound, - "The Python interpreter needs to have a parent directory", - ) - })? - .simplified_display() - .to_string(); - // Validate extra_cfg let reserved_keys = [ "home", @@ -272,7 +369,10 @@ pub fn create_bare_venv( } let mut pyvenv_cfg_data: Vec<(String, String)> = vec![ - ("home".to_string(), python_home), + ( + "home".to_string(), + python_home.simplified_display().to_string(), + ), ( "implementation".to_string(), interpreter.markers().platform_python_implementation.clone(), diff --git a/scripts/check_embedded_python.py b/scripts/check_embedded_python.py index 81b3f6c6c42d..ec9d9358f7de 100755 --- a/scripts/check_embedded_python.py +++ b/scripts/check_embedded_python.py @@ -13,7 +13,9 @@ if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") - parser = argparse.ArgumentParser(description="Check a Python interpreter.") + parser = argparse.ArgumentParser( + description="Check an embedded Python interpreter." + ) parser.add_argument("--uv", help="Path to a uv binary.") args = parser.parse_args() @@ -49,40 +51,47 @@ env = os.environ.copy() env["CONDA_PREFIX"] = "" env["VIRTUAL_ENV"] = "" - subprocess.run( - [uv, "pip", "install", "pylint", "--verbose"], - cwd=temp_dir, - check=True, - env=env, - ) - # Ensure that the package (`pylint`) is installed in the virtual environment. - logging.info("Checking that `pylint` is installed.") - code = subprocess.run( - [executable, "-c", "import pylint"], - cwd=temp_dir, - ) - if code.returncode != 0: - raise Exception( - "The package `pylint` isn't installed in the virtual environment." + # Install, verify, and uninstall a few packages. + for package in ["pylint", "numpy"]: + # Install the package. + logging.info( + f"Installing the package `{package}` into the virtual environment..." + ) + subprocess.run( + [uv, "pip", "install", package, "--verbose"], + cwd=temp_dir, + check=True, + env=env, ) - # Uninstall the package (`pylint`). - logging.info("Uninstalling the package `pylint`.") - subprocess.run( - [uv, "pip", "uninstall", "pylint", "--verbose"], - cwd=temp_dir, - check=True, - env=env, - ) + # Ensure that the package is installed in the virtual environment. + logging.info(f"Checking that `{package}` is installed.") + code = subprocess.run( + [executable, "-c", f"import {package}"], + cwd=temp_dir, + ) + if code.returncode != 0: + raise Exception( + f"The package `{package}` isn't installed in the virtual environment." + ) - # Ensure that the package (`pylint`) isn't installed in the virtual environment. - logging.info("Checking that `pylint` isn't installed.") - code = subprocess.run( - [executable, "-m", "pip", "show", "pylint"], - cwd=temp_dir, - ) - if code.returncode == 0: - raise Exception( - "The package `pylint` is installed in the virtual environment (but shouldn't be)." + # Uninstall the package. + logging.info(f"Uninstalling the package `{package}`.") + subprocess.run( + [uv, "pip", "uninstall", package, "--verbose"], + cwd=temp_dir, + check=True, + env=env, + ) + + # Ensure that the package isn't installed in the virtual environment. + logging.info(f"Checking that `{package}` isn't installed.") + code = subprocess.run( + [executable, "-m", "pip", "show", package], + cwd=temp_dir, ) + if code.returncode == 0: + raise Exception( + f"The package `{package}` is installed in the virtual environment (but shouldn't be)." + ) From ba692729d3aa6d5c56c98f3f7f91edbdc519a9da Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Mon, 22 Apr 2024 13:22:22 -0400 Subject: [PATCH 4/4] Remove nesting --- crates/uv-virtualenv/src/bare.rs | 315 ++++++++++++++++--------------- crates/uv-virtualenv/src/lib.rs | 4 +- 2 files changed, 169 insertions(+), 150 deletions(-) diff --git a/crates/uv-virtualenv/src/bare.rs b/crates/uv-virtualenv/src/bare.rs index a05ae4493ce3..a63b3e33dc63 100644 --- a/crates/uv-virtualenv/src/bare.rs +++ b/crates/uv-virtualenv/src/bare.rs @@ -171,156 +171,23 @@ pub fn create_bare_venv( } // No symlinking on Windows, at least not on a regular non-dev non-admin Windows install. - #[cfg(windows)] - { - // https://github.com/python/cpython/blob/d457345bbc6414db0443819290b04a9a4333313d/Lib/venv/__init__.py#L261-L267 - // https://github.com/pypa/virtualenv/blob/d9fdf48d69f0d0ca56140cf0381edbb5d6fe09f5/src/virtualenv/create/via_global_ref/builtin/cpython/cpython3.py#L78-L83 - // There's two kinds of applications on windows: Those that allocate a console (python.exe) - // and those that don't because they use window(s) (pythonw.exe). - - // First priority: the `python.exe` and `pythonw.exe` shims. - for python_exe in ["python.exe", "pythonw.exe"] { - let shim = interpreter - .stdlib() - .join("venv") - .join("scripts") - .join("nt") - .join(python_exe); - match fs_err::copy(shim, scripts.join(python_exe)) { - Ok(_) => {} - Err(err) if err.kind() == io::ErrorKind::NotFound => { - // Second priority: the `venvlauncher.exe` and `venvwlauncher.exe` shims. - // These are equivalent to the `python.exe` and `pythonw.exe` shims, which were - // renamed in Python 3.13. - let launcher = match python_exe { - "python.exe" => "venvlauncher.exe", - "pythonw.exe" => "venvwlauncher.exe", - _ => unreachable!(), - }; - let shim = interpreter - .stdlib() - .join("venv") - .join("scripts") - .join("nt") - .join(launcher); - - match fs_err::copy(shim, scripts.join(python_exe)) { - Ok(_) => {} - Err(err) if err.kind() == io::ErrorKind::NotFound => { - // Third priority: on Conda at least, we can look for the launcher shim next to - // the Python executable itself. - let shim = base_python.with_file_name(launcher); - match fs_err::copy(shim, scripts.join(python_exe)) { - Ok(_) => {} - Err(err) if err.kind() == io::ErrorKind::NotFound => { - // Fourth priority: if the launcher shim doesn't exist, assume this is - // an embedded Python. Copy the Python executable itself, along with - // the DLLs, `.pyd` files, and `.zip` files in the same directory. - match fs_err::copy( - base_python.with_file_name(python_exe), - scripts.join(python_exe), - ) { - Ok(_) => {} - Err(err) if err.kind() == io::ErrorKind::NotFound => { - return Err(Error::IO(io::Error::new( - io::ErrorKind::NotFound, - format!( - "Could not find a suitable Python executable for the virtual environment (tried: {}, {}, {}, {})", - interpreter - .stdlib() - .join("venv") - .join("scripts") - .join("nt") - .join(python_exe) - .user_display(), - interpreter - .stdlib() - .join("venv") - .join("scripts") - .join("nt") - .join(launcher) - .user_display(), - base_python.with_file_name(launcher).user_display(), - base_python - .with_file_name(python_exe) - .user_display(), - ) - ))); - } - Err(err) => { - return Err(err.into()); - } - } - - // Copy `.dll` and `.pyd` files from the top-level, and from the - // `DLLs` subdirectory (if it exists). - for directory in [ - python_home, - interpreter.base_prefix().join("DLLs").as_path(), - ] { - let entries = match fs_err::read_dir(directory) { - Ok(read_dir) => read_dir, - Err(err) if err.kind() == io::ErrorKind::NotFound => { - continue; - } - Err(err) => { - return Err(err.into()); - } - }; - for entry in entries { - let entry = entry?; - let path = entry.path(); - if path.extension().is_some_and(|ext| { - ext.eq_ignore_ascii_case("dll") - || ext.eq_ignore_ascii_case("pyd") - }) { - if let Some(file_name) = path.file_name() { - fs_err::copy(&path, scripts.join(file_name))?; - } - } - } - } - - // Copy `.zip` files from the top-level. - let entries = match fs_err::read_dir(python_home) { - Ok(read_dir) => read_dir, - Err(err) if err.kind() == io::ErrorKind::NotFound => { - continue; - } - Err(err) => { - return Err(err.into()); - } - }; - - for entry in entries { - let entry = entry?; - let path = entry.path(); - if path - .extension() - .is_some_and(|ext| ext.eq_ignore_ascii_case("zip")) - { - if let Some(file_name) = path.file_name() { - fs_err::copy(&path, scripts.join(file_name))?; - } - } - } - } - Err(err) => { - return Err(err.into()); - } - } - } - Err(err) => { - return Err(err.into()); - } - } - } - Err(err) => { - return Err(err.into()); - } - } - } + if cfg!(windows) { + copy_launcher_windows( + WindowsExecutable::Python, + interpreter, + &base_python, + &scripts, + python_home, + )?; + copy_launcher_windows( + WindowsExecutable::Pythonw, + interpreter, + &base_python, + &scripts, + python_home, + )?; } + #[cfg(not(any(unix, windows)))] { compile_error!("Only Windows and Unix are supported") @@ -422,3 +289,153 @@ pub fn create_bare_venv( executable, }) } + +#[derive(Debug, Copy, Clone)] +enum WindowsExecutable { + /// The `python.exe` executable (or `venvlauncher.exe` launcher shim). + Python, + /// The `pythonw.exe` executable (or `venvwlauncher.exe` launcher shim). + Pythonw, +} + +impl WindowsExecutable { + /// The name of the Python executable. + fn exe(self) -> &'static str { + match self { + WindowsExecutable::Python => "python.exe", + WindowsExecutable::Pythonw => "pythonw.exe", + } + } + + /// The name of the launcher shim. + fn launcher(self) -> &'static str { + match self { + WindowsExecutable::Python => "venvlauncher.exe", + WindowsExecutable::Pythonw => "venvwlauncher.exe", + } + } +} + +/// +/// +/// +/// There's two kinds of applications on windows: Those that allocate a console (python.exe) +/// and those that don't because they use window(s) (pythonw.exe). +fn copy_launcher_windows( + executable: WindowsExecutable, + interpreter: &Interpreter, + base_python: &Path, + scripts: &Path, + python_home: &Path, +) -> Result<(), Error> { + // First priority: the `python.exe` and `pythonw.exe` shims. + let shim = interpreter + .stdlib() + .join("venv") + .join("scripts") + .join("nt") + .join(executable.exe()); + match fs_err::copy(shim, scripts.join(executable.exe())) { + Ok(_) => return Ok(()), + Err(err) if err.kind() == io::ErrorKind::NotFound => {} + Err(err) => { + return Err(err.into()); + } + } + + // Second priority: the `venvlauncher.exe` and `venvwlauncher.exe` shims. + // These are equivalent to the `python.exe` and `pythonw.exe` shims, which were + // renamed in Python 3.13. + let shim = interpreter + .stdlib() + .join("venv") + .join("scripts") + .join("nt") + .join(executable.launcher()); + match fs_err::copy(shim, scripts.join(executable.exe())) { + Ok(_) => return Ok(()), + Err(err) if err.kind() == io::ErrorKind::NotFound => {} + Err(err) => { + return Err(err.into()); + } + } + + // Third priority: on Conda at least, we can look for the launcher shim next to + // the Python executable itself. + let shim = base_python.with_file_name(executable.launcher()); + match fs_err::copy(shim, scripts.join(executable.exe())) { + Ok(_) => return Ok(()), + Err(err) if err.kind() == io::ErrorKind::NotFound => {} + Err(err) => { + return Err(err.into()); + } + } + + // Fourth priority: if the launcher shim doesn't exist, assume this is + // an embedded Python. Copy the Python executable itself, along with + // the DLLs, `.pyd` files, and `.zip` files in the same directory. + match fs_err::copy( + base_python.with_file_name(executable.exe()), + scripts.join(executable.exe()), + ) { + Ok(_) => { + // Copy `.dll` and `.pyd` files from the top-level, and from the + // `DLLs` subdirectory (if it exists). + for directory in [ + python_home, + interpreter.base_prefix().join("DLLs").as_path(), + ] { + let entries = match fs_err::read_dir(directory) { + Ok(read_dir) => read_dir, + Err(err) if err.kind() == io::ErrorKind::NotFound => { + continue; + } + Err(err) => { + return Err(err.into()); + } + }; + for entry in entries { + let entry = entry?; + let path = entry.path(); + if path.extension().is_some_and(|ext| { + ext.eq_ignore_ascii_case("dll") || ext.eq_ignore_ascii_case("pyd") + }) { + if let Some(file_name) = path.file_name() { + fs_err::copy(&path, scripts.join(file_name))?; + } + } + } + } + + // Copy `.zip` files from the top-level. + match fs_err::read_dir(python_home) { + Ok(entries) => { + for entry in entries { + let entry = entry?; + let path = entry.path(); + if path + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("zip")) + { + if let Some(file_name) = path.file_name() { + fs_err::copy(&path, scripts.join(file_name))?; + } + } + } + } + Err(err) if err.kind() == io::ErrorKind::NotFound => {} + Err(err) => { + return Err(err.into()); + } + }; + + return Ok(()); + } + Err(err) if err.kind() == io::ErrorKind::NotFound => {} + Err(err) => { + return Err(err.into()); + } + } + + Err(Error::NotFound(base_python.user_display().to_string())) +} diff --git a/crates/uv-virtualenv/src/lib.rs b/crates/uv-virtualenv/src/lib.rs index a4f996d76371..ce9e730b86b7 100644 --- a/crates/uv-virtualenv/src/lib.rs +++ b/crates/uv-virtualenv/src/lib.rs @@ -1,9 +1,9 @@ use std::io; use std::path::Path; -use platform_tags::PlatformError; use thiserror::Error; +use platform_tags::PlatformError; use uv_interpreter::{Interpreter, PythonEnvironment}; pub use crate::bare::create_bare_venv; @@ -20,6 +20,8 @@ pub enum Error { Platform(#[from] PlatformError), #[error("Reserved key used for pyvenv.cfg: {0}")] ReservedConfigKey(String), + #[error("Could not find a suitable Python executable for the virtual environment based on the interpreter: {0}")] + NotFound(String), } /// The value to use for the shell prompt when inside a virtual environment.