Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for embedded Python on Windows #3161

Merged
merged 4 commits into from
Apr 22, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -814,3 +814,37 @@ jobs:

- name: "Validate global Python install"
run: python3 scripts/check_system_python.py --uv ./uv

system-test-windows-embedded-python-310:
needs: build-binary-windows
name: "check system | embedded python3.10 on windows"
runs-on: windows-latest
env:
# Avoid debug build stack overflows.
UV_STACK_SIZE: 2000000 # 2 megabyte, double the default on windows
steps:
- uses: actions/checkout@v4

- name: "Download binary"
uses: actions/download-artifact@v4
with:
name: uv-windows-${{ github.sha }}

# Download embedded Python.
- name: "Download embedded Python"
run: curl -LsSf https://www.python.org/ftp/python/3.11.8/python-3.11.8-embed-amd64.zip -o python-3.11.8-embed-amd64.zip

- name: "Unzip embedded Python"
run: 7z x python-3.11.8-embed-amd64.zip -oembedded-python

- name: "Show embedded Python contents"
run: ls embedded-python

- name: "Set PATH"
run: echo "${{ github.workspace }}\embedded-python" >> $env:GITHUB_PATH

- name: "Print Python path"
run: echo $(which python)

- name: "Validate embedded Python install"
run: python ./scripts/check_embedded_python.py --uv ./uv.exe
239 changes: 178 additions & 61 deletions crates/uv-virtualenv/src/bare.rs
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,14 @@ pub fn create_bare_venv(
// Create a `.gitignore` file to ignore all files in the venv.
fs::write(location.join(".gitignore"), "*")?;

// Per PEP 405, the Python `home` is the parent directory of the interpreter.
let python_home = base_python.parent().ok_or_else(|| {
io::Error::new(
io::ErrorKind::NotFound,
"The Python interpreter needs to have a parent directory",
)
})?;
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(Just moved this up because I need it for symlinking.)


// Different names for the python interpreter
fs::create_dir(&scripts)?;
let executable = scripts.join(format!("python{EXE_SUFFIX}"));
Expand All @@ -163,55 +171,23 @@ pub fn create_bare_venv(
}

// No symlinking on Windows, at least not on a regular non-dev non-admin Windows install.
#[cfg(windows)]
{
// https://github.com/python/cpython/blob/d457345bbc6414db0443819290b04a9a4333313d/Lib/venv/__init__.py#L261-L267
// https://github.com/pypa/virtualenv/blob/d9fdf48d69f0d0ca56140cf0381edbb5d6fe09f5/src/virtualenv/create/via_global_ref/builtin/cpython/cpython3.py#L78-L83
// There's two kinds of applications on windows: Those that allocate a console (python.exe) and those that
// don't because they use window(s) (pythonw.exe).
for python_exe in ["python.exe", "pythonw.exe"] {
let shim = interpreter
.stdlib()
.join("venv")
.join("scripts")
.join("nt")
.join(python_exe);
match fs_err::copy(shim, scripts.join(python_exe)) {
Ok(_) => {}
Err(err) if err.kind() == io::ErrorKind::NotFound => {
let launcher = match python_exe {
"python.exe" => "venvlauncher.exe",
"pythonw.exe" => "venvwlauncher.exe",
_ => unreachable!(),
};

// If `python.exe` doesn't exist, try the `venvlauncher.exe` shim.
let shim = interpreter
.stdlib()
.join("venv")
.join("scripts")
.join("nt")
.join(launcher);

// If the `venvlauncher.exe` shim doesn't exist, then on Conda at least, we
// can look for it next to the Python executable itself.
match fs_err::copy(shim, scripts.join(python_exe)) {
Ok(_) => {}
Err(err) if err.kind() == io::ErrorKind::NotFound => {
let shim = base_python.with_file_name(launcher);
fs_err::copy(shim, scripts.join(python_exe))?;
}
Err(err) => {
return Err(err.into());
}
}
}
Err(err) => {
return Err(err.into());
}
}
}
if cfg!(windows) {
copy_launcher_windows(
WindowsExecutable::Python,
interpreter,
&base_python,
&scripts,
python_home,
)?;
copy_launcher_windows(
WindowsExecutable::Pythonw,
interpreter,
&base_python,
&scripts,
python_home,
)?;
}

#[cfg(not(any(unix, windows)))]
{
compile_error!("Only Windows and Unix are supported")
Expand Down Expand Up @@ -242,18 +218,6 @@ pub fn create_bare_venv(
fs::write(scripts.join(name), activator)?;
}

// Per PEP 405, the Python `home` is the parent directory of the interpreter.
let python_home = base_python
.parent()
.ok_or_else(|| {
io::Error::new(
io::ErrorKind::NotFound,
"The Python interpreter needs to have a parent directory",
)
})?
.simplified_display()
.to_string();

// Validate extra_cfg
let reserved_keys = [
"home",
Expand All @@ -272,7 +236,10 @@ pub fn create_bare_venv(
}

let mut pyvenv_cfg_data: Vec<(String, String)> = vec![
("home".to_string(), python_home),
(
"home".to_string(),
python_home.simplified_display().to_string(),
),
(
"implementation".to_string(),
interpreter.markers().platform_python_implementation.clone(),
Expand Down Expand Up @@ -322,3 +289,153 @@ pub fn create_bare_venv(
executable,
})
}

#[derive(Debug, Copy, Clone)]
enum WindowsExecutable {
/// The `python.exe` executable (or `venvlauncher.exe` launcher shim).
Python,
/// The `pythonw.exe` executable (or `venvwlauncher.exe` launcher shim).
Pythonw,
}

impl WindowsExecutable {
/// The name of the Python executable.
fn exe(self) -> &'static str {
match self {
WindowsExecutable::Python => "python.exe",
WindowsExecutable::Pythonw => "pythonw.exe",
}
}

/// The name of the launcher shim.
fn launcher(self) -> &'static str {
match self {
WindowsExecutable::Python => "venvlauncher.exe",
WindowsExecutable::Pythonw => "venvwlauncher.exe",
}
}
}

/// <https://github.com/python/cpython/blob/d457345bbc6414db0443819290b04a9a4333313d/Lib/venv/__init__.py#L261-L267>
/// <https://github.com/pypa/virtualenv/blob/d9fdf48d69f0d0ca56140cf0381edbb5d6fe09f5/src/virtualenv/create/via_global_ref/builtin/cpython/cpython3.py#L78-L83>
///
/// There's two kinds of applications on windows: Those that allocate a console (python.exe)
/// and those that don't because they use window(s) (pythonw.exe).
fn copy_launcher_windows(
executable: WindowsExecutable,
interpreter: &Interpreter,
base_python: &Path,
scripts: &Path,
python_home: &Path,
) -> Result<(), Error> {
// First priority: the `python.exe` and `pythonw.exe` shims.
let shim = interpreter
.stdlib()
.join("venv")
.join("scripts")
.join("nt")
.join(executable.exe());
match fs_err::copy(shim, scripts.join(executable.exe())) {
Ok(_) => return Ok(()),
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
Err(err) => {
return Err(err.into());
}
}

// Second priority: the `venvlauncher.exe` and `venvwlauncher.exe` shims.
// These are equivalent to the `python.exe` and `pythonw.exe` shims, which were
// renamed in Python 3.13.
let shim = interpreter
.stdlib()
.join("venv")
.join("scripts")
.join("nt")
.join(executable.launcher());
match fs_err::copy(shim, scripts.join(executable.exe())) {
Ok(_) => return Ok(()),
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
Err(err) => {
return Err(err.into());
}
}

// Third priority: on Conda at least, we can look for the launcher shim next to
// the Python executable itself.
let shim = base_python.with_file_name(executable.launcher());
match fs_err::copy(shim, scripts.join(executable.exe())) {
Ok(_) => return Ok(()),
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
Err(err) => {
return Err(err.into());
}
}

// Fourth priority: if the launcher shim doesn't exist, assume this is
// an embedded Python. Copy the Python executable itself, along with
// the DLLs, `.pyd` files, and `.zip` files in the same directory.
match fs_err::copy(
base_python.with_file_name(executable.exe()),
scripts.join(executable.exe()),
) {
Ok(_) => {
// Copy `.dll` and `.pyd` files from the top-level, and from the
// `DLLs` subdirectory (if it exists).
for directory in [
python_home,
interpreter.base_prefix().join("DLLs").as_path(),
] {
let entries = match fs_err::read_dir(directory) {
Ok(read_dir) => read_dir,
Err(err) if err.kind() == io::ErrorKind::NotFound => {
continue;
}
Err(err) => {
return Err(err.into());
}
};
for entry in entries {
let entry = entry?;
let path = entry.path();
if path.extension().is_some_and(|ext| {
ext.eq_ignore_ascii_case("dll") || ext.eq_ignore_ascii_case("pyd")
}) {
if let Some(file_name) = path.file_name() {
fs_err::copy(&path, scripts.join(file_name))?;
}
}
}
}

// Copy `.zip` files from the top-level.
match fs_err::read_dir(python_home) {
Ok(entries) => {
for entry in entries {
let entry = entry?;
let path = entry.path();
if path
.extension()
.is_some_and(|ext| ext.eq_ignore_ascii_case("zip"))
{
if let Some(file_name) = path.file_name() {
fs_err::copy(&path, scripts.join(file_name))?;
}
}
}
}
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
Err(err) => {
return Err(err.into());
}
};

return Ok(());
}
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
Err(err) => {
return Err(err.into());
}
}

Err(Error::NotFound(base_python.user_display().to_string()))
}
4 changes: 3 additions & 1 deletion crates/uv-virtualenv/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
use std::io;
use std::path::Path;

use platform_tags::PlatformError;
use thiserror::Error;

use platform_tags::PlatformError;
use uv_interpreter::{Interpreter, PythonEnvironment};

pub use crate::bare::create_bare_venv;
Expand All @@ -20,6 +20,8 @@ pub enum Error {
Platform(#[from] PlatformError),
#[error("Reserved key used for pyvenv.cfg: {0}")]
ReservedConfigKey(String),
#[error("Could not find a suitable Python executable for the virtual environment based on the interpreter: {0}")]
NotFound(String),
}

/// The value to use for the shell prompt when inside a virtual environment.
Expand Down
Loading
Loading