Skip to content

Commit

Permalink
Add --python-platform to sync and install commands (#3154)
Browse files Browse the repository at this point in the history
## Summary

pip supports providing a `--platform` to `pip install`, which can be
used to seed an environment (e.g., for use in a container or otherwise).
This PR adds `--python-platform` to our commands to support a similar
workflow. It has some caveats, which are documented on the CLI.

Closes #2079.
  • Loading branch information
charliermarsh authored Apr 22, 2024
1 parent d10903f commit 8536e63
Show file tree
Hide file tree
Showing 5 changed files with 148 additions and 19 deletions.
42 changes: 42 additions & 0 deletions crates/uv/src/cli.rs
Original file line number Diff line number Diff line change
Expand Up @@ -853,6 +853,27 @@ pub(crate) struct PipSyncArgs {
#[arg(long, short = 'C', alias = "config-settings")]
pub(crate) config_setting: Option<Vec<ConfigSettingEntry>>,

/// The minimum Python version that should be supported by the requirements (e.g.,
/// `3.7` or `3.7.9`).
///
/// If a patch version is omitted, the most recent known patch version for that minor version
/// is assumed. For example, `3.7` is mapped to `3.7.17`.
#[arg(long)]
pub(crate) python_version: Option<PythonVersion>,

/// The platform for which requirements should be installed.
///
/// Represented as a "target triple", a string that describes the target platform in terms of
/// its CPU, vendor, and operating system name, like `x86_64-unknown-linux-gnu` or
/// `aaarch64-apple-darwin`.
///
/// WARNING: When specified, uv will select wheels that are compatible with the target platform.
/// The resulting environment may not be fully compatible with the current platform. Further,
/// distributions that are built from source may ultimately be incompatible with the target
/// platform. This option is intended for cross-compilation and other advanced use cases.
#[arg(long)]
pub(crate) python_platform: Option<TargetTriple>,

/// Validate the virtual environment after completing the installation, to detect packages with
/// missing dependencies or other issues.
#[arg(long, overrides_with("no_strict"))]
Expand Down Expand Up @@ -1195,6 +1216,27 @@ pub(crate) struct PipInstallArgs {
#[arg(long, short = 'C', alias = "config-settings")]
pub(crate) config_setting: Option<Vec<ConfigSettingEntry>>,

/// The minimum Python version that should be supported by the requirements (e.g.,
/// `3.7` or `3.7.9`).
///
/// If a patch version is omitted, the most recent known patch version for that minor version
/// is assumed. For example, `3.7` is mapped to `3.7.17`.
#[arg(long)]
pub(crate) python_version: Option<PythonVersion>,

/// The platform for which requirements should be installed.
///
/// Represented as a "target triple", a string that describes the target platform in terms of
/// its CPU, vendor, and operating system name, like `x86_64-unknown-linux-gnu` or
/// `aaarch64-apple-darwin`.
///
/// WARNING: When specified, uv will select wheels that are compatible with the target platform.
/// The resulting environment may not be fully compatible with the current platform. Further,
/// distributions that are built from source may ultimately be incompatible with the target
/// platform. This option is intended for cross-compilation and other advanced use cases.
#[arg(long)]
pub(crate) python_platform: Option<TargetTriple>,

/// Validate the virtual environment after completing the installation, to detect packages with
/// missing dependencies or other issues.
#[arg(long, overrides_with("no_strict"))]
Expand Down
59 changes: 48 additions & 11 deletions crates/uv/src/commands/pip_install.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
use std::borrow::Cow;
use std::fmt::Write;

use std::path::Path;
Expand All @@ -24,11 +25,11 @@ use uv_cache::Cache;
use uv_client::{
BaseClientBuilder, Connectivity, FlatIndexClient, RegistryClient, RegistryClientBuilder,
};
use uv_configuration::KeyringProviderType;
use uv_configuration::{
ConfigSettings, Constraints, IndexStrategy, NoBinary, NoBuild, Overrides, Reinstall,
SetupPyStrategy, Upgrade,
};
use uv_configuration::{KeyringProviderType, TargetTriple};
use uv_dispatch::BuildDispatch;
use uv_fs::Simplified;
use uv_installer::{BuiltEditable, Downloader, Plan, Planner, ResolvedEditable, SitePackages};
Expand All @@ -42,6 +43,7 @@ use uv_resolver::{
DependencyMode, ExcludeNewer, Exclusions, FlatIndex, InMemoryIndex, Manifest, Options,
OptionsBuilder, PreReleaseMode, Preference, ResolutionGraph, ResolutionMode, Resolver,
};
use uv_toolchain::PythonVersion;
use uv_types::{BuildIsolation, HashStrategy, InFlight};
use uv_warnings::warn_user;

Expand Down Expand Up @@ -75,6 +77,8 @@ pub(crate) async fn pip_install(
no_build_isolation: bool,
no_build: NoBuild,
no_binary: NoBinary,
python_version: Option<PythonVersion>,
python_platform: Option<TargetTriple>,
strict: bool,
exclude_newer: Option<ExcludeNewer>,
python: Option<String>,
Expand Down Expand Up @@ -182,10 +186,43 @@ pub(crate) async fn pip_install(
return Ok(ExitStatus::Success);
}

// Determine the tags, markers, and interpreter to use for resolution.
let interpreter = venv.interpreter().clone();
let tags = venv.interpreter().tags()?;
let markers = venv.interpreter().markers();

// Determine the tags, markers, and interpreter to use for resolution.
let tags = match (python_platform, python_version.as_ref()) {
(Some(python_platform), Some(python_version)) => Cow::Owned(Tags::from_env(
&python_platform.platform(),
(python_version.major(), python_version.minor()),
interpreter.implementation_name(),
interpreter.implementation_tuple(),
interpreter.gil_disabled(),
)?),
(Some(python_platform), None) => Cow::Owned(Tags::from_env(
&python_platform.platform(),
interpreter.python_tuple(),
interpreter.implementation_name(),
interpreter.implementation_tuple(),
interpreter.gil_disabled(),
)?),
(None, Some(python_version)) => Cow::Owned(Tags::from_env(
interpreter.platform(),
(python_version.major(), python_version.minor()),
interpreter.implementation_name(),
interpreter.implementation_tuple(),
interpreter.gil_disabled(),
)?),
(None, None) => Cow::Borrowed(interpreter.tags()?),
};

// Apply the platform tags to the markers.
let markers = match (python_platform, python_version) {
(Some(python_platform), Some(python_version)) => {
Cow::Owned(python_version.markers(&python_platform.markers(interpreter.markers())))
}
(Some(python_platform), None) => Cow::Owned(python_platform.markers(interpreter.markers())),
(None, Some(python_version)) => Cow::Owned(python_version.markers(interpreter.markers())),
(None, None) => Cow::Borrowed(interpreter.markers()),
};

// Collect the set of required hashes.
let hasher = if require_hashes {
Expand All @@ -194,7 +231,7 @@ pub(crate) async fn pip_install(
.iter()
.chain(overrides.iter())
.map(|entry| (&entry.requirement, entry.hashes.as_slice())),
markers,
&markers,
)?
} else {
HashStrategy::None
Expand All @@ -216,15 +253,15 @@ pub(crate) async fn pip_install(
.index_urls(index_locations.index_urls())
.index_strategy(index_strategy)
.keyring(keyring_provider)
.markers(markers)
.markers(&markers)
.platform(interpreter.platform())
.build();

// Resolve the flat indexes from `--find-links`.
let flat_index = {
let client = FlatIndexClient::new(&client, &cache);
let entries = client.fetch(index_locations.flat_index()).await?;
FlatIndex::from_entries(entries, tags, &hasher, &no_build, &no_binary)
FlatIndex::from_entries(entries, &tags, &hasher, &no_build, &no_binary)
};

// Determine whether to enable build isolation.
Expand Down Expand Up @@ -317,7 +354,7 @@ pub(crate) async fn pip_install(
&hasher,
&cache,
&interpreter,
tags,
&tags,
&client,
&resolve_dispatch,
printer,
Expand All @@ -344,8 +381,8 @@ pub(crate) async fn pip_install(
&reinstall,
&upgrade,
&interpreter,
tags,
markers,
&tags,
&markers,
&client,
&flat_index,
&index,
Expand Down Expand Up @@ -402,7 +439,7 @@ pub(crate) async fn pip_install(
compile,
&index_locations,
&hasher,
tags,
&tags,
&client,
&in_flight,
&install_dispatch,
Expand Down
54 changes: 46 additions & 8 deletions crates/uv/src/commands/pip_sync.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
use std::borrow::Cow;
use std::fmt::Write;

use anstream::eprint;
Expand All @@ -19,10 +20,10 @@ use uv_cache::{ArchiveTarget, ArchiveTimestamp, Cache};
use uv_client::{
BaseClientBuilder, Connectivity, FlatIndexClient, RegistryClient, RegistryClientBuilder,
};
use uv_configuration::KeyringProviderType;
use uv_configuration::{
ConfigSettings, IndexStrategy, NoBinary, NoBuild, Reinstall, SetupPyStrategy,
};
use uv_configuration::{KeyringProviderType, TargetTriple};
use uv_dispatch::BuildDispatch;
use uv_fs::Simplified;
use uv_installer::{is_dynamic, Downloader, Plan, Planner, ResolvedEditable, SitePackages};
Expand All @@ -32,6 +33,7 @@ use uv_requirements::{
SourceTreeResolver,
};
use uv_resolver::{DependencyMode, FlatIndex, InMemoryIndex, Manifest, OptionsBuilder, Resolver};
use uv_toolchain::PythonVersion;
use uv_types::{BuildIsolation, EmptyInstalledPackages, HashStrategy, InFlight};
use uv_warnings::warn_user;

Expand All @@ -56,6 +58,8 @@ pub(crate) async fn pip_sync(
no_build_isolation: bool,
no_build: NoBuild,
no_binary: NoBinary,
python_version: Option<PythonVersion>,
python_platform: Option<TargetTriple>,
strict: bool,
python: Option<String>,
system: bool,
Expand Down Expand Up @@ -131,17 +135,51 @@ pub(crate) async fn pip_sync(

let _lock = venv.lock()?;

let interpreter = venv.interpreter();

// Determine the current environment markers.
let tags = venv.interpreter().tags()?;
let markers = venv.interpreter().markers();
let tags = match (python_platform, python_version.as_ref()) {
(Some(python_platform), Some(python_version)) => Cow::Owned(Tags::from_env(
&python_platform.platform(),
(python_version.major(), python_version.minor()),
interpreter.implementation_name(),
interpreter.implementation_tuple(),
interpreter.gil_disabled(),
)?),
(Some(python_platform), None) => Cow::Owned(Tags::from_env(
&python_platform.platform(),
interpreter.python_tuple(),
interpreter.implementation_name(),
interpreter.implementation_tuple(),
interpreter.gil_disabled(),
)?),
(None, Some(python_version)) => Cow::Owned(Tags::from_env(
interpreter.platform(),
(python_version.major(), python_version.minor()),
interpreter.implementation_name(),
interpreter.implementation_tuple(),
interpreter.gil_disabled(),
)?),
(None, None) => Cow::Borrowed(interpreter.tags()?),
};

// Apply the platform tags to the markers.
let markers = match (python_platform, python_version) {
(Some(python_platform), Some(python_version)) => {
Cow::Owned(python_version.markers(&python_platform.markers(interpreter.markers())))
}
(Some(python_platform), None) => Cow::Owned(python_platform.markers(interpreter.markers())),
(None, Some(python_version)) => Cow::Owned(python_version.markers(interpreter.markers())),
(None, None) => Cow::Borrowed(interpreter.markers()),
};

// Collect the set of required hashes.
let hasher = if require_hashes {
HashStrategy::from_requirements(
requirements
.iter()
.map(|entry| (&entry.requirement, entry.hashes.as_slice())),
markers,
&markers,
)?
} else {
HashStrategy::None
Expand Down Expand Up @@ -171,7 +209,7 @@ pub(crate) async fn pip_sync(
let flat_index = {
let client = FlatIndexClient::new(&client, &cache);
let entries = client.fetch(index_locations.flat_index()).await?;
FlatIndex::from_entries(entries, tags, &hasher, &no_build, &no_binary)
FlatIndex::from_entries(entries, &tags, &hasher, &no_build, &no_binary)
};

// Create a shared in-memory index.
Expand Down Expand Up @@ -247,7 +285,7 @@ pub(crate) async fn pip_sync(
reinstall,
&hasher,
venv.interpreter(),
tags,
&tags,
&cache,
&client,
&build_dispatch,
Expand All @@ -273,7 +311,7 @@ pub(crate) async fn pip_sync(
&index_locations,
&cache,
&venv,
tags,
&tags,
)
.context("Failed to determine installation plan")?;

Expand Down Expand Up @@ -367,7 +405,7 @@ pub(crate) async fn pip_sync(
} else {
let start = std::time::Instant::now();

let downloader = Downloader::new(&cache, tags, &hasher, &client, &build_dispatch)
let downloader = Downloader::new(&cache, &tags, &hasher, &client, &build_dispatch)
.with_reporter(DownloadReporter::from(printer).with_length(remote.len() as u64));

let wheels = downloader
Expand Down
4 changes: 4 additions & 0 deletions crates/uv/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,8 @@ async fn run() -> Result<ExitStatus> {
args.shared.no_build_isolation,
args.shared.no_build,
args.shared.no_binary,
args.shared.python_version,
args.shared.python_platform,
args.shared.strict,
args.shared.python,
args.shared.system,
Expand Down Expand Up @@ -325,6 +327,8 @@ async fn run() -> Result<ExitStatus> {
args.shared.no_build_isolation,
args.shared.no_build,
args.shared.no_binary,
args.shared.python_version,
args.shared.python_platform,
args.shared.strict,
args.shared.exclude_newer,
args.shared.python,
Expand Down
8 changes: 8 additions & 0 deletions crates/uv/src/settings.rs
Original file line number Diff line number Diff line change
Expand Up @@ -310,6 +310,8 @@ impl PipSyncSettings {
compile_bytecode,
no_compile_bytecode,
config_setting,
python_version,
python_platform,
strict,
no_strict,
compat_args: _,
Expand Down Expand Up @@ -348,6 +350,8 @@ impl PipSyncSettings {
config_settings: config_setting.map(|config_settings| {
config_settings.into_iter().collect::<ConfigSettings>()
}),
python_version,
python_platform,
link_mode,
compile_bytecode: flag(compile_bytecode, no_compile_bytecode),
require_hashes: flag(require_hashes, no_require_hashes),
Expand Down Expand Up @@ -427,6 +431,8 @@ impl PipInstallSettings {
compile_bytecode,
no_compile_bytecode,
config_setting,
python_version,
python_platform,
strict,
no_strict,
exclude_newer,
Expand Down Expand Up @@ -484,6 +490,8 @@ impl PipInstallSettings {
config_settings: config_setting.map(|config_settings| {
config_settings.into_iter().collect::<ConfigSettings>()
}),
python_version,
python_platform,
exclude_newer,
link_mode,
compile_bytecode: flag(compile_bytecode, no_compile_bytecode),
Expand Down

0 comments on commit 8536e63

Please sign in to comment.