Skip to content

Commit

Permalink
Enable environment variable authentication for named indexes
Browse files Browse the repository at this point in the history
  • Loading branch information
charliermarsh committed Sep 30, 2024
1 parent 256a17b commit f1d41c4
Show file tree
Hide file tree
Showing 20 changed files with 289 additions and 66 deletions.
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions crates/distribution-types/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ pep440_rs = { workspace = true }
pep508_rs = { workspace = true, features = ["serde"] }
platform-tags = { workspace = true }
pypi-types = { workspace = true }
uv-auth = { workspace = true }
uv-cache-info = { workspace = true }
uv-fs = { workspace = true }
uv-git = { workspace = true }
Expand Down
14 changes: 14 additions & 0 deletions crates/distribution-types/src/index.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ use crate::{IndexUrl, IndexUrlError};
use std::str::FromStr;
use thiserror::Error;
use url::Url;
use uv_auth::Credentials;

#[derive(Debug, Clone, Hash, Eq, PartialEq, serde::Serialize, serde::Deserialize)]
#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
Expand Down Expand Up @@ -102,6 +103,19 @@ impl Index {
pub fn raw_url(&self) -> &Url {
self.url.url()
}

/// Retrieve the credentials for the index, either from the environment, or from the URL itself.
pub fn credentials(&self) -> Option<Credentials> {
// If the index is named, and credentials are provided via the environment, prefer those.
if let Some(name) = self.name.as_deref() {
if let Some(credentials) = Credentials::from_env(name) {
return Some(credentials);
}
}

// Otherwise, extract the credentials from the URL.
Credentials::from_url(self.url.url())
}
}

impl FromStr for Index {
Expand Down
19 changes: 4 additions & 15 deletions crates/distribution-types/src/index_url.rs
Original file line number Diff line number Diff line change
Expand Up @@ -407,7 +407,7 @@ impl<'a> IndexLocations {
}

/// Return an iterator over the [`FlatIndexLocation`] entries.
pub fn flat_index(&'a self) -> impl Iterator<Item = &'a FlatIndexLocation> + 'a {
pub fn flat_indexes(&'a self) -> impl Iterator<Item = &'a FlatIndexLocation> + 'a {
self.flat_index.iter()
}

Expand All @@ -424,9 +424,10 @@ impl<'a> IndexLocations {
}
}

/// Return an iterator over all allowed [`IndexUrl`] entries.
/// Return an iterator over all allowed [`Index`] entries.
///
/// This includes both explicit and implicit indexes, as well as the default index.
/// This includes both explicit and implicit indexes, as well as the default index (but _not_
/// the flat indexes).
///
/// If `no_index` was enabled, then this always returns an empty
/// iterator.
Expand All @@ -435,18 +436,6 @@ impl<'a> IndexLocations {
.chain(self.implicit_indexes())
.chain(self.default_index())
}

/// Return an iterator over all allowed [`Url`] entries.
///
/// This includes both explicit and implicit index URLs, as well as the default index.
///
/// If `no_index` was enabled, then this always returns an empty
/// iterator.
pub fn allowed_urls(&'a self) -> impl Iterator<Item = &'a Url> + 'a {
self.allowed_indexes()
.map(Index::raw_url)
.chain(self.flat_index().map(FlatIndexLocation::url))
}
}

/// The index URLs to use for fetching packages.
Expand Down
15 changes: 15 additions & 0 deletions crates/uv-auth/src/credentials.rs
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,21 @@ impl Credentials {
})
}

/// Extract the [`Credentials`] from the environment, given a named source.
///
/// For example, given a name of `"pytorch"`, search for `UV_HTTP_BASIC_PYTORCH_USERNAME` and
/// `UV_HTTP_BASIC_PYTORCH_PASSWORD`.
pub fn from_env(name: &str) -> Option<Self> {
let name = name.to_uppercase();
let username = std::env::var(format!("UV_HTTP_BASIC_{name}_USERNAME")).ok();
let password = std::env::var(format!("UV_HTTP_BASIC_{name}_PASSWORD")).ok();
if username.is_none() && password.is_none() {
None
} else {
Some(Self::new(username, password))
}
}

/// Parse [`Credentials`] from an HTTP request, if any.
///
/// Only HTTP Basic Authentication is supported.
Expand Down
8 changes: 8 additions & 0 deletions crates/uv-auth/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,3 +35,11 @@ pub fn store_credentials_from_url(url: &Url) -> bool {
false
}
}

/// Populate the global authentication store with credentials on a URL, if there are any.
///
/// Returns `true` if the store was updated.
pub fn store_credentials(url: &Url, credentials: Credentials) {
trace!("Caching credentials for {url}");
CREDENTIALS_CACHE.insert(url, Arc::new(credentials));
}
2 changes: 1 addition & 1 deletion crates/uv-distribution/src/index/registry_wheel_index.rs
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ impl<'a> RegistryWheelIndex<'a> {
let mut entries = vec![];

let flat_index_urls: Vec<Index> = index_locations
.flat_index()
.flat_indexes()
.map(|flat_index| Index::from_extra_index_url(IndexUrl::from(flat_index.clone())))
.collect();

Expand Down
4 changes: 2 additions & 2 deletions crates/uv-resolver/src/lock/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1055,7 +1055,7 @@ impl Lock {
})
.chain(
locations
.flat_index()
.flat_indexes()
.filter_map(|index_url| match index_url {
FlatIndexLocation::Url(_) => {
Some(UrlString::from(index_url.redacted()))
Expand All @@ -1081,7 +1081,7 @@ impl Lock {
})
.chain(
locations
.flat_index()
.flat_indexes()
.filter_map(|index_url| match index_url {
FlatIndexLocation::Url(_) => None,
FlatIndexLocation::Path(index_url) => {
Expand Down
2 changes: 1 addition & 1 deletion crates/uv-resolver/src/pubgrub/report.rs
Original file line number Diff line number Diff line change
Expand Up @@ -621,7 +621,7 @@ impl PubGrubReportFormatter<'_> {
incomplete_packages: &FxHashMap<PackageName, BTreeMap<Version, IncompletePackage>>,
hints: &mut IndexSet<PubGrubHint>,
) {
let no_find_links = index_locations.flat_index().peekable().peek().is_none();
let no_find_links = index_locations.flat_indexes().peekable().peek().is_none();

// Add hints due to the package being entirely unavailable.
match unavailable_packages.get(name) {
Expand Down
13 changes: 9 additions & 4 deletions crates/uv/src/commands/build.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ use distribution_types::{DependencyMetadata, IndexLocations};
use install_wheel_rs::linker::LinkMode;
use owo_colors::OwoColorize;

use uv_auth::store_credentials_from_url;
use uv_auth::{store_credentials, store_credentials_from_url};
use uv_cache::Cache;
use uv_client::{BaseClientBuilder, Connectivity, FlatIndexClient, RegistryClientBuilder};
use uv_configuration::{
Expand Down Expand Up @@ -395,8 +395,13 @@ async fn build_package(
.into_interpreter();

// Add all authenticated sources to the cache.
for url in index_locations.allowed_urls() {
store_credentials_from_url(url);
for index in index_locations.allowed_indexes() {
if let Some(credentials) = index.credentials() {
store_credentials(index.raw_url(), credentials);
}
}
for index in index_locations.flat_indexes() {
store_credentials_from_url(index.url());
}

// Read build constraints.
Expand Down Expand Up @@ -449,7 +454,7 @@ async fn build_package(
// Resolve the flat indexes from `--find-links`.
let flat_index = {
let client = FlatIndexClient::new(&client, cache);
let entries = client.fetch(index_locations.flat_index()).await?;
let entries = client.fetch(index_locations.flat_indexes()).await?;
FlatIndex::from_entries(entries, None, &hasher, build_options)
};

Expand Down
15 changes: 10 additions & 5 deletions crates/uv/src/commands/pip/compile.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ use distribution_types::{
};
use install_wheel_rs::linker::LinkMode;
use pypi_types::{Requirement, SupportedEnvironments};
use uv_auth::store_credentials_from_url;
use uv_auth::{store_credentials, store_credentials_from_url};
use uv_cache::Cache;
use uv_client::{BaseClientBuilder, Connectivity, FlatIndexClient, RegistryClientBuilder};
use uv_configuration::{
Expand Down Expand Up @@ -285,8 +285,13 @@ pub(crate) async fn pip_compile(
);

// Add all authenticated sources to the cache.
for url in index_locations.allowed_urls() {
store_credentials_from_url(url);
for index in index_locations.allowed_indexes() {
if let Some(credentials) = index.credentials() {
store_credentials(index.raw_url(), credentials);
}
}
for index in index_locations.flat_indexes() {
store_credentials_from_url(index.url());
}

// Initialize the registry client.
Expand All @@ -309,7 +314,7 @@ pub(crate) async fn pip_compile(
// Resolve the flat indexes from `--find-links`.
let flat_index = {
let client = FlatIndexClient::new(&client, &cache);
let entries = client.fetch(index_locations.flat_index()).await?;
let entries = client.fetch(index_locations.flat_indexes()).await?;
FlatIndex::from_entries(entries, tags.as_deref(), &hasher, &build_options)
};

Expand Down Expand Up @@ -459,7 +464,7 @@ pub(crate) async fn pip_compile(

// If necessary, include the `--find-links` locations.
if include_find_links {
for flat_index in index_locations.flat_index() {
for flat_index in index_locations.flat_indexes() {
writeln!(writer, "--find-links {}", flat_index.verbatim())?;
wrote_preamble = true;
}
Expand Down
13 changes: 9 additions & 4 deletions crates/uv/src/commands/pip/install.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ use distribution_types::{
use install_wheel_rs::linker::LinkMode;
use pep508_rs::PackageName;
use pypi_types::Requirement;
use uv_auth::store_credentials_from_url;
use uv_auth::{store_credentials, store_credentials_from_url};
use uv_cache::Cache;
use uv_client::{BaseClientBuilder, Connectivity, FlatIndexClient, RegistryClientBuilder};
use uv_configuration::{
Expand Down Expand Up @@ -283,8 +283,13 @@ pub(crate) async fn pip_install(
);

// Add all authenticated sources to the cache.
for url in index_locations.allowed_urls() {
store_credentials_from_url(url);
for index in index_locations.allowed_indexes() {
if let Some(credentials) = index.credentials() {
store_credentials(index.raw_url(), credentials);
}
}
for index in index_locations.flat_indexes() {
store_credentials_from_url(index.url());
}

// Initialize the registry client.
Expand All @@ -302,7 +307,7 @@ pub(crate) async fn pip_install(
// Resolve the flat indexes from `--find-links`.
let flat_index = {
let client = FlatIndexClient::new(&client, &cache);
let entries = client.fetch(index_locations.flat_index()).await?;
let entries = client.fetch(index_locations.flat_indexes()).await?;
FlatIndex::from_entries(entries, Some(&tags), &hasher, &build_options)
};

Expand Down
13 changes: 9 additions & 4 deletions crates/uv/src/commands/pip/sync.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ use tracing::debug;
use distribution_types::{DependencyMetadata, Index, IndexLocations, Resolution};
use install_wheel_rs::linker::LinkMode;
use pep508_rs::PackageName;
use uv_auth::store_credentials_from_url;
use uv_auth::{store_credentials, store_credentials_from_url};
use uv_cache::Cache;
use uv_client::{BaseClientBuilder, Connectivity, FlatIndexClient, RegistryClientBuilder};
use uv_configuration::{
Expand Down Expand Up @@ -226,8 +226,13 @@ pub(crate) async fn pip_sync(
);

// Add all authenticated sources to the cache.
for url in index_locations.allowed_urls() {
store_credentials_from_url(url);
for index in index_locations.allowed_indexes() {
if let Some(credentials) = index.credentials() {
store_credentials(index.raw_url(), credentials);
}
}
for index in index_locations.flat_indexes() {
store_credentials_from_url(index.url());
}

// Initialize the registry client.
Expand All @@ -245,7 +250,7 @@ pub(crate) async fn pip_sync(
// Resolve the flat indexes from `--find-links`.
let flat_index = {
let client = FlatIndexClient::new(&client, &cache);
let entries = client.fetch(index_locations.flat_index()).await?;
let entries = client.fetch(index_locations.flat_indexes()).await?;
FlatIndex::from_entries(entries, Some(&tags), &hasher, &build_options)
};

Expand Down
15 changes: 11 additions & 4 deletions crates/uv/src/commands/project/add.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ use cache_key::RepositoryUrl;
use distribution_types::UnresolvedRequirement;
use pep508_rs::{ExtraName, Requirement, UnnamedRequirement, VersionOrUrl};
use pypi_types::{redact_git_credentials, ParsedUrl, RequirementSource, VerbatimParsedUrl};
use uv_auth::{store_credentials_from_url, Credentials};
use uv_auth::{store_credentials, store_credentials_from_url, Credentials};
use uv_cache::Cache;
use uv_client::{BaseClientBuilder, Connectivity, FlatIndexClient, RegistryClientBuilder};
use uv_configuration::{
Expand Down Expand Up @@ -256,8 +256,13 @@ pub(crate) async fn add(
resolution_environment(python_version, python_platform, target.interpreter())?;

// Add all authenticated sources to the cache.
for url in settings.index_locations.allowed_urls() {
store_credentials_from_url(url);
for index in settings.index_locations.allowed_indexes() {
if let Some(credentials) = index.credentials() {
store_credentials(index.raw_url(), credentials);
}
}
for index in settings.index_locations.flat_indexes() {
store_credentials_from_url(index.url());
}

// Initialize the registry client.
Expand Down Expand Up @@ -286,7 +291,9 @@ pub(crate) async fn add(
// Resolve the flat indexes from `--find-links`.
let flat_index = {
let client = FlatIndexClient::new(&client, cache);
let entries = client.fetch(settings.index_locations.flat_index()).await?;
let entries = client
.fetch(settings.index_locations.flat_indexes())
.await?;
FlatIndex::from_entries(entries, Some(&tags), &hasher, &settings.build_options)
};

Expand Down
13 changes: 9 additions & 4 deletions crates/uv/src/commands/project/lock.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ use distribution_types::{
};
use pep440_rs::Version;
use pypi_types::{Requirement, SupportedEnvironments};
use uv_auth::store_credentials_from_url;
use uv_auth::{store_credentials, store_credentials_from_url};
use uv_cache::Cache;
use uv_client::{Connectivity, FlatIndexClient, RegistryClientBuilder};
use uv_configuration::{
Expand Down Expand Up @@ -354,8 +354,13 @@ async fn do_lock(
PythonRequirement::from_requires_python(interpreter, requires_python.clone());

// Add all authenticated sources to the cache.
for url in index_locations.allowed_urls() {
store_credentials_from_url(url);
for index in index_locations.allowed_indexes() {
if let Some(credentials) = index.credentials() {
store_credentials(index.raw_url(), credentials);
}
}
for index in index_locations.flat_indexes() {
store_credentials_from_url(index.url());
}

// Initialize the registry client.
Expand Down Expand Up @@ -399,7 +404,7 @@ async fn do_lock(
// Resolve the flat indexes from `--find-links`.
let flat_index = {
let client = FlatIndexClient::new(&client, cache);
let entries = client.fetch(index_locations.flat_index()).await?;
let entries = client.fetch(index_locations.flat_indexes()).await?;
FlatIndex::from_entries(entries, None, &hasher, build_options)
};

Expand Down
Loading

0 comments on commit f1d41c4

Please sign in to comment.