Skip to content

Commit

Permalink
Add support for reading and resolving include-group in dependency g…
Browse files Browse the repository at this point in the history 
…roups (#8266)

Part of #8090

Adds the ability to read group inclusions (`include-group = <name>`) in
the `pyproject.toml`. Resolves groups into concrete dependencies for
resolution.

See #8110 for a bit more commentary
on deferred work.

---------

Co-authored-by: Charlie Marsh <[email protected]>
  • Loading branch information
@zanieb @charliermarsh
zanieb and charliermarsh committed Oct 22, 2024
1 parent fad72f8 commit 6000235
Show file tree
Hide file tree
Showing 6 changed files with 821 additions and 98 deletions.
27 changes: 25 additions & 2 deletions crates/uv-distribution/src/metadata/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,37 @@ pub enum MetadataError {
Workspace(#[from] WorkspaceError),
#[error("Failed to parse entry: `{0}`")]
LoweringError(PackageName, #[source] Box<LoweringError>),
#[error("Failed to parse entry in `{0}`: `{1}`")]
#[error("Failed to parse entry in group `{0}`: `{1}`")]
GroupLoweringError(GroupName, PackageName, #[source] Box<LoweringError>),
#[error("Failed to parse entry in `{0}`: `{1}`")]
#[error("Failed to parse entry in group `{0}`: `{1}`")]
GroupParseError(
GroupName,
String,
#[source] Box<Pep508Error<VerbatimParsedUrl>>,
),
#[error("Failed to find group `{0}` included by `{1}`")]
GroupNotFound(GroupName, GroupName),
#[error("Detected a cycle in `dependency-groups`: {0}")]
DependencyGroupCycle(Cycle),
}

/// A cycle in the `dependency-groups` table.
#[derive(Debug)]
pub struct Cycle(Vec<GroupName>);

/// Display a cycle, e.g., `a -> b -> c -> a`.
impl std::fmt::Display for Cycle {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let [first, rest @ ..] = self.0.as_slice() else {
return Ok(());
};
write!(f, "`{first}`")?;
for group in rest {
write!(f, " -> `{group}`")?;
}
write!(f, " -> `{first}`")?;
Ok(())
}
}

#[derive(Debug, Clone)]
Expand Down
220 changes: 135 additions & 85 deletions crates/uv-distribution/src/metadata/requires_dist.rs
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,19 @@
use crate::metadata::{LoweredRequirement, MetadataError};
use crate::Metadata;

use std::collections::BTreeMap;
use std::path::Path;
use std::str::FromStr;

use tracing::warn;

use uv_configuration::{LowerBound, SourceStrategy};
use uv_distribution_types::IndexLocations;
use uv_normalize::{ExtraName, GroupName, PackageName, DEV_DEPENDENCIES};
use uv_pypi_types::VerbatimParsedUrl;
use uv_workspace::pyproject::{Sources, ToolUvSources};
use uv_workspace::pyproject::{DependencyGroupSpecifier, ToolUvSources};
use uv_workspace::{DiscoveryOptions, ProjectWorkspace};

use crate::metadata::{Cycle, LoweredRequirement, MetadataError};
use crate::Metadata;

#[derive(Debug, Clone)]
pub struct RequiresDist {
pub name: PackageName,
Expand Down Expand Up @@ -116,55 +118,71 @@ impl RequiresDist {
.dependency_groups
.iter()
.flatten()
.map(|(name, requirements)| {
(
name.clone(),
requirements
.iter()
.map(|requirement| {
match uv_pep508::Requirement::<VerbatimParsedUrl>::from_str(
requirement,
) {
Ok(requirement) => Ok(requirement),
Err(err) => Err(MetadataError::GroupParseError(
name.clone(),
requirement.clone(),
Box::new(err),
)),
}
})
.collect::<Result<Vec<_>, _>>(),
)
})
.collect::<BTreeMap<_, _>>();

// Resolve any `include-group` entries in `dependency-groups`.
let dependency_groups = resolve_dependency_groups(&dependency_groups)?
.into_iter()
.chain(
// Only add the `dev` group if `dev-dependencies` is defined
// Only add the `dev` group if `dev-dependencies` is defined.
dev_dependencies
.into_iter()
.map(|requirements| (DEV_DEPENDENCIES.clone(), Ok(requirements.clone()))),
.map(|requirements| (DEV_DEPENDENCIES.clone(), requirements.clone())),
)
.map(|(name, requirements)| {
// Apply sources to the requirements
match requirements {
Ok(requirements) => match apply_source_strategy(
source_strategy,
requirements,
&metadata,
project_sources,
project_indexes,
locations,
project_workspace,
lower_bound,
&name,
) {
Ok(requirements) => Ok((name, requirements)),
Err(err) => Err(err),
},
Err(err) => Err(err),
}
let requirements = match source_strategy {
SourceStrategy::Enabled => requirements
.into_iter()
.flat_map(|requirement| {
let group_name = name.clone();
let requirement_name = requirement.name.clone();
LoweredRequirement::from_requirement(
requirement,
&metadata.name,
project_workspace.project_root(),
project_sources,
project_indexes,
locations,
project_workspace.workspace(),
lower_bound,
)
.map(move |requirement| {
match requirement {
Ok(requirement) => Ok(requirement.into_inner()),
Err(err) => Err(MetadataError::GroupLoweringError(
group_name.clone(),
requirement_name.clone(),
Box::new(err),
)),
}
})
})
.collect::<Result<Vec<_>, _>>(),
SourceStrategy::Disabled => Ok(requirements
.into_iter()
.map(uv_pypi_types::Requirement::from)
.collect()),
}?;
Ok::<(GroupName, Vec<uv_pypi_types::Requirement>), MetadataError>((
name,
requirements,
))
})
.collect::<Result<Vec<_>, _>>()?;

dependency_groups.into_iter().collect::<BTreeMap<_, _>>()
// Merge any overlapping groups.
let mut map = BTreeMap::new();
for (name, dependencies) in dependency_groups {
match map.entry(name) {
std::collections::btree_map::Entry::Vacant(entry) => {
entry.insert(dependencies);
}
std::collections::btree_map::Entry::Occupied(mut entry) => {
entry.get_mut().extend(dependencies);
}
}
}
map
};

let requires_dist = metadata.requires_dist.into_iter();
Expand Down Expand Up @@ -217,47 +235,79 @@ impl From<Metadata> for RequiresDist {
}
}

fn apply_source_strategy(
source_strategy: SourceStrategy,
requirements: Vec<uv_pep508::Requirement<VerbatimParsedUrl>>,
metadata: &uv_pypi_types::RequiresDist,
project_sources: &BTreeMap<PackageName, Sources>,
project_indexes: &[uv_distribution_types::Index],
locations: &IndexLocations,
project_workspace: &ProjectWorkspace,
lower_bound: LowerBound,
group_name: &GroupName,
) -> Result<Vec<uv_pypi_types::Requirement>, MetadataError> {
match source_strategy {
SourceStrategy::Enabled => requirements
.into_iter()
.flat_map(|requirement| {
let requirement_name = requirement.name.clone();
LoweredRequirement::from_requirement(
requirement,
&metadata.name,
project_workspace.project_root(),
project_sources,
project_indexes,
locations,
project_workspace.workspace(),
lower_bound,
)
.map(move |requirement| match requirement {
Ok(requirement) => Ok(requirement.into_inner()),
Err(err) => Err(MetadataError::GroupLoweringError(
group_name.clone(),
requirement_name.clone(),
Box::new(err),
)),
})
})
.collect::<Result<Vec<_>, _>>(),
SourceStrategy::Disabled => Ok(requirements
.into_iter()
.map(uv_pypi_types::Requirement::from)
.collect()),
/// Resolve the dependency groups (which may contain references to other groups) into concrete
/// lists of requirements.
fn resolve_dependency_groups(
groups: &BTreeMap<&GroupName, &Vec<DependencyGroupSpecifier>>,
) -> Result<BTreeMap<GroupName, Vec<uv_pep508::Requirement<VerbatimParsedUrl>>>, MetadataError> {
fn resolve_group<'data>(
resolved: &mut BTreeMap<GroupName, Vec<uv_pep508::Requirement<VerbatimParsedUrl>>>,
groups: &'data BTreeMap<&GroupName, &Vec<DependencyGroupSpecifier>>,
name: &'data GroupName,
parents: &mut Vec<&'data GroupName>,
) -> Result<(), MetadataError> {
let Some(specifiers) = groups.get(name) else {
// Missing group
let parent_name = parents
.iter()
.last()
.copied()
.expect("parent when group is missing");
return Err(MetadataError::GroupNotFound(
name.clone(),
parent_name.clone(),
));
};

// "Dependency Group Includes MUST NOT include cycles, and tools SHOULD report an error if they detect a cycle."
if parents.contains(&name) {
return Err(MetadataError::DependencyGroupCycle(Cycle(
parents.iter().copied().cloned().collect(),
)));
}

// If we already resolved this group, short-circuit.
if resolved.contains_key(name) {
return Ok(());
}

parents.push(name);
let mut requirements = Vec::with_capacity(specifiers.len());
for specifier in *specifiers {
match specifier {
DependencyGroupSpecifier::Requirement(requirement) => {
match uv_pep508::Requirement::<VerbatimParsedUrl>::from_str(requirement) {
Ok(requirement) => requirements.push(requirement),
Err(err) => {
return Err(MetadataError::GroupParseError(
name.clone(),
requirement.clone(),
Box::new(err),
));
}
}
}
DependencyGroupSpecifier::IncludeGroup { include_group } => {
resolve_group(resolved, groups, include_group, parents)?;
requirements.extend(resolved.get(include_group).into_iter().flatten().cloned());
}
DependencyGroupSpecifier::Object(map) => {
warn!("Ignoring Dependency Object Specifier referenced by `{name}`: {map:?}");
}
}
}
parents.pop();

resolved.insert(name.clone(), requirements);
Ok(())
}

let mut resolved = BTreeMap::new();
for name in groups.keys() {
let mut parents = Vec::new();
resolve_group(&mut resolved, groups, name, &mut parents)?;
}
Ok(resolved)
}

#[cfg(test)]
Expand Down
Loading

0 comments on commit 6000235

Please sign in to comment.