diff --git a/Cargo.lock b/Cargo.lock index 68c10fe71..56d5125db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2925,7 +2925,7 @@ dependencies = [ [[package]] name = "rattler" version = "0.16.2" -source = "git+https://github.com/mamba-org/rattler?branch=main#0128bce8617cacc3ea6dcf0948879b5344708f71" +source = "git+https://github.com/mamba-org/rattler?branch=main#14708a2a991dee6c95b776538ece4dcabce82f2e" dependencies = [ "anyhow", "async-compression 0.4.6", @@ -2969,7 +2969,7 @@ dependencies = [ [[package]] name = "rattler_conda_types" version = "0.16.2" -source = "git+https://github.com/mamba-org/rattler?branch=main#0128bce8617cacc3ea6dcf0948879b5344708f71" +source = "git+https://github.com/mamba-org/rattler?branch=main#14708a2a991dee6c95b776538ece4dcabce82f2e" dependencies = [ "chrono", "fxhash", @@ -3013,7 +3013,7 @@ dependencies = [ [[package]] name = "rattler_digest" version = "0.16.2" -source = "git+https://github.com/mamba-org/rattler?branch=main#0128bce8617cacc3ea6dcf0948879b5344708f71" +source = "git+https://github.com/mamba-org/rattler?branch=main#14708a2a991dee6c95b776538ece4dcabce82f2e" dependencies = [ "blake2", "digest", @@ -3086,7 +3086,7 @@ dependencies = [ [[package]] name = "rattler_lock" version = "0.16.2" -source = "git+https://github.com/mamba-org/rattler?branch=main#0128bce8617cacc3ea6dcf0948879b5344708f71" +source = "git+https://github.com/mamba-org/rattler?branch=main#14708a2a991dee6c95b776538ece4dcabce82f2e" dependencies = [ "chrono", "fxhash", @@ -3109,7 +3109,7 @@ dependencies = [ [[package]] name = "rattler_macros" version = "0.16.2" -source = "git+https://github.com/mamba-org/rattler?branch=main#0128bce8617cacc3ea6dcf0948879b5344708f71" +source = "git+https://github.com/mamba-org/rattler?branch=main#14708a2a991dee6c95b776538ece4dcabce82f2e" dependencies = [ "quote", "syn 2.0.48", @@ -3118,7 +3118,7 @@ dependencies = [ [[package]] name = "rattler_networking" version = "0.16.2" -source = "git+https://github.com/mamba-org/rattler?branch=main#0128bce8617cacc3ea6dcf0948879b5344708f71" +source = "git+https://github.com/mamba-org/rattler?branch=main#14708a2a991dee6c95b776538ece4dcabce82f2e" dependencies = [ "anyhow", "async-trait", @@ -3146,7 +3146,7 @@ dependencies = [ [[package]] name = "rattler_package_streaming" version = "0.16.2" -source = "git+https://github.com/mamba-org/rattler?branch=main#0128bce8617cacc3ea6dcf0948879b5344708f71" +source = "git+https://github.com/mamba-org/rattler?branch=main#14708a2a991dee6c95b776538ece4dcabce82f2e" dependencies = [ "bzip2", "chrono", @@ -3172,7 +3172,7 @@ dependencies = [ [[package]] name = "rattler_repodata_gateway" version = "0.16.2" -source = "git+https://github.com/mamba-org/rattler?branch=main#0128bce8617cacc3ea6dcf0948879b5344708f71" +source = "git+https://github.com/mamba-org/rattler?branch=main#14708a2a991dee6c95b776538ece4dcabce82f2e" dependencies = [ "anyhow", "async-compression 0.4.6", @@ -3211,7 +3211,7 @@ dependencies = [ [[package]] name = "rattler_shell" version = "0.16.2" -source = "git+https://github.com/mamba-org/rattler?branch=main#0128bce8617cacc3ea6dcf0948879b5344708f71" +source = "git+https://github.com/mamba-org/rattler?branch=main#14708a2a991dee6c95b776538ece4dcabce82f2e" dependencies = [ "enum_dispatch", "indexmap 2.1.0", @@ -3228,7 +3228,7 @@ dependencies = [ [[package]] name = "rattler_solve" version = "0.16.2" -source = "git+https://github.com/mamba-org/rattler?branch=main#0128bce8617cacc3ea6dcf0948879b5344708f71" +source = "git+https://github.com/mamba-org/rattler?branch=main#14708a2a991dee6c95b776538ece4dcabce82f2e" dependencies = [ "anyhow", "chrono", @@ -3247,7 +3247,7 @@ dependencies = [ [[package]] name = "rattler_virtual_packages" version = "0.16.2" -source = "git+https://github.com/mamba-org/rattler?branch=main#0128bce8617cacc3ea6dcf0948879b5344708f71" +source = "git+https://github.com/mamba-org/rattler?branch=main#14708a2a991dee6c95b776538ece4dcabce82f2e" dependencies = [ "cfg-if", "libloading", diff --git a/src/cli/add.rs b/src/cli/add.rs index a8bc0534a..8d6cb2c47 100644 --- a/src/cli/add.rs +++ b/src/cli/add.rs @@ -6,10 +6,12 @@ use crate::{ use clap::Parser; use itertools::{Either, Itertools}; +use indexmap::IndexMap; use miette::{IntoDiagnostic, WrapErr}; use rattler_conda_types::{ version_spec::{LogicalOperator, RangeOperator}, - MatchSpec, NamelessMatchSpec, PackageName, Platform, Version, VersionBumpType, VersionSpec, + Channel, MatchSpec, NamelessMatchSpec, PackageName, Platform, Version, VersionBumpType, + VersionSpec, }; use rattler_repodata_gateway::sparse::SparseRepoData; use rattler_solve::{resolvo, SolverImpl}; @@ -290,7 +292,7 @@ pub async fn add_conda_specs_to_project( ) { Ok(versions) => versions, Err(err) => { - return Err(err).wrap_err_with(||miette::miette!( + return Err(err).wrap_err_with(|| miette::miette!( "could not determine any available versions for {} on {platform}. Either the package could not be found or version constraints on other dependencies result in a conflict.", new_specs.keys().map(|s| s.as_source()).join(", ") )); @@ -353,7 +355,7 @@ pub fn determine_best_version( project: &Project, new_specs: &HashMap, new_specs_type: SpecType, - sparse_repo_data: &[SparseRepoData], + sparse_repo_data: &IndexMap<(Channel, Platform), SparseRepoData>, platform: Platform, ) -> miette::Result> { // Build the combined set of specs while updating the dependencies with the new specs. @@ -375,9 +377,12 @@ pub fn determine_best_version( let package_names = dependencies.names().cloned().collect_vec(); // Get the repodata for the current platform and for NoArch - let platform_sparse_repo_data = sparse_repo_data.iter().filter(|sparse| { - sparse.subdir() == platform.as_str() || sparse.subdir() == Platform::NoArch.as_str() - }); + let platform_sparse_repo_data = project + .channels() + .into_iter() + .cloned() + .cartesian_product(vec![platform, Platform::NoArch]) + .filter_map(|target| sparse_repo_data.get(&target)); // Load only records we need for this platform let available_packages = SparseRepoData::load_records_recursive( diff --git a/src/cli/global/install.rs b/src/cli/global/install.rs index 87b21de34..b41b849cb 100644 --- a/src/cli/global/install.rs +++ b/src/cli/global/install.rs @@ -3,6 +3,7 @@ use crate::repodata::friendly_channel_name; use crate::{config, prefix::Prefix, progress::await_in_progress, repodata::fetch_sparse_repodata}; use clap::Parser; use dirs::home_dir; +use indexmap::IndexMap; use itertools::Itertools; use miette::IntoDiagnostic; use rattler::install::Transaction; @@ -386,10 +387,10 @@ pub async fn execute(args: Args) -> miette::Result<()> { ) } else { eprintln!("{whitespace}These apps have been added to {}\n{whitespace} - {script_names}\n\n{} To use them, make sure to add {} to your PATH", - console::style(&bin_dir.display()).bold(), - console::style("!").yellow().bold(), - console::style(&bin_dir.display()).bold() - ) + console::style(&bin_dir.display()).bold(), + console::style("!").yellow().bold(), + console::style(&bin_dir.display()).bold() + ) } Ok(()) @@ -397,14 +398,14 @@ pub async fn execute(args: Args) -> miette::Result<()> { pub(super) async fn globally_install_package( package_matchspec: MatchSpec, - platform_sparse_repodata: &[SparseRepoData], + sparse_repodata: &IndexMap<(Channel, Platform), SparseRepoData>, channel_config: &ChannelConfig, authenticated_client: ClientWithMiddleware, ) -> miette::Result<(PrefixRecord, Vec, bool)> { let package_name = package_name(&package_matchspec)?; let available_packages = SparseRepoData::load_records_recursive( - platform_sparse_repodata, + sparse_repodata.values(), vec![package_name.clone()], None, ) diff --git a/src/cli/run.rs b/src/cli/run.rs index e6a70aeaf..32ccc5ccd 100644 --- a/src/cli/run.rs +++ b/src/cli/run.rs @@ -12,6 +12,7 @@ use crate::task::{ }; use crate::Project; +use crate::project::manifest::EnvironmentName; use thiserror::Error; use tracing::Level; @@ -28,13 +29,21 @@ pub struct Args { #[clap(flatten)] pub lock_file_usage: super::LockFileUsageArgs, + + #[arg(long, short)] + pub environment: Option, } /// CLI entry point for `pixi run` /// When running the sigints are ignored and child can react to them. As it pleases. pub async fn execute(args: Args) -> miette::Result<()> { let project = Project::load_or_else_discover(args.manifest_path.as_deref())?; - let environment = project.default_environment(); + let environment_name = args + .environment + .map_or_else(|| EnvironmentName::Default, EnvironmentName::Named); + let environment = project + .environment(&environment_name) + .ok_or_else(|| miette::miette!("unknown environment '{environment_name}'"))?; // Split 'task' into arguments if it's a single string, supporting commands like: // `"test 1 == 0 || echo failed"` or `"echo foo && echo bar"` or `"echo 'Hello World'"` diff --git a/src/cli/search.rs b/src/cli/search.rs index 36b714b97..3cb069e24 100644 --- a/src/cli/search.rs +++ b/src/cli/search.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use std::{cmp::Ordering, path::PathBuf}; use clap::Parser; +use indexmap::IndexMap; use itertools::Itertools; use miette::IntoDiagnostic; use rattler_conda_types::{Channel, ChannelConfig, PackageName, Platform, RepoDataRecord}; @@ -41,7 +42,7 @@ pub struct Args { /// fetch packages from `repo_data` based on `filter_func` fn search_package_by_filter( package: &PackageName, - repo_data: &[SparseRepoData], + repo_data: Arc>, filter_func: F, ) -> miette::Result> where @@ -49,7 +50,7 @@ where { let similar_packages = repo_data .iter() - .flat_map(|repo| { + .flat_map(|(_, repo)| { repo.package_names() .filter(|&name| filter_func(name, package)) }) @@ -59,7 +60,7 @@ where // search for `similar_packages` in all platform's repodata // add the latest version of the fetched package to latest_packages vector - for repo in repo_data { + for repo in repo_data.values() { for package in &similar_packages { let mut records = repo .load_records(&PackageName::new_unchecked(*package)) @@ -104,15 +105,18 @@ pub async fn execute(args: Args) -> miette::Result<()> { }; let package_name_filter = args.package; + let authenticated_client = reqwest_middleware::ClientBuilder::new(reqwest::Client::new()) .with_arc(Arc::new(AuthenticationMiddleware::default())) .build(); - let repo_data = fetch_sparse_repodata( - channels.iter().map(AsRef::as_ref), - [Platform::current()], - &authenticated_client, - ) - .await?; + let repo_data = Arc::new( + fetch_sparse_repodata( + channels.iter().map(AsRef::as_ref), + [Platform::current()], + &authenticated_client, + ) + .await?, + ); // When package name filter contains * (wildcard), it will search and display a list of packages matching this filter if package_name_filter.contains('*') { @@ -135,14 +139,14 @@ pub async fn execute(args: Args) -> miette::Result<()> { async fn search_exact_package( package_name: PackageName, - repo_data: Vec, + repo_data: Arc>, out: W, ) -> miette::Result<()> { let package_name_search = package_name.clone(); let packages = await_in_progress( "searching packages", spawn_blocking(move || { - search_package_by_filter(&package_name_search, &repo_data, |pn, n| { + search_package_by_filter(&package_name_search, repo_data, |pn, n| { pn == n.as_normalized() }) }), @@ -274,7 +278,7 @@ fn print_package_info(package: &RepoDataRecord, mut out: W) -> io::Res async fn search_package_by_wildcard( package_name: PackageName, package_name_filter: &str, - repo_data: Vec, + repo_data: Arc>, limit: usize, out: W, ) -> miette::Result<()> { @@ -285,16 +289,17 @@ async fn search_package_by_wildcard( let mut packages = await_in_progress( "searching packages", spawn_blocking(move || { - let packages = search_package_by_filter(&package_name_search, &repo_data, |pn, _| { - wildcard_pattern.is_match(pn) - }); + let packages = + search_package_by_filter(&package_name_search, repo_data.clone(), |pn, _| { + wildcard_pattern.is_match(pn) + }); match packages { Ok(packages) => { if packages.is_empty() { let similarity = 0.6; return search_package_by_filter( &package_name_search, - &repo_data, + repo_data, |pn, n| jaro(pn, n.as_normalized()) > similarity, ); } diff --git a/src/cli/shell.rs b/src/cli/shell.rs index 4d9a32a9e..8f79f00be 100644 --- a/src/cli/shell.rs +++ b/src/cli/shell.rs @@ -13,6 +13,7 @@ use std::path::PathBuf; use crate::unix::PtySession; use crate::cli::LockFileUsageArgs; +use crate::project::manifest::EnvironmentName; #[cfg(target_family = "windows")] use rattler_shell::shell::CmdExe; @@ -25,6 +26,9 @@ pub struct Args { #[clap(flatten)] lock_file_usage: LockFileUsageArgs, + + #[arg(long, short)] + environment: Option, } fn start_powershell( @@ -192,7 +196,12 @@ async fn start_nu_shell( pub async fn execute(args: Args) -> miette::Result<()> { let project = Project::load_or_else_discover(args.manifest_path.as_deref())?; - let environment = project.default_environment(); + let environment_name = args + .environment + .map_or_else(|| EnvironmentName::Default, EnvironmentName::Named); + let environment = project + .environment(&environment_name) + .ok_or_else(|| miette::miette!("unknown environment '{environment_name}'"))?; // Get the environment variables we need to set activate the project in the shell. let env = get_activation_env(&environment, args.lock_file_usage.into()).await?; diff --git a/src/environment.rs b/src/environment.rs index 14bd7f7f9..41e8fd414 100644 --- a/src/environment.rs +++ b/src/environment.rs @@ -2,20 +2,20 @@ use crate::project::Environment; use crate::{config, consts, install, install_pypi, lock_file, prefix::Prefix, progress, Project}; use miette::{Context, IntoDiagnostic}; -use crate::lock_file::lock_file_satisfies_project; +use crate::lock_file::verify_environment_satisfiability; use crate::project::manifest::SystemRequirements; use crate::project::virtual_packages::verify_current_platform_has_required_virtual_packages; +use crate::repodata::fetch_sparse_repodata_targets; +use indexmap::{IndexMap, IndexSet}; use itertools::Itertools; use rattler::install::{PythonInfo, Transaction}; -use rattler_conda_types::{Platform, PrefixRecord, RepoDataRecord}; +use rattler_conda_types::{Channel, Platform, PrefixRecord, RepoDataRecord}; use rattler_lock::{LockFile, PypiPackageData, PypiPackageEnvironmentData}; use rattler_repodata_gateway::sparse::SparseRepoData; use reqwest_middleware::ClientWithMiddleware; use rip::index::PackageDb; use rip::resolve::SDistResolution; -use std::error::Error; -use std::fmt::Write; -use std::{io::ErrorKind, path::Path}; +use std::{collections::HashMap, error::Error, fmt::Write, io::ErrorKind, path::Path, sync::Arc}; /// Verify the location of the prefix folder is not changed so the applied prefix path is still valid. /// Errors when there is a file system error or the path does not align with the defined prefix. @@ -126,14 +126,14 @@ impl LockFileUsage { /// while to load. If `sparse_repo_data` is `None` it will be downloaded. If the lock-file is not /// updated, the `sparse_repo_data` is ignored. pub async fn get_up_to_date_prefix<'p>( - environment: &'p Environment<'p>, + prefix_env: &'p Environment<'p>, usage: LockFileUsage, mut no_install: bool, - sparse_repo_data: Option>, + sparse_repo_data: Option>, sdist_resolution: SDistResolution, ) -> miette::Result { let current_platform = Platform::current(); - let project = environment.project(); + let project = prefix_env.project(); // Do not install if the platform is not supported if !no_install && !project.platforms().contains(¤t_platform) { @@ -144,181 +144,275 @@ pub async fn get_up_to_date_prefix<'p>( // Make sure the project is in a sane state sanity_check_project(project)?; - // Determine which environment to install. - let environment = project.default_environment(); - - // Early out if If there is no lock-file and we are also not allowed to update it. + // Early out if there is no lock-file and we are also not allowed to update it. if !project.lock_file_path().is_file() && !usage.allows_lock_file_updates() { miette::bail!("no lockfile available, can't do a frozen installation."); } - // Start loading the installed packages in the background - let prefix = Prefix::new(environment.dir())?; - let installed_packages_future = { - let prefix = prefix.clone(); - tokio::spawn(async move { prefix.find_installed_packages(None).await }) - }; - // Load the lock-file into memory. let lock_file = lock_file::load_lock_file(project).await?; - // Check if the lock-file is up to date, but only if the current usage allows it. - let update_lock_file = if usage.should_check_if_out_of_date() { - match lock_file_satisfies_project(project, &lock_file) { - Err(err) => { - // Construct an error message - let mut report = String::new(); - let mut err: &dyn Error = &err; - write!(&mut report, "{}", err).unwrap(); - while let Some(source) = err.source() { - write!(&mut report, "\nbecause {}", source).unwrap(); - err = source - } - - tracing::info!("lock-file is not up to date with the project\nbecause {report}",); - - if !usage.allows_lock_file_updates() { - miette::bail!("lock-file not up-to-date with the project"); + let out_of_date_environments = if usage.should_check_if_out_of_date() { + let mut out_of_date_environments = IndexSet::new(); + for environment in project.environments() { + // Determine if we need to update this environment + match verify_environment_satisfiability( + &environment, + lock_file.environment(environment.name().as_str()), + ) { + Ok(_) => {} + Err(err) => { + // Construct an error message + let mut report = String::new(); + let mut err: &dyn Error = &err; + write!(&mut report, "{}", err).unwrap(); + while let Some(source) = err.source() { + write!(&mut report, ", because {}", source).unwrap(); + err = source + } + + tracing::info!("environment '{}' in the lock-file is not up to date with the project, because {report}", environment.name()); + + out_of_date_environments.insert(environment); } - - true - } - Ok(_) => { - tracing::debug!("the lock-file is up to date with the project.",); - false } } + + out_of_date_environments } else { - false + IndexSet::default() }; - // Get the environment from the lock-file. - let locked_environment = lock_file.environment(environment.name().as_str()); - - // Get all the repodata records from the lock-file - let locked_repodata_records = locked_environment - .as_ref() - .map(|env| env.conda_repodata_records()) - .transpose() - .into_diagnostic() - .context("failed to parse the contents of the lock-file. Try removing the lock-file and running again")? - .unwrap_or_default(); - - // If the lock-file requires an updates, update the conda records. - // - // The `updated_repodata_records` fields holds the updated records if the records are updated. - // - // Depending on whether the lock-filed was updated the `repodata_records` field either points - // to the `locked_repodata_records` or to the `updated_repodata_records`. - let mut updated_repodata_records = None; - let repodata_records: &_ = if update_lock_file { - updated_repodata_records.insert( - lock_file::update_lock_file_conda( + // If there are out of date environments but we are not allowed to update the lock-file, error out. + if !out_of_date_environments.is_empty() && !usage.allows_lock_file_updates() { + miette::bail!("lock-file not up-to-date with the project"); + } + + // Download all the required repodata + let targets_to_fetch = out_of_date_environments + .iter() + .flat_map(|env| { + let mut platforms = env.platforms(); + platforms.insert(Platform::NoArch); + env.channels() + .into_iter() + .cloned() + .cartesian_product(platforms.into_iter().collect_vec()) + }) + .filter(|target| { + sparse_repo_data + .as_ref() + .map(|p| !p.contains_key(target)) + .unwrap_or(true) + }) + .collect::>(); + let mut fetched_repo_data = + fetch_sparse_repodata_targets(targets_to_fetch, project.authenticated_client()).await?; + fetched_repo_data.extend(sparse_repo_data.into_iter().flatten()); + let fetched_repo_data = Arc::new(fetched_repo_data); + + let mut updated_conda_records: HashMap<_, HashMap<_, _>> = HashMap::new(); + let mut updated_pypi_records: HashMap<_, HashMap<_, _>> = HashMap::new(); + let mut old_repodata_records = HashMap::new(); + let mut old_pypi_records = HashMap::new(); + + // Iterate over all environments in the project + for environment in project.environments() { + let is_wanted_environment = environment == *prefix_env; + let is_out_of_date_environment = out_of_date_environments.contains(&environment); + + // If this environment is not out of date and also not the environment we are installing, we + // can skip it. + if !is_out_of_date_environment && !is_wanted_environment { + continue; + } + + // Start loading the installed packages in the background + let prefix = Prefix::new(environment.dir())?; + let installed_packages_future = { + let prefix = prefix.clone(); + tokio::spawn(async move { prefix.find_installed_packages(None).await }) + }; + + // Get the environment from the lock-file. + let locked_environment = lock_file.environment(environment.name().as_str()); + + // Get all the repodata records from the lock-file + let locked_repodata_records = locked_environment + .as_ref() + .map(|env| env.conda_repodata_records()) + .transpose() + .into_diagnostic() + .context("failed to parse the contents of the lock-file. Try removing the lock-file and running again")? + .unwrap_or_default(); + + // If the lock-file requires an updates, update the conda records. + // + // The `updated_repodata_records` fields holds the updated records if the records are updated. + // + // Depending on whether the lock-filed was updated the `repodata_records` field either points + // to the `locked_repodata_records` or to the `updated_repodata_records`. + let repodata_records: &_ = if is_out_of_date_environment { + let records = lock_file::update_lock_file_conda( &environment, &locked_repodata_records, - sparse_repo_data, + &fetched_repo_data, ) - .await?, - ) - } else { - &locked_repodata_records - }; - - // Update the prefix with the conda packages. This will also return the python status. - let python_status = if !no_install { - let installed_prefix_records = installed_packages_future.await.into_diagnostic()??; - let empty_vec = Vec::new(); - update_prefix_conda( - &prefix, - project.authenticated_client().clone(), - installed_prefix_records, - repodata_records - .get(¤t_platform) - .unwrap_or(&empty_vec), - Platform::current(), - ) - .await? - } else { - // We don't know and it won't matter because we won't install pypi either - PythonStatus::DoesNotExist - }; + .await?; + + updated_conda_records + .entry(environment.clone()) + .or_insert(records) + } else { + &locked_repodata_records + }; + + let should_update_prefix = is_wanted_environment + || (is_out_of_date_environment && environment.has_pypi_dependencies()); + + // Update the prefix with the conda packages. This will also return the python status. + let python_status = if should_update_prefix && !no_install { + let installed_prefix_records = installed_packages_future.await.into_diagnostic()??; + let empty_vec = Vec::new(); + update_prefix_conda( + environment.name().as_str(), + &prefix, + project.authenticated_client().clone(), + installed_prefix_records, + repodata_records + .get(¤t_platform) + .unwrap_or(&empty_vec), + Platform::current(), + ) + .await? + } else { + // We don't know and it won't matter because we won't install pypi either + PythonStatus::DoesNotExist + }; + + // If there are no pypi dependencies, we don't need to do anything else. + if !environment.has_pypi_dependencies() { + continue; + } - // Get the current pypi dependencies from the lock-file. - let locked_pypi_records = locked_environment - .map(|env| env.pypi_packages()) - .unwrap_or_default(); - - // If the project has pypi dependencies and we need to update the lock-file lets do so here. - // - // The `updated_pypi_records` fields holds the updated records if the records are updated. - // - // Depending on whether the lock-file was updated the `pypi_records` field either points - // to the `locked_pypi_records` or to the `updated_pypi_records`. - let mut updated_pypi_records = None; - let pypi_records: &_ = if project.has_pypi_dependencies() && update_lock_file { - let python_path = python_status.location().map(|p| prefix.root().join(p)); - updated_pypi_records.insert( - lock_file::update_lock_file_for_pypi( + // Get the current pypi dependencies from the lock-file. + let locked_pypi_records = locked_environment + .map(|env| env.pypi_packages()) + .unwrap_or_default(); + + // If the project has pypi dependencies and we need to update the lock-file lets do so here. + // + // The `updated_pypi_records` fields holds the updated records if the records are updated. + // + // Depending on whether the lock-file was updated the `pypi_records` field either points + // to the `locked_pypi_records` or to the `updated_pypi_records`. + let pypi_records: &_ = if is_out_of_date_environment { + let python_path = python_status.location().map(|p| prefix.root().join(p)); + let records = lock_file::update_lock_file_for_pypi( &environment, repodata_records, &locked_pypi_records, python_path.as_deref(), sdist_resolution, ) - .await?, - ) - } else { - &locked_pypi_records - }; + .await?; + + updated_pypi_records + .entry(environment.clone()) + .or_insert(records) + } else { + &locked_pypi_records + }; + + // If there are + if is_wanted_environment && pypi_records.get(¤t_platform).is_some() && !no_install { + // Then update the pypi packages. + let empty_repodata_vec = Vec::new(); + let empty_pypi_vec = Vec::new(); + update_prefix_pypi( + environment.name().as_str(), + &prefix, + current_platform, + project.pypi_package_db()?, + repodata_records + .get(¤t_platform) + .unwrap_or(&empty_repodata_vec), + pypi_records + .get(¤t_platform) + .unwrap_or(&empty_pypi_vec), + &python_status, + &project.system_requirements(), + sdist_resolution, + ) + .await?; + } - if project.has_pypi_dependencies() && !no_install { - // Then update the pypi packages. - let empty_repodata_vec = Vec::new(); - let empty_pypi_vec = Vec::new(); - update_prefix_pypi( - &prefix, - current_platform, - project.pypi_package_db()?, - repodata_records - .get(¤t_platform) - .unwrap_or(&empty_repodata_vec), - pypi_records - .get(¤t_platform) - .unwrap_or(&empty_pypi_vec), - &python_status, - &project.system_requirements(), - sdist_resolution, - ) - .await?; + old_repodata_records.insert(environment.clone(), locked_repodata_records); + old_pypi_records.insert(environment, locked_pypi_records); } // If any of the records have changed we need to update the contents of the lock-file. - if updated_repodata_records.is_some() || updated_pypi_records.is_some() { + if !updated_conda_records.is_empty() || !updated_pypi_records.is_empty() { let mut builder = LockFile::builder(); - let channels = environment - .channels() - .into_iter() - .map(|channel| rattler_lock::Channel::from(channel.base_url().to_string())) - .collect_vec(); - builder.set_channels(environment.name().as_str(), channels); - - // Add the conda records - for (platform, records) in updated_repodata_records.unwrap_or(locked_repodata_records) { - for record in records { - builder.add_conda_package(environment.name().as_str(), platform, record.into()); - } - } + for environment in project.environments() { + let channels = environment + .channels() + .into_iter() + .map(|channel| rattler_lock::Channel::from(channel.base_url().to_string())) + .collect_vec(); + builder.set_channels(environment.name().as_str(), channels); + + let mut loaded_repodata_records = old_repodata_records + .remove(&environment) + .unwrap_or_default(); + let mut loaded_pypi_records = old_pypi_records.remove(&environment).unwrap_or_default(); + + let mut updated_repodata_records = updated_conda_records + .remove(&environment) + .unwrap_or_default(); + let mut updated_pypi_records = updated_pypi_records + .remove(&environment) + .unwrap_or_default(); + + let locked_environment = lock_file.environment(environment.name().as_str()); + + for platform in environment.platforms() { + let repodata_records = if let Some(records) = updated_repodata_records + .remove(&platform) + .or_else(|| loaded_repodata_records.remove(&platform)) + { + Some(records) + } else if let Some(locked_environment) = locked_environment.as_ref() { + locked_environment + .conda_repodata_records_for_platform(platform) + .into_diagnostic() + .context("failed to load conda repodata records from the lock-file")? + } else { + None + }; + for record in repodata_records.into_iter().flatten() { + builder.add_conda_package(environment.name().as_str(), platform, record.into()); + } - // Add the PyPi records - for (platform, packages) in updated_pypi_records.unwrap_or(locked_pypi_records) { - for (pkg_data, pkg_env_data) in packages { - builder.add_pypi_package( - environment.name().as_str(), - platform, - pkg_data, - pkg_env_data, - ); + let pypi_records = if let Some(records) = updated_pypi_records + .remove(&platform) + .or_else(|| loaded_pypi_records.remove(&platform)) + { + Some(records) + } else if let Some(locked_environment) = locked_environment.as_ref() { + locked_environment.pypi_packages_for_platform(platform) + } else { + None + }; + for (pkg_data, env_data) in pypi_records.into_iter().flatten() { + builder.add_pypi_package( + environment.name().as_str(), + platform, + pkg_data, + env_data, + ); + } } } @@ -330,12 +424,13 @@ pub async fn get_up_to_date_prefix<'p>( .context("failed to write updated lock-file to disk")?; } - Ok(prefix) + Prefix::new(prefix_env.dir()) } #[allow(clippy::too_many_arguments)] // TODO: refactor args into struct pub async fn update_prefix_pypi( + name: &str, prefix: &Prefix, platform: Platform, package_db: &PackageDb, @@ -350,7 +445,7 @@ pub async fn update_prefix_pypi( // Install and/or remove python packages progress::await_in_progress( - "updating python packages", + format!("updating pypi packages in '{0}' environment", name), install_pypi::update_python_distributions( package_db, prefix, @@ -421,6 +516,7 @@ impl PythonStatus { /// Updates the environment to contain the packages from the specified lock-file pub async fn update_prefix_conda( + name: &str, prefix: &Prefix, authenticated_client: ClientWithMiddleware, installed_packages: Vec, @@ -440,7 +536,7 @@ pub async fn update_prefix_conda( if !transaction.operations.is_empty() { // Execute the operations that are returned by the solver. progress::await_in_progress( - "updating environment", + format!("updating packages in '{0}' environment", name), install::execute_transaction( &transaction, &installed_packages, diff --git a/src/lock_file/mod.rs b/src/lock_file/mod.rs index 7466a5df5..55c5d7f6b 100644 --- a/src/lock_file/mod.rs +++ b/src/lock_file/mod.rs @@ -6,11 +6,12 @@ mod satisfiability; use crate::{progress, Project}; use futures::TryStreamExt; use futures::{stream, StreamExt}; +use indexmap::IndexMap; use indicatif::ProgressBar; use itertools::{izip, Itertools}; use miette::{Context, IntoDiagnostic}; use rattler_conda_types::{ - GenericVirtualPackage, MatchSpec, PackageName, Platform, RepoDataRecord, + Channel, GenericVirtualPackage, MatchSpec, PackageName, Platform, RepoDataRecord, }; use rattler_lock::{ LockFile, PackageHashes, PypiPackageData, PypiPackageDataRef, PypiPackageEnvironmentData, @@ -18,12 +19,15 @@ use rattler_lock::{ use rattler_repodata_gateway::sparse::SparseRepoData; use rattler_solve::{resolvo, SolverImpl}; use rip::resolve::SDistResolution; +use std::borrow::Cow; use std::collections::HashMap; use std::path::Path; use std::{sync::Arc, time::Duration}; use crate::project::Environment; -pub use satisfiability::lock_file_satisfies_project; +pub use satisfiability::{ + lock_file_satisfies_project, verify_environment_satisfiability, EnvironmentUnsat, +}; /// A list of conda packages that are locked for a specific platform. pub type LockedCondaPackages = Vec; @@ -59,7 +63,7 @@ pub async fn load_lock_file(project: &Project) -> miette::Result { } } -fn main_progress_bar(num_bars: u64, message: &'static str) -> ProgressBar { +fn main_progress_bar(num_bars: u64, message: impl Into>) -> ProgressBar { let multi_progress = progress::global_multi_progress(); let top_level_progress = multi_progress.add(ProgressBar::new(num_bars)); top_level_progress.set_style(progress::long_running_progress_style()); @@ -90,21 +94,15 @@ fn platform_solve_bars(platforms: impl IntoIterator) -> Vec, existing_lock_file: &LockedCondaEnvironment, - repodata: Option>, + repodata: &Arc>, ) -> miette::Result { let platforms = environment.platforms(); - // Get the repodata for the project - let sparse_repo_data: Arc<[_]> = if let Some(sparse_repo_data) = repodata { - sparse_repo_data - } else { - environment.fetch_sparse_repodata().await? - } - .into(); - // Construct a progress bar, a main one and one for each platform. - let _top_level_progress = - main_progress_bar(platforms.len() as u64, "resolving conda dependencies"); + let _top_level_progress = main_progress_bar( + platforms.len() as u64, + format!("resolving conda dependencies for '{0}'", environment.name()), + ); let solve_bars = platform_solve_bars(platforms.iter().copied()); let result = stream::iter(platforms.iter().zip(solve_bars.iter().cloned())) @@ -119,13 +117,13 @@ pub async fn update_lock_file_conda( ); let existing_lock_file = &existing_lock_file; - let sparse_repo_data = sparse_repo_data.clone(); + let sparse_repo_data = repodata.clone(); async move { let empty_vec = vec![]; let result = resolve_platform( environment, existing_lock_file.get(platform).unwrap_or(&empty_vec), - sparse_repo_data.clone(), + &sparse_repo_data, *platform, pb.clone(), ) @@ -166,8 +164,10 @@ pub async fn update_lock_file_for_pypi( let platforms = environment.platforms().into_iter().collect_vec(); // Construct the progress bars - let _top_level_progress = - main_progress_bar(platforms.len() as u64, "resolving pypi dependencies"); + let _top_level_progress = main_progress_bar( + platforms.len() as u64, + format!("resolving pypi dependencies for '{0}'", environment.name()), + ); let solve_bars = platform_solve_bars(platforms.iter().copied()); // Extract conda packages per platform. @@ -302,7 +302,7 @@ async fn resolve_pypi( async fn resolve_platform( environment: &Environment<'_>, existing_lock_file: &LockedCondaPackages, - sparse_repo_data: Arc<[SparseRepoData]>, + sparse_repo_data: &Arc>, platform: Platform, pb: ProgressBar, ) -> miette::Result { @@ -320,8 +320,13 @@ async fn resolve_platform( // Get the repodata for the current platform and for NoArch pb.set_message("loading repodata"); - let available_packages = - load_sparse_repo_data_async(platform, package_names.clone(), sparse_repo_data).await?; + let available_packages = load_sparse_repo_data_async( + platform, + environment.channels().into_iter().cloned().collect(), + package_names.clone(), + sparse_repo_data.clone(), + ) + .await?; // Solve conda packages pb.set_message("resolving conda"); @@ -368,16 +373,19 @@ async fn resolve_conda_dependencies( /// is a CPU and IO intensive task so we run it in a blocking task to not block the main task. async fn load_sparse_repo_data_async( platform: Platform, + channels: Vec, package_names: Vec, - sparse_repo_data: Arc<[SparseRepoData]>, + sparse_repo_data: Arc>, ) -> miette::Result>> { tokio::task::spawn_blocking(move || { - let platform_sparse_repo_data = sparse_repo_data.iter().filter(|sparse| { - sparse.subdir() == platform.as_str() || sparse.subdir() == Platform::NoArch.as_str() - }); + let sparse_repo_data = channels + .iter() + .cloned() + .cartesian_product([platform, Platform::NoArch]) + .filter_map(|target| sparse_repo_data.get(&target)); // Load only records we need for this platform - SparseRepoData::load_records_recursive(platform_sparse_repo_data, package_names, None) + SparseRepoData::load_records_recursive(sparse_repo_data, package_names, None) .into_diagnostic() }) .await diff --git a/src/lock_file/satisfiability.rs b/src/lock_file/satisfiability.rs index d2e12d573..c0b2e96aa 100644 --- a/src/lock_file/satisfiability.rs +++ b/src/lock_file/satisfiability.rs @@ -24,40 +24,42 @@ pub enum Unsat { #[derive(Debug, Error, Diagnostic)] pub enum EnvironmentUnsat { - #[error("the environment is not present in the lock-file")] + #[error("there are no recorded packages for the environment in the lock-file")] Missing, - #[error("channels mismatch")] + #[error("the channels in the lock-file do not match the environments channels")] ChannelsMismatch, - #[error("{0} is unsatisfiable")] + #[error("the packages for '{0}' do not satisfy the requirements of the environment")] PlatformUnsatisfiable(Platform, #[source] PlatformUnsat), } #[derive(Debug, Error, Diagnostic)] pub enum PlatformUnsat { - #[error("could not satisfy '{0}' (required by '{1}')")] + #[error("the requirement '{0}' could not be satisfied (required by '{1}')")] UnsatisfiableMatchSpec(MatchSpec, String), - #[error("could not satisfy '{0}' (required by '{1}')")] + #[error("the requirement '{0}' could not be satisfied (required by '{1}')")] UnsatisfiableRequirement(Requirement, String), - #[error("found a duplicate entry for '{0}'")] + #[error("there was a duplicate entry for '{0}'")] DuplicateEntry(String), - #[error("failed to parse requirement '{0}'")] + #[error("the requirement '{0}' failed to parse")] FailedToParseMatchSpec(String, #[source] ParseMatchSpecError), - #[error("too many conda packages in the lock-file")] + #[error("there are more conda packages in the lock-file than are used by the environment")] TooManyCondaPackages, - #[error("too many pypi packages in the lock-file")] + #[error("there are more pypi packages in the lock-file than are used by the environment")] TooManyPypiPackages, #[error("there are PyPi dependencies but a python interpreter is missing from the lock-file")] MissingPythonInterpreter, - #[error("failed to determine marker environment from the python interpreter in the lock-file")] + #[error( + "a marker environment could not be derived from the python interpreter in the lock-file" + )] FailedToDetermineMarkerEnvironment(#[source] Box), #[error("{0} requires python version {1} but the python interpreter in the lock-file has version {2}")] diff --git a/src/project/environment.rs b/src/project/environment.rs index 00f21602d..2bed90d7e 100644 --- a/src/project/environment.rs +++ b/src/project/environment.rs @@ -8,6 +8,7 @@ use crate::{task::Task, Project}; use indexmap::{IndexMap, IndexSet}; use itertools::{Either, Itertools}; use rattler_conda_types::{Channel, Platform}; +use std::hash::{Hash, Hasher}; use std::{ borrow::Cow, collections::{HashMap, HashSet}, @@ -44,6 +45,20 @@ impl Debug for Environment<'_> { } } +impl Hash for Environment<'_> { + fn hash(&self, state: &mut H) { + self.environment.name.hash(state) + } +} + +impl<'p> PartialEq for Environment<'p> { + fn eq(&self, other: &Self) -> bool { + self.environment.name == other.environment.name + } +} + +impl Eq for Environment<'_> {} + impl<'p> Environment<'p> { /// Returns the project this environment belongs to. pub fn project(&self) -> &'p Project { diff --git a/src/repodata.rs b/src/repodata.rs index 3a5c8e7d9..bef45bb73 100644 --- a/src/repodata.rs +++ b/src/repodata.rs @@ -1,6 +1,7 @@ use crate::project::Environment; use crate::{config, progress, project::Project}; use futures::{stream, StreamExt, TryStreamExt}; +use indexmap::IndexMap; use indicatif::ProgressBar; use itertools::Itertools; use miette::{Context, IntoDiagnostic}; @@ -10,13 +11,17 @@ use reqwest_middleware::ClientWithMiddleware; use std::{path::Path, time::Duration}; impl Project { // TODO: Remove this function once everything is migrated to the new environment system. - pub async fn fetch_sparse_repodata(&self) -> miette::Result> { + pub async fn fetch_sparse_repodata( + &self, + ) -> miette::Result> { self.default_environment().fetch_sparse_repodata().await } } impl Environment<'_> { - pub async fn fetch_sparse_repodata(&self) -> miette::Result> { + pub async fn fetch_sparse_repodata( + &self, + ) -> miette::Result> { let channels = self.channels(); let platforms = self.platforms(); fetch_sparse_repodata(channels, platforms, self.project().authenticated_client()).await @@ -27,7 +32,7 @@ pub async fn fetch_sparse_repodata( channels: impl IntoIterator, target_platforms: impl IntoIterator, authenticated_client: &ClientWithMiddleware, -) -> miette::Result> { +) -> miette::Result> { let channels = channels.into_iter(); let target_platforms = target_platforms.into_iter().collect_vec(); @@ -47,22 +52,31 @@ pub async fn fetch_sparse_repodata( } } - if fetch_targets.is_empty() { - return Ok(vec![]); + fetch_sparse_repodata_targets(fetch_targets, authenticated_client).await +} + +pub async fn fetch_sparse_repodata_targets( + fetch_targets: impl IntoIterator, + authenticated_client: &ClientWithMiddleware, +) -> miette::Result> { + let mut fetch_targets = fetch_targets.into_iter().peekable(); + if fetch_targets.peek().is_none() { + return Ok(IndexMap::new()); } // Construct a top-level progress bar let multi_progress = progress::global_multi_progress(); - let top_level_progress = multi_progress.add(ProgressBar::new(fetch_targets.len() as u64)); + let top_level_progress = + multi_progress.add(ProgressBar::new(fetch_targets.size_hint().0 as u64)); top_level_progress.set_style(progress::long_running_progress_style()); - top_level_progress.set_message("fetching latest repodata"); + top_level_progress.set_message("fetching package metadata"); top_level_progress.enable_steady_tick(Duration::from_millis(50)); let repodata_cache_path = config::get_cache_dir()?.join("repodata"); let multi_progress = progress::global_multi_progress(); let mut progress_bars = Vec::new(); - let repo_data = stream::iter(fetch_targets.into_iter()) + let repo_data = stream::iter(fetch_targets) .map(|(channel, platform)| { // Construct a progress bar for the fetch let progress_bar = multi_progress.add( @@ -80,7 +94,7 @@ pub async fn fetch_sparse_repodata( async move { let result = fetch_repo_data_records_with_progress( - channel, + channel.clone(), platform, &repodata_cache, download_client, @@ -91,12 +105,12 @@ pub async fn fetch_sparse_repodata( top_level_progress.tick(); - result + result.map(|repo_data| repo_data.map(|repo_data| ((channel, platform), repo_data))) } }) .buffered(20) .filter_map(|result| async move { result.transpose() }) - .try_collect::>() + .try_collect() .await; // Clear all the progressbars together