Merge branch 'main' into feat/migrate-to-strict-version

.github/workflows/python-bindings.yml

@@ -0,0 +1,37 @@
+name: Python bindings CI
+
+on:
+  push:
+    branches: [ $default-branch ]
+    paths:
+      - 'py-rattler/**/*'
+      - '.github/workflows/python-bindings.yml'
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  format_lint_test:
+    name: Format, Lint and Test the Python bindings
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: recursive
+      - run: |
+          curl -fsSL https://pixi.sh/install.sh | bash
+          echo "/home/runner/.pixi/bin" >> $GITHUB_PATH
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          components: clippy, rustfmt
+      - name: Format and Lint
+        run: |
+          cd py-rattler
+          pixi run lint
+          pixi run fmt-check
+      - name: Run tests
+        run: |
+          cd py-rattler
+          pixi run test

.gitignore

@@ -1,5 +1,7 @@
 .idea/
 
+.vscode/
+
 # Generated by Cargo
 # will have compiled files and executables
 debug/

crates/rattler-bin/src/commands/create.rs

@@ -157,13 +157,13 @@ pub async fn create(opt: Opt) -> anyhow::Result<()> {
         .collect::<Result<Vec<_>, _>>()?;
 
     // Get the package names from the matchspecs so we can only load the package records that we need.
-    let package_names = specs.iter().filter_map(|spec| spec.name.as_ref());
+    let package_names = specs.iter().filter_map(|spec| spec.name.as_ref().cloned());
     let repodatas = wrap_in_progress("parsing repodata", move || {
         SparseRepoData::load_records_recursive(
             &sparse_repo_datas,
             package_names,
             Some(|record| {
-                if record.name == "python" {
+                if record.name.as_normalized() == "python" {
                     record.depends.push("pip".to_string());
                 }
             }),
@@ -179,24 +179,26 @@ pub async fn create(opt: Opt) -> anyhow::Result<()> {
                 .iter()
                 .map(|virt_pkg| {
                     let elems = virt_pkg.split('=').collect::<Vec<&str>>();
-                    GenericVirtualPackage {
-                        name: elems[0].to_string(),
+                    Ok(GenericVirtualPackage {
+                        name: elems[0].try_into()?,
                         version: elems
                             .get(1)
                             .map(|s| Version::from_str(s))
                             .unwrap_or(Version::from_str("0"))
                             .expect("Could not parse virtual package version"),
                         build_string: elems.get(2).unwrap_or(&"").to_string(),
-                    }
+                    })
                 })
-                .collect::<Vec<_>>())
+                .collect::<anyhow::Result<Vec<_>>>()?)
         } else {
-            rattler_virtual_packages::VirtualPackage::current().map(|vpkgs| {
-                vpkgs
-                    .iter()
-                    .map(|vpkg| GenericVirtualPackage::from(vpkg.clone()))
-                    .collect::<Vec<_>>()
-            })
+            rattler_virtual_packages::VirtualPackage::current()
+                .map(|vpkgs| {
+                    vpkgs
+                        .iter()
+                        .map(|vpkg| GenericVirtualPackage::from(vpkg.clone()))
+                        .collect::<Vec<_>>()
+                })
+                .map_err(anyhow::Error::from)
         }
     })?;
 
@@ -247,7 +249,9 @@ pub async fn create(opt: Opt) -> anyhow::Result<()> {
         let format_record = |r: &RepoDataRecord| {
             format!(
                 "{} {} {}",
-                r.package_record.name, r.package_record.version, r.package_record.build
+                r.package_record.name.as_normalized(),
+                r.package_record.version,
+                r.package_record.build
             )
         };
 
@@ -492,7 +496,11 @@ async fn install_package_to_environment(
         // Write the conda-meta information
         let pkg_meta_path = conda_meta_path.join(format!(
             "{}-{}-{}.json",
-            prefix_record.repodata_record.package_record.name,
+            prefix_record
+                .repodata_record
+                .package_record
+                .name
+                .as_normalized(),
             prefix_record.repodata_record.package_record.version,
             prefix_record.repodata_record.package_record.build
         ));
@@ -536,7 +544,7 @@ async fn remove_package_from_environment(
     // Remove the conda-meta file
     let conda_meta_path = target_prefix.join("conda-meta").join(format!(
         "{}-{}-{}.json",
-        package.repodata_record.package_record.name,
+        package.repodata_record.package_record.name.as_normalized(),
         package.repodata_record.package_record.version,
         package.repodata_record.package_record.build
     ));
@@ -620,7 +628,7 @@ async fn fetch_repo_data_records_with_progress(
             platform.to_string(),
             repo_data_json_path,
             Some(|record: &mut PackageRecord| {
-                if record.name == "python" {
+                if record.name.as_normalized() == "python" {
                     record.depends.push("pip".to_string());
                 }
             }),

crates/rattler/Cargo.toml

@@ -39,7 +39,7 @@ regex = "1.9.1"
 reqwest = { version = "0.11.18", default-features = false, features = ["stream", "json", "gzip"] }
 serde = { version = "1.0.171", features = ["derive"] }
 serde_json = { version = "1.0.102", features = ["raw_value"] }
-serde_with = "3.0.0"
+serde_with = "3.3.0"
 smallvec = { version = "1.11.0", features = ["serde", "const_new", "const_generics", "union"] }
 tempfile = "3.6.0"
 thiserror = "1.0.43"

crates/rattler/src/install/transaction.rs

@@ -166,7 +166,7 @@ fn find_python_info(
 
 /// Returns true if the specified record refers to Python.
 fn is_python_record(record: &PackageRecord) -> bool {
-    record.name == "python"
+    record.name.as_normalized() == "python"
 }
 
 /// Returns true if the `from` and `to` describe the same package content

crates/rattler/src/package_cache.rs

@@ -56,7 +56,7 @@ impl From<ArchiveIdentifier> for CacheKey {
 impl From<&PackageRecord> for CacheKey {
     fn from(record: &PackageRecord) -> Self {
         Self {
-            name: record.name.to_string(),
+            name: record.name.as_normalized().to_string(),
             version: record.version.to_string(),
             build_string: record.build.to_string(),
         }

crates/rattler_conda_types/Cargo.toml

@@ -14,6 +14,7 @@ readme.workspace = true
 chrono = "0.4.26"
 fxhash = "0.2.1"
 hex = "0.4.3"
+indexmap = { version = "2.0.0", features = ["serde"] }
 itertools = "0.11.0"
 lazy-regex = "3.0.0"
 nom = "7.1.3"
@@ -22,7 +23,7 @@ serde = { version = "1.0.171", features = ["derive"] }
 serde_json = "1.0.102"
 serde-json-python-formatter = "0.1.0"
 serde_yaml = "0.9.22"
-serde_with = "3.0.0"
+serde_with = { version = "3.3.0", features = ["indexmap_2"] }
 serde_repr = "0.1"
 smallvec = { version = "1.11.0", features = ["serde", "const_new", "const_generics", "union"] }
 strum = { version = "0.25.0", features = ["derive"] }

crates/rattler_conda_types/src/channel/mod.rs

@@ -66,13 +66,20 @@ impl Channel {
             Channel::from_url(url, platforms, config)
         } else if is_path(channel) {
             let path = PathBuf::from(channel);
-            let absolute_path = absolute_path(&path);
-            let url = Url::from_directory_path(absolute_path)
-                .map_err(|_| ParseChannelError::InvalidPath(path))?;
-            Self {
-                platforms,
-                base_url: url,
-                name: Some(channel.to_owned()),
+
+            #[cfg(target_arch = "wasm32")]
+            return Err(ParseChannelError::InvalidPath(path));
+
+            #[cfg(not(target_arch = "wasm32"))]
+            {
+                let absolute_path = absolute_path(&path);
+                let url = Url::from_directory_path(absolute_path)
+                    .map_err(|_| ParseChannelError::InvalidPath(path))?;
+                Self {
+                    platforms,
+                    base_url: url,
+                    name: Some(channel.to_owned()),
+                }
             }
         } else {
             Channel::from_name(channel, platforms, config)

crates/rattler_conda_types/src/conda_lock/builder.rs

@@ -6,8 +6,9 @@ use crate::conda_lock::{
     content_hash, Channel, CondaLock, GitMeta, LockMeta, LockedDependency, Manager, PackageHashes,
     TimeMeta,
 };
-use crate::{MatchSpec, NamelessMatchSpec, NoArchType, Platform, RepoDataRecord};
-use fxhash::{FxHashMap, FxHashSet};
+use crate::{MatchSpec, NamelessMatchSpec, NoArchType, PackageName, Platform, RepoDataRecord};
+use fxhash::{FxBuildHasher, FxHashMap, FxHashSet};
+use indexmap::IndexMap;
 use std::str::FromStr;
 use url::Url;
 
@@ -76,7 +77,7 @@ impl LockFileBuilder {
                     content_hash::calculate_content_hash(plat, &self.input_specs, &self.channels)?,
                 ))
             })
-            .collect::<Result<FxHashMap<_, _>, CalculateContentHashError>>()?;
+            .collect::<Result<_, CalculateContentHashError>>()?;
 
         let lock = CondaLock {
             metadata: LockMeta {
@@ -94,7 +95,6 @@ impl LockFileBuilder {
                 .into_values()
                 .flat_map(|package| package.build())
                 .collect(),
-            version: super::default_version(),
         };
         Ok(lock)
     }
@@ -169,7 +169,7 @@ impl LockedPackages {
 /// Short-hand for creating a LockedPackage that transforms into a [`LockedDependency`]
 pub struct LockedPackage {
     /// Name of the locked package
-    pub name: String,
+    pub name: PackageName,
     /// Package version
     pub version: String,
     /// Package build string
@@ -179,7 +179,7 @@ pub struct LockedPackage {
     /// Collection of package hash fields
     pub package_hashes: PackageHashes,
     /// List of dependencies for this package
-    pub dependency_list: FxHashMap<String, NamelessMatchSpec>,
+    pub dependency_list: IndexMap<PackageName, NamelessMatchSpec, FxBuildHasher>,
     /// Check if package is optional
     pub optional: Option<bool>,
 
@@ -236,7 +236,7 @@ impl TryFrom<RepoDataRecord> for LockedPackage {
         let hashes = hashes.ok_or_else(|| ConversionError::Missing("md5 or sha265".to_string()))?;
 
         // Convert dependencies
-        let mut dependencies = FxHashMap::default();
+        let mut dependencies = IndexMap::default();
         for match_spec_str in record.package_record.depends.iter() {
             let matchspec = MatchSpec::from_str(match_spec_str)?;
             let name = matchspec
@@ -245,9 +245,9 @@ impl TryFrom<RepoDataRecord> for LockedPackage {
                 .ok_or_else(|| {
                     ConversionError::Missing(format!("dependency name for {}", match_spec_str))
                 })?
-                .to_string();
+                .clone();
             let version_constraint = NamelessMatchSpec::from(matchspec);
-            dependencies.insert(name, version_constraint);
+            dependencies.insert(name.clone(), version_constraint);
         }
 
         Ok(Self {
@@ -281,20 +281,19 @@ impl LockedPackage {
     }
 
     /// Add a single dependency
-    pub fn add_dependency<S: AsRef<str>>(
+    pub fn add_dependency(
         mut self,
-        key: S,
+        key: PackageName,
         version_constraint: NamelessMatchSpec,
     ) -> Self {
-        self.dependency_list
-            .insert(key.as_ref().to_string(), version_constraint);
+        self.dependency_list.insert(key, version_constraint);
         self
     }
 
     /// Add multiple dependencies
     pub fn add_dependencies(
         mut self,
-        value: impl IntoIterator<Item = (String, NamelessMatchSpec)>,
+        value: impl IntoIterator<Item = (PackageName, NamelessMatchSpec)>,
     ) -> Self {
         self.dependency_list.extend(value);
         self
@@ -385,13 +384,13 @@ impl LockedPackage {
 #[cfg(test)]
 mod tests {
     use chrono::Utc;
-    use fxhash::FxHashMap;
     use std::str::FromStr;
 
     use crate::conda_lock::builder::{LockFileBuilder, LockedPackage, LockedPackages};
     use crate::conda_lock::PackageHashes;
     use crate::{
-        ChannelConfig, MatchSpec, NamelessMatchSpec, NoArchType, Platform, RepoDataRecord,
+        ChannelConfig, MatchSpec, NamelessMatchSpec, NoArchType, PackageName, Platform,
+        RepoDataRecord,
     };
     use rattler_digest::parse_digest_from_hex;
 
@@ -405,13 +404,13 @@ mod tests {
         )
             .add_locked_packages(LockedPackages::new(Platform::Osx64)
                 .add_locked_package(LockedPackage {
-                    name: "python".to_string(),
+                    name: PackageName::new_unchecked("python"),
                     version: "3.11.0".to_string(),
                     build_string: "h4150a38_1_cpython".to_string(),
                     url: "https://conda.anaconda.org/conda-forge/osx-64/python-3.11.0-h4150a38_1_cpython.conda".parse().unwrap(),
                     package_hashes:  PackageHashes::Md5Sha256(parse_digest_from_hex::<rattler_digest::Md5>("c6f4b87020c72e2700e3e94c1fc93b70").unwrap(),
                                                                parse_digest_from_hex::<rattler_digest::Sha256>("7c58de8c7d98b341bd9be117feec64782e704fec5c30f6e14713ebccaab9b5d8").unwrap()),
-                    dependency_list: FxHashMap::from_iter([("python".to_string(), NamelessMatchSpec::from_str("3.11.0.*").unwrap())]),
+                    dependency_list: FromIterator::from_iter([(PackageName::new_unchecked("python"), NamelessMatchSpec::from_str("3.11.0.*").unwrap())]),
                     optional: None,
                     arch: Some("x86_64".to_string()),
                     subdir: Some("noarch".to_string()),

crates/rattler_conda_types/src/conda_lock/content_hash.rs

@@ -58,9 +58,14 @@ pub fn calculate_content_data(
         .iter()
         .map(|spec| {
             Ok(CondaLockVersionedDependency {
-                name: spec.name.clone().ok_or_else(|| {
-                    CalculateContentHashError::RequiredAttributeMissing("name".to_string())
-                })?,
+                name: spec
+                    .name
+                    .clone()
+                    .ok_or_else(|| {
+                        CalculateContentHashError::RequiredAttributeMissing("name".to_string())
+                    })?
+                    .as_source()
+                    .to_string(),
                 manager: "conda".to_string(),
                 optional: false,
                 category: "main".to_string(),