Adds support for locking with source references (#540)

CHANGELOG.md

@@ -13,6 +13,10 @@ _Unreleased_
 
 - Detect when a virtualenv relocates and automatically re-create it on sync.  #538
 
+- Added `lock --with-sources`, `sync --with-sources` and the new `rye.tool.lock-with-sources`
+  config.  Passing this will ensure that source references are included in the
+  lock files.  #540
+
 <!-- released start -->
 
 ## 0.17.0

docs/guide/pyproject.md

@@ -60,6 +60,20 @@ pulled in as indirect dependencies.  These are added here automatically with `ry
 excluded-dependencies = ["cffi"]
 ```
 
+## `tool.rye.lock-with-sources`
+
++++ 0.18.0
+
+When this flag is enabled all `lock` and `sync` operations in the project or workspace
+operate as if `--with-sources` is passed.  This means that all lock files contain the
+full source references.  Note that this can create lock files that contain credentials
+if the sources have credentials included in the URL.
+
+```toml
+[tool.rye]
+lock-with-sources = true
+```
+
 ## `tool.rye.managed`
 
 +++ 0.3.0

docs/guide/sources.md

@@ -114,4 +114,4 @@ can be supplied with environment variables.
     [[sources]]
     name = "company-internal"
     url = "https://${INDEX_USERNAME}:${INDEX_PASSWORD}@company.internal/simple/"
-    ```
+    ```

docs/guide/sync.md

@@ -55,6 +55,23 @@ do want to include those, pass `--pre`
 rye lock Flask --pre
 ```
 
+### `--with-sources`
+
++++ 0.18.0
+
+By default (unless the `tool.rye.lock-with-sources` config key is set to `true` in the
+`pyproject.toml`) lock files are not generated with source references.  This means that
+if custom sources are used the lock file cannot be installed via `pip` unless also
+`--find-links` and other parameters are manually passed.  This can be particularly useful
+when the lock file is used for docker image builds.
+
+When this flag is passed then the lock file is generated with references to `--index-url`,
+`--extra-index-url` or `--find-links`.
+
+```
+rye lock --with-sources
+```
+
 ## Sync
 
 Syncing takes the same parameters as `lock` and then some.  Sync will usually first do what

rye/src/cli/lock.rs

@@ -31,6 +31,9 @@ pub struct Args {
     /// Enables all features.
     #[arg(long)]
     all_features: bool,
+    /// Set to true to lock with sources in the lockfile.
+    #[arg(long)]
+    with_sources: bool,
     /// Use this pyproject.toml file
     #[arg(long, value_name = "PYPROJECT_TOML")]
     pyproject: Option<PathBuf>,
@@ -47,6 +50,7 @@ pub fn execute(cmd: Args) -> Result<(), Error> {
             pre: cmd.pre,
             features: cmd.features,
             all_features: cmd.all_features,
+            with_sources: cmd.with_sources,
         },
         pyproject: cmd.pyproject,
         ..SyncOptions::default()

rye/src/cli/sync.rs

@@ -40,6 +40,9 @@ pub struct Args {
     /// Enables all features.
     #[arg(long)]
     all_features: bool,
+    /// Set to true to lock with sources in the lockfile.
+    #[arg(long)]
+    with_sources: bool,
     /// Use this pyproject.toml file
     #[arg(long, value_name = "PYPROJECT_TOML")]
     pyproject: Option<PathBuf>,
@@ -63,6 +66,7 @@ pub fn execute(cmd: Args) -> Result<(), Error> {
             pre: cmd.pre,
             features: cmd.features,
             all_features: cmd.all_features,
+            with_sources: cmd.with_sources,
         },
         pyproject: cmd.pyproject,
     })?;

rye/src/lock.rs

@@ -30,6 +30,7 @@ static REQUIREMENTS_HEADER: &str = r#"# generated by rye
 #   pre: {{ lock_options.pre }}
 #   features: {{ lock_options.features }}
 #   all-features: {{ lock_options.all_features }}
+#   with-sources: {{ lock_options.with_sources }}
 
 "#;
 
@@ -65,6 +66,8 @@ pub struct LockOptions {
     pub features: Vec<String>,
     /// Enable all features in the workspace.
     pub all_features: bool,
+    /// Should locking happen with sources?
+    pub with_sources: bool,
 }
 
 /// Creates lockfiles for all projects in the workspace.
@@ -360,6 +363,7 @@ fn generate_lockfile(
         lockfile,
         workspace_path,
         exclusions,
+        sources,
         lock_options,
     )?;
 
@@ -371,14 +375,20 @@ fn finalize_lockfile(
     out: &Path,
     workspace_root: &Path,
     exclusions: &HashSet<Requirement>,
+    sources: &ExpandedSources,
     lock_options: &LockOptions,
 ) -> Result<(), Error> {
     let mut rv = BufWriter::new(fs::File::create(out)?);
     writeln!(rv, "{}", render!(REQUIREMENTS_HEADER, lock_options))?;
+
+    // only if we are asked to include sources we do that.
+    if lock_options.with_sources {
+        sources.add_to_lockfile(&mut rv)?;
+        writeln!(rv)?;
+    }
+
     for line in fs::read_to_string(generated)?.lines() {
-        // we do not want to persist these pieces of information as we always
-        // provide it explicitly on the command line.  This is particularly
-        // important as we might include auth info here.
+        // we deal with this explicitly.
         if line.trim().is_empty()
             || line.starts_with("--index-url ")
             || line.starts_with("--extra-index-url ")

rye/src/pyproject.rs

@@ -494,6 +494,11 @@ impl Workspace {
     pub fn rye_managed(&self) -> bool {
         is_rye_managed(&self.doc)
     }
+
+    /// Should requirements.txt based locking include a find-links reference?
+    pub fn lock_with_sources(&self) -> bool {
+        lock_with_sources(&self.doc)
+    }
 }
 
 /// Check if recurse should be skipped into directory with this name
@@ -947,6 +952,14 @@ impl PyProject {
         }
     }
 
+    /// Should requirements.txt based locking include a find-links reference?
+    pub fn lock_with_sources(&self) -> bool {
+        match self.workspace {
+            Some(ref workspace) => workspace.lock_with_sources(),
+            None => lock_with_sources(&self.doc),
+        }
+    }
+
     /// Save back changes
     pub fn save(&self) -> Result<(), Error> {
         fs::write(self.toml_path(), self.doc.to_string()).with_context(|| {
@@ -1194,6 +1207,14 @@ fn is_rye_managed(doc: &Document) -> bool {
         .unwrap_or(false)
 }
 
+fn lock_with_sources(doc: &Document) -> bool {
+    doc.get("tool")
+        .and_then(|x| x.get("rye"))
+        .and_then(|x| x.get("lock-with-sources"))
+        .and_then(|x| x.as_bool())
+        .unwrap_or(false)
+}
+
 fn get_project_metadata(path: &Path) -> Result<Metadata, Error> {
     let self_venv = ensure_self_venv(CommandOutput::Normal)?;
     let mut metadata = Command::new(self_venv.join(VENV_BIN).join("python"));
@@ -1208,7 +1229,7 @@ fn get_project_metadata(path: &Path) -> Result<Metadata, Error> {
 /// Represents expanded sources.
 #[derive(Debug, Clone, Serialize)]
 pub struct ExpandedSources {
-    pub index_urls: Vec<Url>,
+    pub index_urls: Vec<(Url, bool)>,
     pub find_links: Vec<Url>,
     pub trusted_hosts: HashSet<String>,
 }
@@ -1228,7 +1249,7 @@ impl ExpandedSources {
                 }
             }
             match source.ty {
-                SourceRefType::Index => index_urls.push(url),
+                SourceRefType::Index => index_urls.push((url, source.name == "default")),
                 SourceRefType::FindLinks => find_links.push(url),
             }
         }
@@ -1242,8 +1263,8 @@ impl ExpandedSources {
 
     /// Attach common pip args to a command.
     pub fn add_as_pip_args(&self, cmd: &mut Command) {
-        for (idx, url) in self.index_urls.iter().enumerate() {
-            if idx == 0 {
+        for (url, default) in self.index_urls.iter() {
+            if *default {
                 cmd.arg("--index-url");
             } else {
                 cmd.arg("--extra-index-url");
@@ -1259,6 +1280,24 @@ impl ExpandedSources {
             cmd.arg(host);
         }
     }
+
+    /// Write the sources to a lockfile.
+    pub fn add_to_lockfile(&self, out: &mut dyn std::io::Write) -> std::io::Result<()> {
+        for (url, default) in self.index_urls.iter() {
+            if *default {
+                writeln!(out, "--index-url {}", url)?;
+            } else {
+                writeln!(out, "--extra-index-url {}", url)?;
+            }
+        }
+        for link in &self.find_links {
+            writeln!(out, "--find-links {}", link)?;
+        }
+        for host in &self.trusted_hosts {
+            writeln!(out, "--trusted-host {}", host)?;
+        }
+        Ok(())
+    }
 }
 
 #[derive(ValueEnum, Copy, Clone, Serialize, Debug, PartialEq)]

rye/src/sync.rs

@@ -76,7 +76,7 @@ pub struct VenvMarker {
 }
 
 /// Synchronizes a project's virtualenv.
-pub fn sync(cmd: SyncOptions) -> Result<(), Error> {
+pub fn sync(mut cmd: SyncOptions) -> Result<(), Error> {
     let pyproject = PyProject::load_or_discover(cmd.pyproject.as_deref())?;
     let lockfile = pyproject.workspace_path().join("requirements.lock");
     let dev_lockfile = pyproject.workspace_path().join("requirements-dev.lock");
@@ -92,6 +92,11 @@ pub fn sync(cmd: SyncOptions) -> Result<(), Error> {
         bail!("cannot sync or generate lockfile: package needs 'pyproject.toml'");
     }
 
+    // Turn on locking with sources if the project demands it.
+    if pyproject.lock_with_sources() {
+        cmd.lock_options.with_sources = true;
+    }
+
     // ensure we are bootstrapped
     let self_venv = ensure_self_venv(output).context("could not sync because bootstrap failed")?;
 
@@ -261,15 +266,6 @@ pub fn sync(cmd: SyncOptions) -> Result<(), Error> {
 
             sources.add_as_pip_args(&mut pip_sync_cmd);
 
-            for (idx, url) in sources.index_urls.iter().enumerate() {
-                if idx == 0 {
-                    pip_sync_cmd.arg("--index-url");
-                } else {
-                    pip_sync_cmd.arg("--extra-index-url");
-                }
-                pip_sync_cmd.arg(&url.to_string());
-            }
-
             if cmd.dev && dev_lockfile.is_file() {
                 pip_sync_cmd.arg(&dev_lockfile);
             } else {