copy-tracking: Initial Implementation

Cargo.toml

@@ -51,7 +51,9 @@ git2 = "0.18.3"
 gix = { version = "0.63.0", default-features = false, features = [
     "index",
     "max-performance-safe",
+    "blob-diff",
 ] }
+gix-filter = "0.11.2"
 glob = "0.3.1"
 hex = "0.4.3"
 ignore = "0.4.20"

cli/Cargo.toml

@@ -65,6 +65,7 @@ esl01-renderdag = { workspace = true }
 futures = { workspace = true }
 git2 = { workspace = true }
 gix = { workspace = true }
+gix-filter = { workspace = true }
 hex = { workspace = true }
 indexmap = { workspace = true }
 itertools = { workspace = true }

cli/examples/custom-backend/main.rs

@@ -18,17 +18,18 @@ use std::path::Path;
 use std::time::SystemTime;
 
 use async_trait::async_trait;
+use futures::stream::BoxStream;
 use jj_cli::cli_util::{CliRunner, CommandHelper};
 use jj_cli::command_error::CommandError;
 use jj_cli::ui::Ui;
 use jj_lib::backend::{
     Backend, BackendInitError, BackendLoadError, BackendResult, ChangeId, Commit, CommitId,
-    Conflict, ConflictId, FileId, SigningFn, SymlinkId, Tree, TreeId,
+    Conflict, ConflictId, CopyRecord, FileId, SigningFn, SymlinkId, Tree, TreeId,
 };
 use jj_lib::git_backend::GitBackend;
 use jj_lib::index::Index;
 use jj_lib::repo::StoreFactories;
-use jj_lib::repo_path::RepoPath;
+use jj_lib::repo_path::{RepoPath, RepoPathBuf};
 use jj_lib::settings::UserSettings;
 use jj_lib::signing::Signer;
 use jj_lib::workspace::{Workspace, WorkspaceInitError};
@@ -174,6 +175,15 @@ impl Backend for JitBackend {
         self.inner.write_commit(contents, sign_with)
     }
 
+    fn get_copy_records(
+        &self,
+        paths: &[RepoPathBuf],
+        roots: &[CommitId],
+        heads: &[CommitId],
+    ) -> BackendResult<BoxStream<BackendResult<CopyRecord>>> {
+        self.inner.get_copy_records(paths, roots, heads)
+    }
+
     fn gc(&self, index: &dyn Index, keep_newer: SystemTime) -> BackendResult<()> {
         self.inner.gc(index, keep_newer)
     }

cli/src/commands/debug/copy_detection.rs

@@ -0,0 +1,79 @@
+// Copyright 2024 The Jujutsu Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(unused, dead_code)]
+
+use std::fmt::Debug;
+use std::io::Write as _;
+use std::path::{Path, PathBuf};
+
+use futures::executor::block_on_stream;
+use futures::StreamExt;
+use jj_lib::backend::{Backend, CopyRecord, CopySource, CopySources};
+use jj_lib::default_index::{AsCompositeIndex as _, DefaultIndexStore};
+use jj_lib::op_walk;
+use jj_lib::repo_path::{RepoPath, RepoPathBuf};
+
+use crate::cli_util::{CommandHelper, RevisionArg};
+use crate::command_error::{internal_error, user_error, CommandError};
+use crate::ui::Ui;
+
+/// Rebuild commit index
+#[derive(clap::Args, Clone, Debug)]
+pub struct CopyDetectionArgs {
+    /// Show changes in this revision, compared to its parent(s)
+    #[arg(default_value = "@")]
+    revision: RevisionArg,
+}
+
+pub fn cmd_debug_copy_detection(
+    ui: &mut Ui,
+    command: &CommandHelper,
+    args: &CopyDetectionArgs,
+) -> Result<(), CommandError> {
+    let ws = command.workspace_helper(ui)?;
+    let Some(git) = ws.git_backend() else {
+        writeln!(ui.stderr(), "Not a git backend.")?;
+        return Ok(());
+    };
+    let commit = ws.resolve_single_rev(&args.revision)?;
+    let tree = commit.tree()?;
+
+    let paths: Vec<RepoPathBuf> = tree.entries().map(|(path, _)| path).collect();
+    let commits = [commit.id().clone()];
+    let parents = commit.parent_ids();
+    for copy_record in
+        block_on_stream(git.get_copy_records(&paths, parents, &commits)?).filter_map(|r| r.ok())
+    {
+        match copy_record.sources {
+            CopySources::Resolved(CopySource { path, .. }) => {
+                write!(ui.stdout(), "{}", path.as_internal_file_string());
+            }
+            CopySources::Conflict(conflicting) => {
+                let mut sorted: Vec<_> = conflicting
+                    .iter()
+                    .map(|s| s.path.as_internal_file_string())
+                    .collect();
+                sorted.sort();
+                write!(ui.stdout(), "{{ {} }}", sorted.join(", "));
+            }
+        }
+        writeln!(
+            ui.stdout(),
+            " -> {}",
+            copy_record.target.as_internal_file_string()
+        );
+    }
+    Ok(())
+}

cli/src/commands/debug/mod.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+pub mod copy_detection;
 pub mod fileset;
 pub mod index;
 pub mod local_working_copy;
@@ -30,6 +31,7 @@ use std::fmt::Debug;
 use clap::Subcommand;
 use jj_lib::local_working_copy::LocalWorkingCopy;
 
+use self::copy_detection::{cmd_debug_copy_detection, CopyDetectionArgs};
 use self::fileset::{cmd_debug_fileset, DebugFilesetArgs};
 use self::index::{cmd_debug_index, DebugIndexArgs};
 use self::local_working_copy::{cmd_debug_local_working_copy, DebugLocalWorkingCopyArgs};
@@ -49,6 +51,7 @@ use crate::ui::Ui;
 #[derive(Subcommand, Clone, Debug)]
 #[command(hide = true)]
 pub enum DebugCommand {
+    CopyDetection(CopyDetectionArgs),
     Fileset(DebugFilesetArgs),
     Index(DebugIndexArgs),
     LocalWorkingCopy(DebugLocalWorkingCopyArgs),
@@ -75,6 +78,7 @@ pub fn cmd_debug(
         DebugCommand::LocalWorkingCopy(args) => cmd_debug_local_working_copy(ui, command, args),
         DebugCommand::Operation(args) => cmd_debug_operation(ui, command, args),
         DebugCommand::Reindex(args) => cmd_debug_reindex(ui, command, args),
+        DebugCommand::CopyDetection(args) => cmd_debug_copy_detection(ui, command, args),
         DebugCommand::Revset(args) => cmd_debug_revset(ui, command, args),
         DebugCommand::Snapshot(args) => cmd_debug_snapshot(ui, command, args),
         DebugCommand::Template(args) => cmd_debug_template(ui, command, args),

cli/tests/runner.rs

@@ -19,6 +19,7 @@ mod test_commit_command;
 mod test_commit_template;
 mod test_concurrent_operations;
 mod test_config_command;
+mod test_copy_detection;
 mod test_debug_command;
 mod test_describe_command;
 mod test_diff_command;

cli/tests/test_copy_detection.rs

@@ -0,0 +1,34 @@
+// Copyright 2022 The Jujutsu Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::common::TestEnvironment;
+
+#[test]
+fn test_simple_rename() {
+    let test_env = TestEnvironment::default();
+    test_env.jj_cmd_ok(test_env.env_root(), &["git", "init", "repo"]);
+    let repo_path = test_env.env_root().join("repo");
+
+    test_env.jj_cmd_ok(&repo_path, &["new"]);
+    std::fs::write(repo_path.join("original"), "original").unwrap();
+    std::fs::write(repo_path.join("something"), "something").unwrap();
+    test_env.jj_cmd_ok(&repo_path, &["commit", "-mfirst"]);
+    std::fs::remove_file(repo_path.join("original")).unwrap();
+    std::fs::write(repo_path.join("modified"), "original").unwrap();
+    std::fs::write(repo_path.join("something"), "changed").unwrap();
+    insta::assert_snapshot!(test_env.jj_cmd_success(&repo_path, &["debug", "copy-detection"]).replace('\\', "/"),
+    @r###"
+    original -> modified
+    "###);
+}

lib/Cargo.toml

@@ -44,6 +44,7 @@ either = { workspace = true }
 futures = { workspace = true }
 git2 = { workspace = true, optional = true }
 gix = { workspace = true, optional = true }
+gix-filter = { workspace = true, optional = true }
 glob = { workspace = true }
 hex = { workspace = true }
 ignore = { workspace = true }
@@ -92,7 +93,7 @@ tokio = { workspace = true, features = ["full"] }
 
 [features]
 default = ["git"]
-git = ["dep:git2", "dep:gix"]
+git = ["dep:git2", "dep:gix", "dep:gix-filter"]
 vendored-openssl = ["git2/vendored-openssl"]
 watchman = ["dep:tokio", "dep:watchman_client"]
 testing = ["git"]

lib/src/backend.rs

@@ -15,12 +15,13 @@
 #![allow(missing_docs)]
 
 use std::any::Any;
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, HashSet};
 use std::fmt::Debug;
 use std::io::Read;
 use std::time::SystemTime;
 
 use async_trait::async_trait;
+use futures::stream::BoxStream;
 use thiserror::Error;
 
 use crate::content_hash::ContentHash;
@@ -152,6 +153,47 @@ pub struct Conflict {
     pub adds: Vec<ConflictTerm>,
 }
 
+/// An individual copy source.
+#[derive(Debug, PartialEq, Eq, Hash, Clone)]
+pub struct CopySource {
+    /// The source path a target was copied from.
+    ///
+    /// It is not required that the source path is different than the target
+    /// path. A custom backend may choose to represent 'rollbacks' as copies
+    /// from a file unto itself, from a specific prior commit.
+    pub path: RepoPathBuf,
+    pub file: FileId,
+    /// The source commit the target was copied from. If not specified, then the
+    /// parent of the target commit is the source commit. Backends may use this
+    /// field to implement 'integration' logic, where a source may be
+    /// periodically merged into a target, similar to a branch, but the
+    /// branching occurs at the file level rather than the repository level. It
+    /// also follows naturally that any copy source targeted to a specific
+    /// commit should avoid copy propagation on rebasing, which is desirable
+    /// for 'fork' style copies.
+    ///
+    /// If specified, it is required that the commit id is an ancestor of the
+    /// commit with which this copy source is associated.
+    pub commit: Option<CommitId>,
+}
+
+#[derive(Debug, PartialEq, Eq, Clone)]
+pub enum CopySources {
+    Resolved(CopySource),
+    Conflict(HashSet<CopySource>),
+}
+
+/// An individual copy event, from file A -> B.
+#[derive(Debug, PartialEq, Eq, Clone)]
+pub struct CopyRecord {
+    /// The destination of the copy, B.
+    pub target: RepoPathBuf,
+    /// The CommitId where the copy took place.
+    pub id: CommitId,
+    /// The source of the copy, A.
+    pub sources: CopySources,
+}
+
 /// Error that may occur during backend initialization.
 #[derive(Debug, Error)]
 #[error(transparent)]
@@ -416,6 +458,24 @@ pub trait Backend: Send + Sync + Debug {
         sign_with: Option<&mut SigningFn>,
     ) -> BackendResult<(CommitId, Commit)>;
 
+    /// Get all copy records for `paths` in the dag range `roots..heads`.
+    ///
+    /// The exact order these are returned is unspecified, but it is guaranteed
+    /// to be reverse-topological. That is, for any two copy records with
+    /// different commit ids A and B, if A is an ancestor of B, A is streamed
+    /// after B.
+    ///
+    /// Streaming by design to better support large backends which may have very
+    /// large single-file histories. This also allows more iterative algorithms
+    /// like blame/annotate to short-circuit after a point without wasting
+    /// unnecessary resources.
+    fn get_copy_records(
+        &self,
+        paths: &[RepoPathBuf],
+        roots: &[CommitId],
+        heads: &[CommitId],
+    ) -> BackendResult<BoxStream<BackendResult<CopyRecord>>>;
+
     /// Perform garbage collection.
     ///
     /// All commits found in the `index` won't be removed. In addition to that,