PoC: Implement jj git sync - v1

This is not meant to be merged in this form.
I am bashing everything on the head with a large bat, until it works.
Once everything works nicely in this draft, I will break it up into
smaller tested pieces that are intended to be merged.

* Grab current heads and build a set of [(Parent, Child)...]
* Start a transaction.
* Fetch in the transaction.
* Grab current heads on transaction:
  * Transaction MutableRepo should be updated after fetch.
  * Build a Map where old heads are the keys and the new heads are values.
    * If old == new, skip it.
    * This way if nothing changed, we end up with an empty map and avoid bugs
      downstream (index.is_ancestor returns true if old == new).
  * Relationship is figured out by index.is_ancestor(old_head_id, new_head_id) in a loop.
* Check if rebase is needed.
  * old_heads.set_diffence(new_heads) gives old heads that are no longer heads, which would
    mean, their descendants need to be rebased.
* Find children needing rebase:
  * Build list of rebase specs with: (commit, old_parent, new_parent)
  * IMPORTANT: Not YET handling parent commits deleted in remotes.
* Perform simple rebase which abandons emptied commits.


Part of: #1039

cli/src/commands/git/mod.rs

@@ -20,6 +20,7 @@ pub mod init;
 pub mod push;
 pub mod remote;
 pub mod submodule;
+pub mod sync;
 
 use clap::Subcommand;
 
@@ -39,6 +40,8 @@ use self::remote::cmd_git_remote;
 use self::remote::RemoteCommand;
 use self::submodule::cmd_git_submodule;
 use self::submodule::GitSubmoduleCommand;
+use self::sync::cmd_git_sync;
+use self::sync::GitSyncArgs;
 use crate::cli_util::CommandHelper;
 use crate::cli_util::WorkspaceCommandHelper;
 use crate::command_error::user_error_with_message;
@@ -61,6 +64,7 @@ pub enum GitCommand {
     Remote(RemoteCommand),
     #[command(subcommand, hide = true)]
     Submodule(GitSubmoduleCommand),
+    Sync(GitSyncArgs),
 }
 
 pub fn cmd_git(
@@ -77,6 +81,7 @@ pub fn cmd_git(
         GitCommand::Push(args) => cmd_git_push(ui, command, args),
         GitCommand::Remote(args) => cmd_git_remote(ui, command, args),
         GitCommand::Submodule(args) => cmd_git_submodule(ui, command, args),
+        GitCommand::Sync(args) => cmd_git_sync(ui, command, args),
     }
 }
 

cli/src/commands/git/sync.rs

@@ -0,0 +1,300 @@
+use std::collections::BTreeMap;
+use std::collections::BTreeSet;
+use std::fmt;
+
+use itertools::Itertools;
+//use jj_lib::backend::CommitId;
+use jj_lib::commit::Commit;
+use jj_lib::repo::Repo;
+use jj_lib::revset::RevsetExpression;
+use jj_lib::revset::RevsetIteratorExt;
+use jj_lib::rewrite::EmptyBehaviour;
+use jj_lib::rewrite::RebaseOptions;
+use jj_lib::str_util::StringPattern;
+
+use crate::cli_util::short_change_hash;
+use crate::cli_util::short_commit_hash;
+use crate::cli_util::CommandHelper;
+use crate::cli_util::WorkspaceCommandHelper;
+use crate::cli_util::WorkspaceCommandTransaction;
+use crate::commands::CommandError;
+use crate::git_util::get_fetch_remotes;
+use crate::git_util::get_git_repo;
+use crate::git_util::git_fetch;
+use crate::git_util::FetchArgs;
+use crate::rebase_util::rebase_commits;
+use crate::rebase_util::RebaseArgs;
+use crate::ui::Ui;
+
+/// Sync the local JJ repo to specified Git remote branch(es).
+///
+/// The sync command will first fetch from the Git remote, then
+/// rebase all local changes onto the appropriate updated
+/// heads that were fetched.
+///
+/// Changes that are made empty by the rebase are dropped.
+#[derive(clap::Args, Clone, Debug)]
+pub struct GitSyncArgs {
+    #[command(flatten)]
+    fetch: FetchArgs,
+}
+
+pub fn cmd_git_sync(
+    ui: &mut Ui,
+    command: &CommandHelper,
+    args: &GitSyncArgs,
+) -> Result<(), CommandError> {
+    let mut workspace_command = command.workspace_helper(ui)?;
+
+    let git_repo = get_git_repo(workspace_command.repo().store())?;
+    let remotes = get_fetch_remotes(ui, command.settings(), &git_repo, &args.fetch)?;
+
+    let remote_patterns = remotes
+        .iter()
+        .map(|p| StringPattern::Exact(p.to_string()))
+        .collect_vec();
+
+    let current_heads = get_current_heads(&workspace_command, &args.fetch.branch)?;
+    let pairs = list_local_commits_starting_from(
+        &workspace_command,
+        &current_heads.iter().map(|c| c.clone()).collect_vec(),
+        &args.fetch.branch,
+        &remote_patterns,
+    )?;
+
+    // prep to git fetch
+    let mut tx = workspace_command.start_transaction();
+    git_fetch(
+        ui,
+        &mut tx,
+        &git_repo,
+        &FetchArgs {
+            branch: args.fetch.branch.clone(),
+            remotes: remotes.clone(),
+            all_remotes: args.fetch.all_remotes,
+        },
+    )?;
+
+    // find rebase targets
+    let old_to_new = get_updated_heads(
+        &tx,
+        &current_heads.iter().map(|c| c.clone()).collect_vec(),
+        &args.fetch.branch,
+    )?;
+
+    let remote_heads = old_to_new.values().cloned().collect::<BTreeSet<_>>();
+
+    for commit in &remote_heads {
+        let commit = format_commit(commit);
+        writeln!(ui.status(), "NewHead: {commit}")?;
+    }
+
+    let needs_rebase = if remote_heads.is_empty() {
+        vec![]
+    } else {
+        current_heads.difference(&remote_heads).collect_vec()
+    };
+
+    if needs_rebase.is_empty() {
+        writeln!(ui.status(), "Rebase not needed")?;
+        return Ok(());
+    }
+
+    let rebase_specs = pairs
+        .iter()
+        .filter_map(|pair| {
+            if let Some(parent) = &pair.parent {
+                if needs_rebase.contains(&parent) {
+                    if let Some(new) = old_to_new.get(&parent) {
+                        Some(RebaseSpec {
+                            commit: pair.child.clone(),
+                            old_parent: parent.clone(),
+                            new_parent: new.clone(),
+                        })
+                    } else {
+                        None
+                    }
+                } else {
+                    None
+                }
+            } else {
+                None
+            }
+        })
+        .collect_vec();
+
+    for spec in &rebase_specs {
+        writeln!(ui.status(), "{spec}")?;
+    }
+
+    // // rebase revisions
+    rebase_commits(
+        ui,
+        &mut tx,
+        &RebaseArgs {
+            roots: rebase_specs
+                .iter()
+                .map(|spec| spec.commit.clone())
+                .collect::<Vec<_>>(),
+            new_parents: rebase_specs
+                .iter()
+                .map(|spec| spec.new_parent.clone())
+                .collect::<Vec<_>>(),
+            new_children: vec![],
+            options: RebaseOptions {
+                empty: EmptyBehaviour::AbandonNewlyEmpty,
+                simplify_ancestor_merge: true,
+            },
+        },
+    )?;
+
+    //tx.finish(ui, format!("sync completed"))?;
+
+    Ok(())
+}
+
+fn get_current_heads(
+    workspace_command: &WorkspaceCommandHelper,
+    branches: &[StringPattern],
+) -> Result<BTreeSet<Commit>, CommandError> {
+    let mut commits: BTreeSet<Commit> = BTreeSet::from([]);
+
+    for branch in branches {
+        let mut branch_commits: BTreeSet<Commit> = RevsetExpression::bookmarks(branch.clone())
+            .evaluate_programmatic(workspace_command.repo().as_ref())?
+            .iter()
+            .commits(workspace_command.repo().store())
+            .try_collect()?;
+
+        commits.append(&mut branch_commits);
+    }
+
+    Ok(commits)
+}
+
+fn get_updated_heads(
+    tx: &WorkspaceCommandTransaction,
+    old_heads: &[Commit],
+    branches: &[StringPattern],
+) -> Result<BTreeMap<Commit, Commit>, CommandError> {
+    let mut updated_heads: Vec<Commit> = vec![];
+
+    for branch in branches {
+        let mut branch_commits: Vec<Commit> = RevsetExpression::bookmarks(branch.clone())
+            .evaluate_programmatic(tx.repo())?
+            .iter()
+            .commits(tx.repo().store())
+            .try_collect()?;
+        updated_heads.append(&mut branch_commits);
+    }
+
+    let mut out: BTreeMap<Commit, Commit> = BTreeMap::from([]);
+    let index = tx.repo().index();
+    updated_heads.iter().for_each(|new| {
+        old_heads.iter().for_each(|old| {
+            if old != new && index.is_ancestor(old.id(), new.id()) {
+                out.insert(old.clone(), new.clone());
+            }
+        });
+    });
+
+    Ok(out)
+}
+
+fn format_commit(commit: &Commit) -> String {
+    let change_hash = short_change_hash(commit.change_id());
+    let commit_hash = short_commit_hash(commit.id());
+    let description = if let Some(description) = commit.description().split("\n").next() {
+        description
+    } else {
+        "<empty description>"
+    };
+
+    format!("{change_hash} <{description}> {commit_hash}")
+}
+
+pub struct CommitPair {
+    parent: Option<Commit>,
+    child: Commit,
+}
+
+impl fmt::Display for CommitPair {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        let parent = if let Some(parent) = &self.parent {
+            format_commit(parent)
+        } else {
+            // trunk
+            "ROOT".to_string()
+        };
+
+        let child = format_commit(&self.child);
+
+        write!(f, "{parent} => {child}")
+    }
+}
+
+pub struct RebaseSpec {
+    commit: Commit,
+    old_parent: Commit,
+    new_parent: Commit,
+}
+
+impl fmt::Display for RebaseSpec {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        let commit = short_change_hash(self.commit.change_id());
+        let old = short_change_hash(self.old_parent.change_id());
+        let new = short_change_hash(self.new_parent.change_id());
+        write!(f, "Commit {commit} will move from {old} --> {new}")
+    }
+}
+
+fn list_local_commits_starting_from(
+    workspace_command: &WorkspaceCommandHelper,
+    start: &[Commit],
+    branches: &[StringPattern],
+    remotes: &[StringPattern],
+) -> Result<Vec<CommitPair>, CommandError> {
+    let mut pairs = start
+        .iter()
+        .map(|commit| CommitPair {
+            parent: None,
+            child: commit.clone(),
+        })
+        .collect_vec();
+
+    let start = start.iter().map(|c| c.id().clone()).collect_vec();
+
+    for remote in remotes {
+        for branch in branches {
+            let commits: Vec<Commit> = RevsetExpression::commits(start.to_vec())
+                .descendants()
+                .minus(&RevsetExpression::commits(start.to_vec()))
+                .minus(&RevsetExpression::remote_bookmarks(
+                    branch.clone(),
+                    remote.clone(),
+                    None,
+                ))
+                .evaluate_programmatic(workspace_command.repo().as_ref())?
+                .iter()
+                .commits(workspace_command.repo().store())
+                .try_collect()?;
+
+            for commit in commits {
+                let parents: Vec<Commit> = RevsetExpression::commits(commit.parent_ids().to_vec())
+                    .evaluate_programmatic(workspace_command.repo().as_ref())?
+                    .iter()
+                    .commits(workspace_command.repo().store())
+                    .try_collect()?;
+
+                for parent in parents {
+                    pairs.push(CommitPair {
+                        parent: Some(parent),
+                        child: commit.clone(),
+                    })
+                }
+            }
+        }
+    }
+
+    Ok(pairs)
+}

cli/src/lib.rs

@@ -30,6 +30,7 @@ pub mod merge_tools;
 pub mod movement_util;
 pub mod operation_templater;
 mod progress;
+pub mod rebase_util;
 pub mod revset_util;
 pub mod template_builder;
 pub mod template_parser;