diff --git a/lib/src/refs.rs b/lib/src/refs.rs
index ee0c33669f..c183086680 100644
--- a/lib/src/refs.rs
+++ b/lib/src/refs.rs
@@ -14,11 +14,39 @@
 
 #![allow(missing_docs)]
 
+use itertools::EitherOrBoth;
+
 use crate::backend::CommitId;
 use crate::index::Index;
 use crate::merge::{trivial_merge, Merge};
 use crate::op_store::{BranchTarget, RefTarget, RefTargetOptionExt};
 
+/// Compares `refs1` and `refs2` targets, yields entry if they differ.
+///
+/// `refs1` and `refs2` must be sorted by `K`.
+pub fn diff_named_refs<'a, 'b, K: Ord>(
+    refs1: impl IntoIterator<Item = (K, &'a RefTarget)>,
+    refs2: impl IntoIterator<Item = (K, &'b RefTarget)>,
+) -> impl Iterator<Item = (K, (&'a RefTarget, &'b RefTarget))> {
+    iter_named_ref_pairs(refs1, refs2).filter(|(_, (target1, target2))| target1 != target2)
+}
+
+/// Iterates `refs1` and `refs2` target pairs by name.
+///
+/// `refs1` and `refs2` must be sorted by `K`.
+fn iter_named_ref_pairs<'a, 'b, K: Ord>(
+    refs1: impl IntoIterator<Item = (K, &'a RefTarget)>,
+    refs2: impl IntoIterator<Item = (K, &'b RefTarget)>,
+) -> impl Iterator<Item = (K, (&'a RefTarget, &'b RefTarget))> {
+    itertools::merge_join_by(refs1, refs2, |(name1, _), (name2, _)| name1.cmp(name2)).map(|entry| {
+        match entry {
+            EitherOrBoth::Both((name, target1), (_, target2)) => (name, (target1, target2)),
+            EitherOrBoth::Left((name, target1)) => (name, (target1, RefTarget::absent_ref())),
+            EitherOrBoth::Right((name, target2)) => (name, (RefTarget::absent_ref(), target2)),
+        }
+    })
+}
+
 pub fn merge_ref_targets(
     index: &dyn Index,
     left: &RefTarget,
diff --git a/lib/src/repo.rs b/lib/src/repo.rs
index 88880d0cb7..2705ff8f37 100644
--- a/lib/src/repo.rs
+++ b/lib/src/repo.rs
@@ -44,7 +44,7 @@ use crate::local_backend::LocalBackend;
 use crate::op_heads_store::{self, OpHeadResolutionError, OpHeadsStore};
 use crate::op_store::{BranchTarget, OpStore, OpStoreError, OperationId, RefTarget, WorkspaceId};
 use crate::operation::Operation;
-use crate::refs::merge_ref_targets;
+use crate::refs::{diff_named_refs, merge_ref_targets};
 use crate::revset::{self, ChangeIdIndex, Revset, RevsetExpression};
 use crate::rewrite::DescendantRebaser;
 use crate::settings::{RepoSettings, UserSettings};
@@ -1102,54 +1102,24 @@ impl MutableRepo {
             self.view_mut().add_head(added_head);
         }
 
-        let mut maybe_changed_ref_names = HashSet::new();
-
-        let base_branches: HashSet<_> = base.branches().keys().cloned().collect();
-        let other_branches: HashSet<_> = other.branches().keys().cloned().collect();
-        for branch_name in base_branches.union(&other_branches) {
-            let base_branch = base.get_branch(branch_name);
-            let other_branch = other.get_branch(branch_name);
-            if other_branch == base_branch {
-                // Unchanged on other side
-                continue;
-            }
-
-            maybe_changed_ref_names.insert(RefName::LocalBranch(branch_name.clone()));
-            if let Some(branch) = base_branch {
-                for remote in branch.remote_targets.keys() {
-                    maybe_changed_ref_names.insert(RefName::RemoteBranch {
-                        branch: branch_name.clone(),
-                        remote: remote.clone(),
-                    });
-                }
-            }
-            if let Some(branch) = other_branch {
-                for remote in branch.remote_targets.keys() {
-                    maybe_changed_ref_names.insert(RefName::RemoteBranch {
-                        branch: branch_name.clone(),
-                        remote: remote.clone(),
-                    });
+        let changed_refs = itertools::chain!(
+            diff_named_refs(base.local_branches(), other.local_branches())
+                .map(|(name, diff)| (RefName::LocalBranch(name.to_owned()), diff)),
+            diff_named_refs(base.remote_branches(), other.remote_branches()).map(
+                |((branch, remote), diff)| {
+                    let ref_name = RefName::RemoteBranch {
+                        branch: branch.to_owned(),
+                        remote: remote.to_owned(),
+                    };
+                    (ref_name, diff)
                 }
-            }
-        }
-
-        for tag_name in base.tags().keys() {
-            maybe_changed_ref_names.insert(RefName::Tag(tag_name.clone()));
-        }
-        for tag_name in other.tags().keys() {
-            maybe_changed_ref_names.insert(RefName::Tag(tag_name.clone()));
-        }
-
-        for git_ref_name in base.git_refs().keys() {
-            maybe_changed_ref_names.insert(RefName::GitRef(git_ref_name.clone()));
-        }
-        for git_ref_name in other.git_refs().keys() {
-            maybe_changed_ref_names.insert(RefName::GitRef(git_ref_name.clone()));
-        }
-
-        for ref_name in maybe_changed_ref_names {
-            let base_target = base.get_ref(&ref_name);
-            let other_target = other.get_ref(&ref_name);
+            ),
+            diff_named_refs(base.tags(), other.tags())
+                .map(|(name, diff)| (RefName::Tag(name.to_owned()), diff)),
+            diff_named_refs(base.git_refs(), other.git_refs())
+                .map(|(name, diff)| (RefName::GitRef(name.to_owned()), diff)),
+        );
+        for (ref_name, (base_target, other_target)) in changed_refs {
             self.view.get_mut().merge_single_ref(
                 self.index.as_index(),
                 &ref_name,