kv: retry AdminSplit on LeaseTransferRejectedBecauseTargetMayNeedSnap…

…shot

Informs cockroachdb#84635
Informs cockroachdb#84162

This commit considers the `LeaseTransferRejectedBecauseTargetMayNeedSnapshot`
status to be a form of a retriable replication change error. It then hooks
`Replica.executeAdminCommandWithDescriptor` up to consult this status in its
retry loop.

This avoids spurious errors when a split gets blocked behind a lateral replica
move like we see in the following situation:
1. issue AdminSplit
2. range in joint config, first needs to leave (maybeLeaveAtomicChangeReplicas)
3. to leave, needs to transfer lease from voter_outgoing to voter_incoming
4. can’t transfer lease because doing so is deemed to be potentially unsafe

Release note: None

pkg/kv/kvserver/markers.go

@@ -32,7 +32,9 @@ var errMarkCanRetryReplicationChangeWithUpdatedDesc = errors.New("should retry w
 // assumed to have been emitted by the KV layer during a replication change
 // operation) is likely to succeed when retried with an updated descriptor.
 func IsRetriableReplicationChangeError(err error) bool {
-	return errors.Is(err, errMarkCanRetryReplicationChangeWithUpdatedDesc) || isSnapshotError(err)
+	return errors.Is(err, errMarkCanRetryReplicationChangeWithUpdatedDesc) ||
+		IsLeaseTransferRejectedBecauseTargetMayNeedSnapshotError(err) ||
+		isSnapshotError(err)
 }
 
 const (

pkg/kv/kvserver/replica_command.go

@@ -559,9 +559,9 @@ func (r *Replica) executeAdminCommandWithDescriptor(
 		}
 
 		lastErr = updateDesc(r.Desc())
-		// On seeing a ConditionFailedError or an AmbiguousResultError, retry the
-		// command with the updated descriptor.
-		if !errors.HasType(lastErr, (*roachpb.ConditionFailedError)(nil)) &&
+		// On seeing a retryable replication change or an AmbiguousResultError,
+		// retry the command with the updated descriptor.
+		if !IsRetriableReplicationChangeError(lastErr) &&
 			!errors.HasType(lastErr, (*roachpb.AmbiguousResultError)(nil)) {
 			break
 		}