kv: don't leak raft application tracing spans on ErrRemoved

Fixes cockroachdb#53677.

This commit ensures that we properly finish tracing spans of
Raft commands that throw ErrRemoved errors in ApplySideEffects.

I was originally intending to do something more dramatic and make
`replicaStateMachine.ApplySideEffects` responsible for acknowledging
proposers in all cases, but doing so turned out to be pretty invasive
so I was concerned that it would be harder to backport to v20.2 and
to v20.1. I may revisit that in the future.

Release justification: low risk, high benefit changes to existing
functionality.

pkg/kv/kvserver/replica_application_cmd.go

@@ -155,13 +155,27 @@ func (c *replicatedCmd) AckSuccess() error {
 
 // FinishAndAckOutcome implements the apply.AppliedCommand interface.
 func (c *replicatedCmd) FinishAndAckOutcome(ctx context.Context) error {
-	tracing.FinishSpan(c.sp)
 	if c.IsLocal() {
 		c.proposal.finishApplication(ctx, c.response)
 	}
+	c.finishTracingSpan()
 	return nil
 }
 
+// FinishNonLocal is like AckOutcomeAndFinish, but instead of acknowledging the
+// command's proposal if it is local, it asserts that the proposal is not local.
+func (c *replicatedCmd) FinishNonLocal(ctx context.Context) {
+	if c.IsLocal() {
+		log.Fatalf(ctx, "proposal unexpectedly local: %v", c.replicatedResult())
+	}
+	c.finishTracingSpan()
+}
+
+func (c *replicatedCmd) finishTracingSpan() {
+	tracing.FinishSpan(c.sp)
+	c.ctx, c.sp = nil, nil
+}
+
 // decode decodes the entry e into the decodedRaftEntry.
 func (d *decodedRaftEntry) decode(ctx context.Context, e *raftpb.Entry) error {
 	*d = decodedRaftEntry{}

pkg/kv/kvserver/replica_application_state_machine.go

@@ -1044,8 +1044,10 @@ func (sm *replicaStateMachine) ApplySideEffects(
 	clearTrivialReplicatedEvalResultFields(cmd.replicatedResult())
 	if !cmd.IsTrivial() {
 		shouldAssert, isRemoved := sm.handleNonTrivialReplicatedEvalResult(ctx, cmd.replicatedResult())
-
 		if isRemoved {
+			// The proposal must not have been local, because we don't allow a
+			// proposing replica to remove itself from the Range.
+			cmd.FinishNonLocal(ctx)
 			return nil, apply.ErrRemoved
 		}
 		// NB: Perform state assertion before acknowledging the client.