kvserver: introduce and disable eager replicate queue on span cfg

Replicas were enqueued into the replicate queue, upon the store
receiving a span config update which could affect the replica. The
replicate queue `shouldQueue` is relatively more expensive than other
queues.

Introduce the cluster setting
`kv.eager_replicate_enqueue_on_span_config_update.enabled`, which when
set to true, enables queuing up replicas on span config updates; when
set to false, disables queuing replicas on span config updates.

By default, this settings is set to false.

Resolves: cockroachdb#108724
Release note: None

pkg/kv/kvserver/replicate_queue.go

@@ -102,6 +102,17 @@ var MinLeaseTransferInterval = settings.RegisterDurationSetting(
 	settings.NonNegativeDuration,
 )
 
+// EagerReplicateEnqueueOnSpanConfigUpdateEnabled controls whether replicas are
+// enqueued into the replicate queue, following a span config update which
+// affects the replica.
+var EagerReplicateEnqueueOnSpanConfigUpdateEnabled = settings.RegisterBoolSetting(
+	settings.SystemOnly,
+	"kv.eager_replicate_enqueue_on_span_config_update.enabled",
+	"controls whether replicas are enqueued into the replicate queue for "+
+		"processing, when a span config update occurs, which affects the replica",
+	false,
+)
+
 var (
 	metaReplicateQueueAddReplicaCount = metric.Metadata{
 		Name:        "queue.replicate.addreplica",

pkg/kv/kvserver/store.go

@@ -2444,9 +2444,15 @@ func (s *Store) onSpanConfigUpdate(ctx context.Context, updated roachpb.Span) {
 			s.mergeQueue.Async(replCtx, "span config update", true /* wait */, func(ctx context.Context, h queueHelper) {
 				h.MaybeAdd(ctx, repl, now)
 			})
-			s.replicateQueue.Async(replCtx, "span config update", true /* wait */, func(ctx context.Context, h queueHelper) {
-				h.MaybeAdd(ctx, repl, now)
-			})
+
+			// The replicate queue has a relatively more expensive queue check
+			// (shouldQueue), because it scales with the number of stores, and
+			// performs more checks.
+			if EagerReplicateEnqueueOnSpanConfigUpdateEnabled.Get(&s.GetStoreConfig().Settings.SV) {
+				s.replicateQueue.Async(replCtx, "span config update", true /* wait */, func(ctx context.Context, h queueHelper) {
+					h.MaybeAdd(ctx, repl, now)
+				})
+			}
 			return nil // more
 		},
 	); err != nil {