kvflowcontrol: set kvadmission.flow_control.mode default to apply_to_all

With testing now in place, this commit changes the default cluster
setting value of `kvadmission.flow_control.mode` from
`apply_to_elastic`, to `apply_to_all`.

Now by default, regular work will be subject to replication admission
control, whereby a quorum will be allowed to proceed, queuing entries
to any non-quorum required replica, when send tokens are exhausted.

For more details, see cockroachdb#123509.

Resolves: cockroachdb#133838
Release note (performance improvement): Regular writes are now subject
to admission control by default, meaning that non-quorum required
replicas may not be told about new writes from the leader if they are
unable to keep up. This brings a large performance improvement during
instances where there is a large backlog of replication work towards a
subset of node(s), such as node restarts. The setting can be reverted to
the <=24.3 default by setting `kvadmission.flow_control.mode` to
`apply_to_elastic`.

docs/generated/metrics/metrics.html

@@ -19,6 +19,7 @@
 <tr><td>STORAGE</td><td>admission.admitted.kv-stores.high-pri</td><td>Number of requests admitted</td><td>Requests</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
 <tr><td>STORAGE</td><td>admission.admitted.kv-stores.locking-normal-pri</td><td>Number of requests admitted</td><td>Requests</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
 <tr><td>STORAGE</td><td>admission.admitted.kv-stores.normal-pri</td><td>Number of requests admitted</td><td>Requests</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
+<tr><td>STORAGE</td><td>admission.admitted.kv-stores.user-high-pri</td><td>Number of requests admitted</td><td>Requests</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
 <tr><td>STORAGE</td><td>admission.admitted.kv.high-pri</td><td>Number of requests admitted</td><td>Requests</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
 <tr><td>STORAGE</td><td>admission.admitted.kv.locking-normal-pri</td><td>Number of requests admitted</td><td>Requests</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
 <tr><td>STORAGE</td><td>admission.admitted.kv.normal-pri</td><td>Number of requests admitted</td><td>Requests</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
@@ -54,6 +55,7 @@
 <tr><td>STORAGE</td><td>admission.errored.kv-stores.high-pri</td><td>Number of requests not admitted due to error</td><td>Requests</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
 <tr><td>STORAGE</td><td>admission.errored.kv-stores.locking-normal-pri</td><td>Number of requests not admitted due to error</td><td>Requests</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
 <tr><td>STORAGE</td><td>admission.errored.kv-stores.normal-pri</td><td>Number of requests not admitted due to error</td><td>Requests</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
+<tr><td>STORAGE</td><td>admission.errored.kv-stores.user-high-pri</td><td>Number of requests not admitted due to error</td><td>Requests</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
 <tr><td>STORAGE</td><td>admission.errored.kv.high-pri</td><td>Number of requests not admitted due to error</td><td>Requests</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
 <tr><td>STORAGE</td><td>admission.errored.kv.locking-normal-pri</td><td>Number of requests not admitted due to error</td><td>Requests</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
 <tr><td>STORAGE</td><td>admission.errored.kv.normal-pri</td><td>Number of requests not admitted due to error</td><td>Requests</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
@@ -101,6 +103,7 @@
 <tr><td>STORAGE</td><td>admission.requested.kv-stores.high-pri</td><td>Number of requests</td><td>Requests</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
 <tr><td>STORAGE</td><td>admission.requested.kv-stores.locking-normal-pri</td><td>Number of requests</td><td>Requests</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
 <tr><td>STORAGE</td><td>admission.requested.kv-stores.normal-pri</td><td>Number of requests</td><td>Requests</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
+<tr><td>STORAGE</td><td>admission.requested.kv-stores.user-high-pri</td><td>Number of requests</td><td>Requests</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
 <tr><td>STORAGE</td><td>admission.requested.kv.high-pri</td><td>Number of requests</td><td>Requests</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
 <tr><td>STORAGE</td><td>admission.requested.kv.locking-normal-pri</td><td>Number of requests</td><td>Requests</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
 <tr><td>STORAGE</td><td>admission.requested.kv.normal-pri</td><td>Number of requests</td><td>Requests</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
@@ -128,6 +131,7 @@
 <tr><td>STORAGE</td><td>admission.wait_durations.kv-stores.high-pri</td><td>Wait time durations for requests that waited</td><td>Wait time Duration</td><td>HISTOGRAM</td><td>NANOSECONDS</td><td>AVG</td><td>NONE</td></tr>
 <tr><td>STORAGE</td><td>admission.wait_durations.kv-stores.locking-normal-pri</td><td>Wait time durations for requests that waited</td><td>Wait time Duration</td><td>HISTOGRAM</td><td>NANOSECONDS</td><td>AVG</td><td>NONE</td></tr>
 <tr><td>STORAGE</td><td>admission.wait_durations.kv-stores.normal-pri</td><td>Wait time durations for requests that waited</td><td>Wait time Duration</td><td>HISTOGRAM</td><td>NANOSECONDS</td><td>AVG</td><td>NONE</td></tr>
+<tr><td>STORAGE</td><td>admission.wait_durations.kv-stores.user-high-pri</td><td>Wait time durations for requests that waited</td><td>Wait time Duration</td><td>HISTOGRAM</td><td>NANOSECONDS</td><td>AVG</td><td>NONE</td></tr>
 <tr><td>STORAGE</td><td>admission.wait_durations.kv.high-pri</td><td>Wait time durations for requests that waited</td><td>Wait time Duration</td><td>HISTOGRAM</td><td>NANOSECONDS</td><td>AVG</td><td>NONE</td></tr>
 <tr><td>STORAGE</td><td>admission.wait_durations.kv.locking-normal-pri</td><td>Wait time durations for requests that waited</td><td>Wait time Duration</td><td>HISTOGRAM</td><td>NANOSECONDS</td><td>AVG</td><td>NONE</td></tr>
 <tr><td>STORAGE</td><td>admission.wait_durations.kv.normal-pri</td><td>Wait time durations for requests that waited</td><td>Wait time Duration</td><td>HISTOGRAM</td><td>NANOSECONDS</td><td>AVG</td><td>NONE</td></tr>
@@ -155,6 +159,7 @@
 <tr><td>STORAGE</td><td>admission.wait_queue_length.kv-stores.high-pri</td><td>Length of wait queue</td><td>Requests</td><td>GAUGE</td><td>COUNT</td><td>AVG</td><td>NONE</td></tr>
 <tr><td>STORAGE</td><td>admission.wait_queue_length.kv-stores.locking-normal-pri</td><td>Length of wait queue</td><td>Requests</td><td>GAUGE</td><td>COUNT</td><td>AVG</td><td>NONE</td></tr>
 <tr><td>STORAGE</td><td>admission.wait_queue_length.kv-stores.normal-pri</td><td>Length of wait queue</td><td>Requests</td><td>GAUGE</td><td>COUNT</td><td>AVG</td><td>NONE</td></tr>
+<tr><td>STORAGE</td><td>admission.wait_queue_length.kv-stores.user-high-pri</td><td>Length of wait queue</td><td>Requests</td><td>GAUGE</td><td>COUNT</td><td>AVG</td><td>NONE</td></tr>
 <tr><td>STORAGE</td><td>admission.wait_queue_length.kv.high-pri</td><td>Length of wait queue</td><td>Requests</td><td>GAUGE</td><td>COUNT</td><td>AVG</td><td>NONE</td></tr>
 <tr><td>STORAGE</td><td>admission.wait_queue_length.kv.locking-normal-pri</td><td>Length of wait queue</td><td>Requests</td><td>GAUGE</td><td>COUNT</td><td>AVG</td><td>NONE</td></tr>
 <tr><td>STORAGE</td><td>admission.wait_queue_length.kv.normal-pri</td><td>Length of wait queue</td><td>Requests</td><td>GAUGE</td><td>COUNT</td><td>AVG</td><td>NONE</td></tr>

pkg/kv/kvserver/kvflowcontrol/kvflowcontrol.go

@@ -44,8 +44,8 @@ var Mode = settings.RegisterEnumSetting(
 	"determines the 'mode' of flow control we use for replication traffic in KV, if enabled",
 	metamorphic.ConstantWithTestChoice(
 		"kvadmission.flow_control.mode",
-		modeDict[ApplyToElastic], /* default value */
-		modeDict[ApplyToAll],     /* other value */
+		modeDict[ApplyToAll],     /* default value */
+		modeDict[ApplyToElastic], /* other value */
 	),
 	modeDict,
 )