pkg/kv/kvclient: update per-vCPU concurrency limits

Increase the current values (arbitrarily selected) by 6x (also an
arbitrary increase).  Annecdotal evidence from customers suggests
this may unlock some workloads where vCPU utilization appears to
"plateau" around 70%, and now allows for ~100% vCPU utilization.
Until cockroachdb#131125 and cockroachdb#131126 are implemented, it is not possible to
directly monitor these incremental improvements to the defaults.

Epic: CRDB-42388
Epic: CRDB-42389

Release note (performance improvement): Increase the per-vCPU
concurrency limits for KV operations.  Specifically, increase
`kv.dist_sender.concurrency_limit` to 384/vCPU (up from 64/vCPU)
and `kv.streamer.concurrency_limit` to 96/vCPU (up from 8/vCPU).

pkg/kv/kvclient/kvcoord/dist_sender.go

@@ -343,8 +343,10 @@ var CanSendToFollower = func(
 }
 
 const (
-	// The default limit for asynchronous senders.
-	defaultSenderConcurrency = 1024
+	// The default scaling factor for the number of async ops per vCPU.
+	DefaultSenderStreamsPerVCPU = 384
+	// The minimum number of recommended vCPUs.
+	MinViableProcs = 2
 	// RangeLookupPrefetchCount is the maximum number of range descriptors to prefetch
 	// during range lookups.
 	RangeLookupPrefetchCount = 8
@@ -368,7 +370,7 @@ var senderConcurrencyLimit = settings.RegisterIntSetting(
 	settings.ApplicationLevel,
 	"kv.dist_sender.concurrency_limit",
 	"maximum number of asynchronous send requests",
-	max(defaultSenderConcurrency, int64(64*runtime.GOMAXPROCS(0))),
+	DefaultSenderStreamsPerVCPU*max(MinViableProcs, int64(runtime.GOMAXPROCS(0))),
 	settings.NonNegativeInt,
 )
 

pkg/kv/kvclient/kvstreamer/streamer.go

@@ -340,15 +340,20 @@ type streamerStatistics struct {
 	enqueuedSingleRangeRequests int
 }
 
+const (
+	// The default scaling factor for the number of asynchronous requests per
+	// Streamer per vCPU. The value for this setting is chosen arbitrarily as
+	// 1/4th of the default value for the DefaultSenderStreamsPerVCPU.
+	defaultStreamerStreamsPerVCPU = kvcoord.DefaultSenderStreamsPerVCPU / 4
+)
+
 // streamerConcurrencyLimit is an upper bound on the number of asynchronous
-// requests that a single Streamer can have in flight. The default value for
-// this setting is chosen arbitrarily as 1/8th of the default value for the
-// senderConcurrencyLimit.
+// requests that a single Streamer can have in flight.
 var streamerConcurrencyLimit = settings.RegisterIntSetting(
 	settings.ApplicationLevel,
 	"kv.streamer.concurrency_limit",
 	"maximum number of asynchronous requests by a single streamer",
-	max(128, int64(8*runtime.GOMAXPROCS(0))),
+	defaultStreamerStreamsPerVCPU*max(kvcoord.MinViableProcs, int64(runtime.GOMAXPROCS(0))),
 	settings.PositiveInt,
 )