Merge pull request #105950 from wenyihu6/backport23.1-105924

release-23.1: allocator: expose LeaseRebalanceThreshold as a cluster setting

docs/generated/settings/settings.html

@@ -45,6 +45,7 @@
 <tr><td><div id="setting-feature-schema-change-enabled" class="anchored"><code>feature.schema_change.enabled</code></div></td><td>boolean</td><td><code>true</code></td><td>set to true to enable schema changes, false to disable; default is true</td></tr>
 <tr><td><div id="setting-feature-stats-enabled" class="anchored"><code>feature.stats.enabled</code></div></td><td>boolean</td><td><code>true</code></td><td>set to true to enable CREATE STATISTICS/ANALYZE, false to disable; default is true</td></tr>
 <tr><td><div id="setting-jobs-retention-time" class="anchored"><code>jobs.retention_time</code></div></td><td>duration</td><td><code>336h0m0s</code></td><td>the amount of time for which records for completed jobs are retained</td></tr>
+<tr><td><div id="setting-kv-allocator-lease-rebalance-threshold" class="anchored"><code>kv.allocator.lease_rebalance_threshold</code></div></td><td>float</td><td><code>0.05</code></td><td>minimum fraction away from the mean a store&#39;s lease count can be before it is considered for lease-transfers</td></tr>
 <tr><td><div id="setting-kv-allocator-load-based-lease-rebalancing-enabled" class="anchored"><code>kv.allocator.load_based_lease_rebalancing.enabled</code></div></td><td>boolean</td><td><code>true</code></td><td>set to enable rebalancing of range leases based on load and latency</td></tr>
 <tr><td><div id="setting-kv-allocator-load-based-rebalancing" class="anchored"><code>kv.allocator.load_based_rebalancing</code></div></td><td>enumeration</td><td><code>leases and replicas</code></td><td>whether to rebalance based on the distribution of load across stores [off = 0, leases = 1, leases and replicas = 2]</td></tr>
 <tr><td><div id="setting-kv-allocator-load-based-rebalancing-objective" class="anchored"><code>kv.allocator.load_based_rebalancing.objective</code></div></td><td>enumeration</td><td><code>cpu</code></td><td>what objective does the cluster use to rebalance; if set to `qps` the cluster will attempt to balance qps among stores, if set to `cpu` the cluster will attempt to balance cpu usage among stores [qps = 0, cpu = 1]</td></tr>

pkg/kv/kvserver/allocator/allocatorimpl/allocator.go

@@ -46,18 +46,26 @@ const (
 // to the mean range/lease count that permits lease-transfers away from that
 // store.
 // Made configurable for the sake of testing.
-var LeaseRebalanceThreshold = 0.05
+var LeaseRebalanceThreshold = settings.RegisterFloatSetting(
+	settings.SystemOnly,
+	"kv.allocator.lease_rebalance_threshold",
+	"minimum fraction away from the mean a store's lease count can be before "+
+		"it is considered for lease-transfers",
+	0.05,
+).WithPublic()
 
 // LeaseRebalanceThresholdMin is the absolute number of leases above/below the
 // mean lease count that a store can have before considered overfull/underfull.
 // Made configurable for the sake of testing.
 var LeaseRebalanceThresholdMin = 5.0
 
-// baseLoadBasedLeaseRebalanceThreshold is the equivalent of
+// getBaseLoadBasedLeaseRebalanceThreshold returns the equivalent of
 // LeaseRebalanceThreshold for load-based lease rebalance decisions (i.e.
 // "follow-the-workload"). It's the base threshold for decisions that get
 // adjusted based on the load and latency of the involved ranges/nodes.
-var baseLoadBasedLeaseRebalanceThreshold = 2 * LeaseRebalanceThreshold
+func getBaseLoadBasedLeaseRebalanceThreshold(leaseRebalanceThreshold float64) float64 {
+	return 2 * leaseRebalanceThreshold
+}
 
 // MinLeaseTransferStatsDuration configures the minimum amount of time a
 // replica must wait for stats about request counts to accumulate before
@@ -2560,6 +2568,7 @@ func (a Allocator) FollowTheWorkloadPrefersLocal(
 		return false
 	}
 	adjustment := adjustments[candidate]
+	baseLoadBasedLeaseRebalanceThreshold := getBaseLoadBasedLeaseRebalanceThreshold(LeaseRebalanceThreshold.Get(&a.st.SV))
 	if adjustment > baseLoadBasedLeaseRebalanceThreshold {
 		log.KvDistribution.VEventf(ctx, 3,
 			"s%d is a better fit than s%d due to follow-the-workload (score: %.2f; threshold: %.2f)",
@@ -2732,7 +2741,7 @@ func loadBasedLeaseRebalanceScore(
 	// Start with twice the base rebalance threshold in order to fight more
 	// strongly against thrashing caused by small variances in the distribution
 	// of request weights.
-	rebalanceThreshold := baseLoadBasedLeaseRebalanceThreshold - rebalanceAdjustment
+	rebalanceThreshold := getBaseLoadBasedLeaseRebalanceThreshold(LeaseRebalanceThreshold.Get(&st.SV)) - rebalanceAdjustment
 
 	overfullLeaseThreshold := int32(math.Ceil(meanLeases * (1 + rebalanceThreshold)))
 	overfullScore := source.Capacity.LeaseCount - overfullLeaseThreshold
@@ -2767,7 +2776,8 @@ func (a Allocator) shouldTransferLeaseForLeaseCountConvergence(
 
 	// Allow lease transfer if we're above the overfull threshold, which is
 	// mean*(1+LeaseRebalanceThreshold).
-	overfullLeaseThreshold := int32(math.Ceil(sl.CandidateLeases.Mean * (1 + LeaseRebalanceThreshold)))
+	leaseRebalanceThreshold := LeaseRebalanceThreshold.Get(&a.st.SV)
+	overfullLeaseThreshold := int32(math.Ceil(sl.CandidateLeases.Mean * (1 + leaseRebalanceThreshold)))
 	minOverfullThreshold := int32(math.Ceil(sl.CandidateLeases.Mean + LeaseRebalanceThresholdMin))
 	if overfullLeaseThreshold < minOverfullThreshold {
 		overfullLeaseThreshold = minOverfullThreshold
@@ -2777,7 +2787,7 @@ func (a Allocator) shouldTransferLeaseForLeaseCountConvergence(
 	}
 
 	if float64(source.Capacity.LeaseCount) > sl.CandidateLeases.Mean {
-		underfullLeaseThreshold := int32(math.Ceil(sl.CandidateLeases.Mean * (1 - LeaseRebalanceThreshold)))
+		underfullLeaseThreshold := int32(math.Ceil(sl.CandidateLeases.Mean * (1 - leaseRebalanceThreshold)))
 		minUnderfullThreshold := int32(math.Ceil(sl.CandidateLeases.Mean - LeaseRebalanceThresholdMin))
 		if underfullLeaseThreshold > minUnderfullThreshold {
 			underfullLeaseThreshold = minUnderfullThreshold

pkg/kv/kvserver/replicate_queue_test.go

@@ -191,8 +191,8 @@ func TestReplicateQueueRebalanceMultiStore(t *testing.T) {
 
 	// Speed up the test.
 	allocatorimpl.MinLeaseTransferStatsDuration = 1 * time.Millisecond
-	allocatorimpl.LeaseRebalanceThreshold = 0.01
 	allocatorimpl.LeaseRebalanceThresholdMin = 0.0
+	const leaseRebalanceThreshold = 0.01
 
 	const useDisk = false // for debugging purposes
 	spec := func(node int, store int) base.StoreSpec {
@@ -241,6 +241,7 @@ func TestReplicateQueueRebalanceMultiStore(t *testing.T) {
 			for _, server := range tc.Servers {
 				st := server.ClusterSettings()
 				st.Manual.Store(true)
+				allocatorimpl.LeaseRebalanceThreshold.Override(ctx, &st.SV, leaseRebalanceThreshold)
 			}
 
 			// Add a few ranges per store.