kvserver: stop transferring leases to draining/suspect nodes in the S…

…toreRebalancer

This commit prevents the StoreRebalancer from transferring leases to replicas
on draining or suspect nodes. In some cases, we've seen this to cause new
leases to be pushed to nodes that take too long to drain or that are stuck
while draining due to other bugs.

Informs https://github.com/cockroachlabs/support/issues/1105

Release note: None

pkg/kv/kvserver/store_rebalancer.go

@@ -425,6 +425,14 @@ func (sr *StoreRebalancer) chooseLeaseToTransfer(
 		var raftStatus *raft.Status
 
 		preferred := sr.rq.allocator.preferredLeaseholders(zone, candidates)
+
+		// Filter both the list of preferred stores as well as the list of all
+		// candidate replicas to only consider live (non-suspect, non-draining)
+		// nodes.
+		const includeSuspectAndDrainingStores = false
+		preferred, _ = sr.rq.allocator.storePool.liveAndDeadReplicas(preferred, includeSuspectAndDrainingStores)
+		candidates, _ = sr.rq.allocator.storePool.liveAndDeadReplicas(candidates, includeSuspectAndDrainingStores)
+
 		for _, candidate := range candidates {
 			if candidate.StoreID == localDesc.StoreID {
 				continue