From c5f374a3a280d264aa12fe26aee2efb6db92eed3 Mon Sep 17 00:00:00 2001 From: Aayush Shah Date: Wed, 8 Dec 2021 03:36:55 -0500 Subject: [PATCH] kvserver: fix bug with computing equivalence class Previously, while computing the stores that belong inside the same equivalence class, we were incorrectly including an existing store in the list of candidate stores. For instance, an equivalence class that should look like the following: ``` eqClass: existing: [n1, n2] candidates: [n3, n4] ``` was instead being computed as the following: ``` eqClass: existing: [n1, n2] candidates: [n1, n3, n4] ``` This was throwing things off in the logic implemented by https://github.com/cockroachdb/cockroach/pull/72296 Release note: None --- pkg/kv/kvserver/allocator_scorer.go | 25 +++-- pkg/kv/kvserver/allocator_test.go | 112 ++++++++++++----------- pkg/kv/kvserver/store_rebalancer_test.go | 10 +- 3 files changed, 80 insertions(+), 67 deletions(-) diff --git a/pkg/kv/kvserver/allocator_scorer.go b/pkg/kv/kvserver/allocator_scorer.go index 9284791d3c54..cdaf14293fca 100644 --- a/pkg/kv/kvserver/allocator_scorer.go +++ b/pkg/kv/kvserver/allocator_scorer.go @@ -135,6 +135,9 @@ func (o rangeCountScorerOptions) deterministicForTesting() bool { func (o rangeCountScorerOptions) shouldRebalanceBasedOnThresholds( ctx context.Context, store roachpb.StoreDescriptor, sl StoreList, ) bool { + if len(sl.stores) == 0 { + return false + } overfullThreshold := int32(math.Ceil(overfullRangeThreshold(o, sl.candidateRanges.mean))) // 1. We rebalance if `store` is too far above the mean (i.e. stores // that are overfull). @@ -221,6 +224,9 @@ func (o qpsScorerOptions) deterministicForTesting() bool { func (o qpsScorerOptions) shouldRebalanceBasedOnThresholds( ctx context.Context, store roachpb.StoreDescriptor, sl StoreList, ) bool { + if len(sl.stores) == 0 { + return false + } // 1. We rebalance if `store` is too far above the mean (i.e. stores // that are overfull). overfullThreshold := overfullQPSThreshold(o, sl.candidateQueriesPerSecond.mean) @@ -866,6 +872,11 @@ func rankedCandidateListForRebalancing( } var comparableCands candidateList for _, store := range allStores.stores { + // Only process replacement candidates, not existing stores. + if store.StoreID == existing.store.StoreID { + continue + } + // Ignore any stores on dead nodes or stores that contain any of the // replicas within `replicasOnExemptedStores`. if !isStoreValidForRoutineReplicaTransfer(ctx, store.StoreID) { @@ -951,22 +962,20 @@ func rankedCandidateListForRebalancing( var shouldRebalanceCheck bool if !needRebalance { for _, existing := range existingStores { - var sl StoreList + var candidateSL StoreList outer: for _, comparable := range equivalenceClasses { for _, existingCand := range comparable.existing { if existing.store.StoreID == existingCand.StoreID { - sl = comparable.candidateSL + candidateSL = comparable.candidateSL break outer } } } - // NB: Due to step 2 from above, we're guaranteed to have a non-empty `sl` - // at this point. - // - // TODO(a-robinson): Some moderate refactoring could extract this logic - // out into the loop below, avoiding duplicate balanceScore calculations. - if options.shouldRebalanceBasedOnThresholds(ctx, existing.store, sl) { + // NB: If we have any candidates that are at least as good as the existing + // replicas in terms of diversity and disk fullness, check whether the + // existing replicas' stats are divergent enough to justify a rebalance. + if options.shouldRebalanceBasedOnThresholds(ctx, existing.store, candidateSL) { shouldRebalanceCheck = true break } diff --git a/pkg/kv/kvserver/allocator_test.go b/pkg/kv/kvserver/allocator_test.go index ad9c4f3d5dff..1399fc114b54 100644 --- a/pkg/kv/kvserver/allocator_test.go +++ b/pkg/kv/kvserver/allocator_test.go @@ -1072,10 +1072,13 @@ func TestAllocatorRebalanceTarget(t *testing.T) { } // Set up a second round of testing where the other two stores in the big - // locality actually have fewer replicas, but enough that it still isn't - // worth rebalancing to them. - stores[1].Capacity.RangeCount = 46 - stores[2].Capacity.RangeCount = 46 + // locality actually have fewer replicas, but enough that it still isn't worth + // rebalancing to them. We create a situation where the replacement candidates + // for s1 (i.e. s2 and s3) have an average of 48 replicas each (leading to an + // overfullness threshold of 51, which is greater than the replica count of + // s1). + stores[1].Capacity.RangeCount = 48 + stores[2].Capacity.RangeCount = 48 sg.GossipStores(stores, t) for i := 0; i < 10; i++ { target, _, details, ok := a.RebalanceVoter( @@ -7593,61 +7596,60 @@ func Example_rebalancing() { } table.Render() fmt.Printf("Total bytes=%d, ranges=%d\n", totBytes, totRanges) - // Output: // +-----+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ // | gen | store 0 | store 1 | store 2 | store 3 | store 4 | store 5 | store 6 | store 7 | store 8 | store 9 | store 10 | store 11 | store 12 | store 13 | store 14 | store 15 | store 16 | store 17 | store 18 | store 19 | // +-----+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ // | 0 | 2 100% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | - // | 2 | 3 77% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | 1 22% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | - // | 4 | 3 49% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | 2 18% | 0 0% | 0 0% | 0 0% | 0 0% | 3 17% | 1 14% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | - // | 6 | 3 38% | 0 0% | 0 0% | 0 0% | 0 0% | 2 3% | 1 8% | 3 15% | 0 0% | 0 0% | 2 4% | 0 0% | 4 11% | 3 12% | 0 0% | 0 0% | 0 0% | 0 0% | 1 6% | 0 0% | - // | 8 | 4 22% | 0 0% | 0 0% | 1 6% | 0 0% | 4 6% | 3 8% | 4 8% | 2 9% | 0 0% | 4 6% | 1 0% | 4 3% | 4 14% | 0 0% | 0 0% | 0 0% | 0 0% | 3 9% | 2 4% | - // | 10 | 5 15% | 3 3% | 2 3% | 3 9% | 2 5% | 5 3% | 4 6% | 5 5% | 3 8% | 0 0% | 5 5% | 3 4% | 5 4% | 5 8% | 1 1% | 1 1% | 1 2% | 0 0% | 4 7% | 3 2% | - // | 12 | 6 10% | 4 1% | 4 4% | 4 6% | 4 5% | 6 1% | 5 6% | 6 5% | 4 6% | 4 2% | 6 4% | 4 4% | 6 4% | 6 4% | 4 5% | 3 3% | 3 3% | 10 9% | 5 6% | 4 2% | - // | 14 | 8 9% | 6 2% | 6 3% | 6 5% | 6 5% | 8 2% | 7 4% | 8 5% | 6 6% | 6 4% | 8 4% | 6 4% | 8 4% | 8 4% | 6 6% | 5 3% | 6 4% | 10 5% | 7 5% | 7 4% | - // | 16 | 10 8% | 8 3% | 8 4% | 8 6% | 9 5% | 10 2% | 9 5% | 10 5% | 8 5% | 8 4% | 10 4% | 8 5% | 10 4% | 10 5% | 8 5% | 7 4% | 8 4% | 11 3% | 9 4% | 9 4% | - // | 18 | 12 7% | 10 4% | 10 4% | 10 6% | 11 5% | 12 3% | 11 5% | 12 5% | 10 5% | 10 4% | 12 4% | 10 5% | 12 4% | 12 5% | 10 4% | 9 4% | 10 3% | 13 4% | 11 4% | 11 4% | - // | 20 | 14 7% | 12 4% | 12 4% | 12 6% | 13 5% | 14 3% | 13 5% | 14 4% | 12 5% | 12 4% | 14 4% | 12 5% | 14 4% | 14 5% | 12 4% | 11 5% | 12 4% | 15 4% | 13 4% | 13 4% | - // | 22 | 16 7% | 14 3% | 14 5% | 14 6% | 15 6% | 16 3% | 15 5% | 16 5% | 14 5% | 14 4% | 16 4% | 14 4% | 16 4% | 16 5% | 14 4% | 13 4% | 14 4% | 17 5% | 15 4% | 15 4% | - // | 24 | 18 7% | 16 3% | 16 5% | 16 6% | 17 6% | 18 3% | 17 5% | 18 5% | 16 5% | 16 4% | 18 4% | 16 5% | 18 4% | 18 4% | 16 4% | 15 5% | 16 4% | 19 5% | 17 4% | 17 4% | - // | 26 | 20 6% | 18 3% | 18 4% | 18 5% | 19 6% | 20 3% | 19 5% | 20 5% | 18 4% | 18 5% | 20 4% | 18 5% | 20 4% | 20 5% | 18 4% | 17 5% | 18 4% | 21 5% | 19 4% | 19 4% | - // | 28 | 22 6% | 20 3% | 20 4% | 20 5% | 21 6% | 22 3% | 21 5% | 22 5% | 20 4% | 20 5% | 22 4% | 20 5% | 22 4% | 22 5% | 20 4% | 19 4% | 20 4% | 23 5% | 21 4% | 21 4% | - // | 30 | 24 6% | 22 3% | 22 4% | 22 5% | 23 6% | 24 3% | 23 5% | 24 5% | 22 4% | 22 5% | 24 4% | 22 5% | 24 4% | 24 5% | 22 4% | 21 4% | 22 4% | 25 5% | 23 4% | 23 4% | - // | 32 | 26 6% | 24 3% | 24 4% | 24 5% | 25 5% | 26 3% | 25 5% | 26 5% | 24 4% | 24 4% | 26 4% | 24 5% | 26 4% | 26 5% | 24 4% | 23 4% | 24 4% | 27 4% | 25 4% | 25 4% | - // | 34 | 28 6% | 26 3% | 26 4% | 26 5% | 27 5% | 28 4% | 27 5% | 28 6% | 26 4% | 26 4% | 28 4% | 26 5% | 28 4% | 28 5% | 26 5% | 25 4% | 26 4% | 29 4% | 27 4% | 27 4% | - // | 36 | 30 6% | 28 3% | 28 4% | 28 5% | 29 5% | 30 4% | 29 5% | 30 5% | 28 4% | 28 4% | 30 4% | 28 5% | 30 4% | 30 5% | 28 5% | 27 4% | 28 4% | 31 4% | 29 4% | 29 4% | - // | 38 | 32 6% | 30 3% | 30 4% | 30 5% | 31 5% | 32 4% | 31 5% | 32 5% | 30 4% | 30 4% | 32 4% | 30 5% | 32 4% | 32 4% | 30 5% | 29 5% | 30 5% | 33 4% | 31 4% | 31 4% | - // | 40 | 34 6% | 32 3% | 32 4% | 32 5% | 33 5% | 34 4% | 33 5% | 34 5% | 32 4% | 32 4% | 34 4% | 32 5% | 34 4% | 34 5% | 32 5% | 31 5% | 32 5% | 35 4% | 33 4% | 33 4% | - // | 42 | 36 6% | 34 4% | 34 4% | 34 4% | 35 5% | 36 4% | 35 5% | 36 5% | 34 4% | 34 4% | 36 5% | 34 5% | 36 4% | 36 5% | 34 5% | 33 5% | 34 5% | 37 4% | 35 4% | 35 4% | - // | 44 | 38 6% | 36 4% | 36 4% | 36 5% | 37 5% | 38 3% | 37 5% | 38 5% | 36 4% | 36 4% | 38 4% | 36 5% | 38 4% | 38 5% | 36 5% | 35 5% | 36 5% | 39 4% | 37 4% | 37 5% | - // | 46 | 40 6% | 38 3% | 38 4% | 38 5% | 39 5% | 40 4% | 39 5% | 40 5% | 38 4% | 38 4% | 40 4% | 38 5% | 40 4% | 40 5% | 38 5% | 37 5% | 38 4% | 41 4% | 39 5% | 39 4% | - // | 48 | 42 6% | 40 3% | 40 4% | 40 4% | 41 5% | 42 4% | 41 5% | 42 5% | 40 4% | 40 4% | 42 5% | 40 5% | 42 5% | 42 5% | 40 5% | 39 5% | 40 4% | 43 4% | 41 5% | 41 5% | - // | 50 | 44 6% | 42 3% | 42 4% | 42 4% | 43 5% | 44 4% | 43 5% | 44 5% | 42 4% | 42 4% | 44 5% | 42 5% | 44 4% | 44 5% | 42 5% | 41 5% | 42 4% | 45 4% | 43 5% | 43 4% | - // | 52 | 46 6% | 44 4% | 44 4% | 44 4% | 45 5% | 46 4% | 45 5% | 46 5% | 44 4% | 44 4% | 46 5% | 44 5% | 46 4% | 46 5% | 44 5% | 43 5% | 44 5% | 47 4% | 45 5% | 45 4% | - // | 54 | 48 5% | 46 4% | 46 4% | 46 4% | 47 5% | 48 4% | 47 5% | 48 5% | 46 4% | 46 4% | 48 5% | 46 5% | 48 4% | 48 5% | 46 4% | 45 5% | 46 5% | 49 4% | 47 5% | 47 4% | - // | 56 | 50 5% | 48 4% | 48 4% | 48 4% | 49 5% | 50 4% | 49 5% | 50 5% | 48 4% | 48 4% | 50 4% | 48 5% | 50 4% | 50 5% | 48 4% | 47 5% | 48 5% | 51 4% | 49 5% | 49 4% | - // | 58 | 52 5% | 50 4% | 50 4% | 50 4% | 51 5% | 52 4% | 51 5% | 52 5% | 50 4% | 50 4% | 52 4% | 50 5% | 52 4% | 52 5% | 50 4% | 49 5% | 50 5% | 53 4% | 51 5% | 51 4% | - // | 60 | 54 5% | 52 4% | 52 4% | 52 4% | 53 5% | 54 4% | 53 4% | 54 5% | 52 4% | 52 4% | 54 5% | 52 5% | 54 4% | 54 5% | 52 4% | 51 5% | 52 5% | 55 4% | 53 5% | 53 4% | - // | 62 | 56 5% | 54 4% | 54 4% | 54 4% | 55 5% | 56 4% | 55 5% | 56 5% | 54 4% | 54 4% | 56 4% | 54 5% | 56 4% | 56 5% | 54 4% | 53 5% | 54 5% | 57 4% | 55 5% | 55 4% | - // | 64 | 58 5% | 56 4% | 56 4% | 56 4% | 57 5% | 58 4% | 57 4% | 58 5% | 56 4% | 56 4% | 58 4% | 56 5% | 58 4% | 58 5% | 56 4% | 55 5% | 56 5% | 59 4% | 57 5% | 57 4% | - // | 66 | 60 6% | 58 4% | 58 4% | 58 4% | 59 5% | 60 4% | 59 4% | 60 5% | 58 4% | 58 4% | 60 5% | 58 5% | 60 4% | 60 5% | 58 4% | 57 5% | 58 5% | 61 4% | 59 5% | 59 4% | - // | 68 | 62 5% | 60 4% | 60 4% | 60 4% | 61 5% | 62 4% | 61 5% | 62 5% | 60 4% | 60 4% | 62 5% | 60 5% | 62 4% | 62 5% | 60 4% | 59 5% | 60 5% | 63 4% | 61 5% | 61 4% | - // | 70 | 64 5% | 62 4% | 62 4% | 62 4% | 63 5% | 64 4% | 63 5% | 64 5% | 62 4% | 62 4% | 64 5% | 62 5% | 64 5% | 64 5% | 62 4% | 61 5% | 62 5% | 65 4% | 63 5% | 63 4% | - // | 72 | 66 6% | 64 4% | 64 4% | 64 4% | 65 5% | 66 4% | 65 5% | 66 5% | 64 4% | 64 4% | 66 5% | 64 4% | 66 4% | 66 5% | 64 4% | 63 5% | 64 5% | 67 4% | 65 5% | 65 4% | - // | 74 | 68 6% | 66 4% | 66 4% | 66 4% | 67 5% | 68 4% | 67 5% | 68 5% | 66 4% | 66 4% | 68 5% | 66 4% | 68 5% | 68 5% | 66 4% | 65 5% | 66 5% | 69 4% | 67 5% | 67 4% | - // | 76 | 70 5% | 68 4% | 68 4% | 68 4% | 69 5% | 70 4% | 69 5% | 70 5% | 68 4% | 68 4% | 70 5% | 68 4% | 70 4% | 70 5% | 68 4% | 67 5% | 68 5% | 71 4% | 69 4% | 69 4% | - // | 78 | 72 5% | 70 4% | 70 4% | 70 4% | 71 5% | 72 4% | 71 5% | 72 5% | 70 4% | 70 4% | 72 5% | 70 4% | 72 5% | 72 5% | 70 4% | 69 5% | 70 4% | 73 4% | 71 4% | 71 4% | - // | 80 | 74 5% | 72 4% | 72 4% | 72 4% | 73 5% | 74 4% | 73 5% | 74 5% | 72 4% | 72 4% | 74 5% | 72 4% | 74 4% | 74 5% | 72 4% | 71 5% | 72 4% | 75 4% | 73 4% | 73 4% | - // | 82 | 76 5% | 74 4% | 74 4% | 74 4% | 75 5% | 76 4% | 75 5% | 76 5% | 74 4% | 74 4% | 76 5% | 74 4% | 76 4% | 76 5% | 74 4% | 73 5% | 74 4% | 77 4% | 75 4% | 75 4% | - // | 84 | 78 6% | 76 4% | 76 4% | 76 4% | 77 5% | 78 4% | 77 5% | 78 5% | 76 4% | 76 4% | 78 5% | 76 4% | 78 5% | 78 5% | 76 4% | 75 5% | 76 4% | 79 4% | 77 4% | 77 4% | - // | 86 | 80 5% | 78 4% | 78 4% | 78 4% | 79 5% | 80 4% | 79 5% | 80 5% | 78 4% | 78 4% | 80 5% | 78 4% | 80 4% | 80 5% | 78 5% | 77 5% | 78 4% | 81 4% | 79 4% | 79 4% | - // | 88 | 82 5% | 80 4% | 80 4% | 80 4% | 81 5% | 82 4% | 81 5% | 82 5% | 80 4% | 80 4% | 82 5% | 80 4% | 82 4% | 82 5% | 80 5% | 79 5% | 80 4% | 83 4% | 81 4% | 81 4% | - // | 90 | 84 5% | 82 4% | 82 4% | 82 4% | 83 5% | 84 4% | 83 5% | 84 5% | 82 4% | 82 4% | 84 5% | 82 5% | 84 4% | 84 5% | 82 5% | 81 5% | 82 4% | 85 4% | 83 4% | 83 4% | - // | 92 | 86 5% | 84 4% | 84 4% | 84 4% | 85 5% | 86 4% | 85 5% | 86 5% | 84 4% | 84 4% | 86 5% | 84 4% | 86 4% | 86 5% | 84 5% | 83 5% | 84 4% | 87 4% | 85 4% | 85 4% | - // | 94 | 88 5% | 86 4% | 86 4% | 86 4% | 87 5% | 88 4% | 87 5% | 88 5% | 86 4% | 86 4% | 88 5% | 86 4% | 88 4% | 88 5% | 86 5% | 85 5% | 86 4% | 89 4% | 87 5% | 87 4% | - // | 96 | 90 5% | 88 4% | 88 4% | 88 4% | 89 5% | 90 4% | 89 5% | 90 5% | 88 4% | 88 4% | 90 5% | 88 4% | 90 5% | 90 5% | 88 4% | 87 5% | 88 4% | 91 4% | 89 5% | 89 4% | - // | 98 | 92 5% | 90 4% | 90 4% | 90 4% | 91 5% | 92 4% | 91 4% | 92 5% | 90 4% | 90 4% | 92 5% | 90 4% | 92 5% | 92 4% | 90 5% | 89 5% | 90 4% | 93 4% | 91 5% | 91 4% | + // | 2 | 3 54% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | 2 45% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | + // | 4 | 3 35% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | 2 20% | 1 8% | 3 20% | 0 0% | 0 0% | 0 0% | 0 0% | 1 15% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | 0 0% | + // | 6 | 3 9% | 0 0% | 0 0% | 1 8% | 0 0% | 1 3% | 3 9% | 3 12% | 3 12% | 0 0% | 1 7% | 0 0% | 2 5% | 3 23% | 0 0% | 0 0% | 2 6% | 0 0% | 0 0% | 0 0% | + // | 8 | 4 5% | 2 5% | 0 0% | 3 9% | 0 0% | 3 5% | 4 9% | 4 5% | 4 7% | 0 0% | 3 7% | 1 3% | 4 10% | 4 11% | 0 0% | 0 0% | 4 11% | 1 3% | 1 3% | 0 0% | + // | 10 | 5 7% | 3 5% | 1 0% | 4 7% | 3 3% | 4 3% | 5 5% | 5 6% | 5 6% | 2 1% | 4 7% | 3 5% | 5 7% | 5 7% | 2 4% | 2 0% | 5 8% | 3 5% | 3 5% | 1 0% | + // | 12 | 7 6% | 5 5% | 3 2% | 6 7% | 5 3% | 6 3% | 7 5% | 7 4% | 7 5% | 4 2% | 6 6% | 5 4% | 7 7% | 7 7% | 4 4% | 4 1% | 7 8% | 5 4% | 5 4% | 3 3% | + // | 14 | 9 5% | 7 5% | 5 2% | 8 6% | 7 4% | 8 5% | 9 5% | 9 5% | 9 5% | 6 3% | 8 5% | 7 3% | 9 6% | 9 6% | 6 4% | 6 3% | 9 7% | 7 5% | 7 4% | 5 3% | + // | 16 | 11 4% | 9 5% | 7 2% | 10 6% | 9 5% | 10 5% | 11 4% | 11 4% | 11 5% | 8 3% | 10 5% | 9 4% | 11 6% | 11 6% | 8 4% | 8 3% | 11 6% | 9 5% | 9 5% | 7 3% | + // | 18 | 13 4% | 11 5% | 9 1% | 12 6% | 11 5% | 12 5% | 13 5% | 13 5% | 13 5% | 10 4% | 12 5% | 11 4% | 13 5% | 13 5% | 10 4% | 10 3% | 13 6% | 11 5% | 11 5% | 9 4% | + // | 20 | 15 5% | 13 5% | 11 2% | 14 5% | 13 4% | 14 5% | 15 4% | 15 4% | 15 5% | 12 4% | 14 5% | 13 5% | 15 5% | 15 5% | 12 4% | 12 3% | 15 6% | 13 4% | 13 5% | 11 4% | + // | 22 | 17 5% | 15 5% | 13 3% | 16 5% | 15 4% | 16 5% | 17 4% | 17 4% | 17 5% | 14 4% | 16 5% | 15 4% | 17 5% | 17 4% | 14 5% | 14 3% | 17 6% | 15 4% | 15 4% | 13 4% | + // | 24 | 19 5% | 17 5% | 15 3% | 18 5% | 17 4% | 18 5% | 19 4% | 19 4% | 19 5% | 16 4% | 18 5% | 17 4% | 19 5% | 19 4% | 16 5% | 16 3% | 19 6% | 17 5% | 17 5% | 15 4% | + // | 26 | 21 4% | 19 5% | 17 3% | 20 6% | 19 4% | 20 5% | 21 4% | 21 4% | 21 4% | 18 4% | 20 5% | 19 4% | 21 5% | 21 4% | 18 5% | 18 3% | 21 6% | 19 4% | 19 5% | 17 4% | + // | 28 | 23 4% | 21 5% | 19 3% | 22 6% | 21 4% | 22 5% | 23 4% | 23 4% | 23 5% | 20 4% | 22 5% | 21 4% | 23 5% | 23 4% | 20 5% | 20 3% | 23 6% | 21 4% | 21 4% | 19 4% | + // | 30 | 25 4% | 23 5% | 21 3% | 24 6% | 23 4% | 24 5% | 25 5% | 25 4% | 25 5% | 22 4% | 24 5% | 23 4% | 25 5% | 25 4% | 22 5% | 22 3% | 25 6% | 23 4% | 23 4% | 21 5% | + // | 32 | 27 4% | 25 5% | 23 3% | 26 6% | 25 4% | 26 5% | 27 5% | 27 4% | 27 5% | 24 4% | 26 5% | 25 4% | 27 5% | 27 4% | 24 4% | 24 3% | 27 6% | 25 4% | 25 5% | 23 4% | + // | 34 | 29 4% | 27 5% | 25 4% | 28 6% | 27 4% | 28 5% | 29 5% | 29 4% | 29 5% | 26 4% | 28 5% | 27 5% | 29 5% | 29 4% | 26 5% | 26 4% | 29 6% | 27 4% | 27 4% | 25 5% | + // | 36 | 31 4% | 29 4% | 27 4% | 30 5% | 29 4% | 30 5% | 31 5% | 31 4% | 31 5% | 28 4% | 30 5% | 29 4% | 31 5% | 31 4% | 28 5% | 28 4% | 31 5% | 29 4% | 29 5% | 27 5% | + // | 38 | 33 4% | 31 5% | 29 3% | 32 5% | 31 4% | 32 5% | 33 5% | 33 4% | 33 5% | 30 4% | 32 5% | 31 4% | 33 5% | 33 5% | 30 4% | 30 4% | 33 5% | 31 4% | 31 4% | 29 5% | + // | 40 | 35 4% | 33 4% | 31 3% | 34 5% | 33 4% | 34 5% | 35 5% | 35 4% | 35 5% | 32 4% | 34 5% | 33 4% | 35 5% | 35 5% | 32 4% | 32 4% | 35 5% | 33 4% | 33 5% | 31 5% | + // | 42 | 37 4% | 35 4% | 33 4% | 36 5% | 35 4% | 36 5% | 37 5% | 37 4% | 37 4% | 34 4% | 36 5% | 35 5% | 37 5% | 37 5% | 34 5% | 34 4% | 37 5% | 35 4% | 35 5% | 33 5% | + // | 44 | 39 4% | 37 4% | 35 4% | 38 5% | 37 4% | 38 5% | 39 5% | 39 4% | 39 4% | 36 4% | 38 5% | 37 5% | 39 5% | 39 5% | 36 5% | 36 4% | 39 5% | 37 4% | 37 5% | 35 5% | + // | 46 | 41 4% | 39 4% | 37 3% | 40 5% | 39 4% | 40 5% | 41 5% | 41 4% | 41 4% | 38 4% | 40 5% | 39 5% | 41 5% | 41 5% | 38 5% | 38 4% | 41 5% | 39 4% | 39 5% | 37 5% | + // | 48 | 43 4% | 41 4% | 39 3% | 42 5% | 41 4% | 42 5% | 43 5% | 43 4% | 43 4% | 40 4% | 42 5% | 41 4% | 43 5% | 43 5% | 40 5% | 40 4% | 43 5% | 41 4% | 41 5% | 39 5% | + // | 50 | 45 4% | 43 4% | 41 4% | 44 5% | 43 4% | 44 5% | 45 5% | 45 4% | 45 4% | 42 4% | 44 5% | 43 4% | 45 5% | 45 5% | 42 5% | 42 4% | 45 5% | 43 5% | 43 5% | 41 5% | + // | 52 | 47 4% | 45 4% | 43 3% | 46 5% | 45 4% | 46 5% | 47 5% | 47 4% | 47 4% | 44 4% | 46 5% | 45 5% | 47 5% | 47 5% | 44 5% | 44 4% | 47 5% | 45 5% | 45 5% | 43 5% | + // | 54 | 49 4% | 47 4% | 45 3% | 48 5% | 47 4% | 48 5% | 49 5% | 49 4% | 49 4% | 46 4% | 48 5% | 47 4% | 49 5% | 49 5% | 46 5% | 46 4% | 49 5% | 47 5% | 47 5% | 45 5% | + // | 56 | 51 4% | 49 5% | 47 3% | 50 5% | 49 4% | 50 5% | 51 5% | 51 4% | 51 4% | 48 4% | 50 5% | 49 4% | 51 5% | 51 5% | 48 5% | 48 4% | 51 5% | 49 5% | 49 4% | 47 5% | + // | 58 | 53 4% | 51 5% | 49 3% | 52 5% | 51 4% | 52 5% | 53 5% | 53 4% | 53 4% | 50 4% | 52 5% | 51 4% | 53 5% | 53 4% | 50 5% | 50 4% | 53 5% | 51 5% | 51 4% | 49 5% | + // | 60 | 55 4% | 53 5% | 51 3% | 54 5% | 53 4% | 54 5% | 55 5% | 55 4% | 55 4% | 52 4% | 54 5% | 53 4% | 55 5% | 55 4% | 52 5% | 52 4% | 55 5% | 53 5% | 53 4% | 51 5% | + // | 62 | 57 4% | 55 5% | 53 3% | 56 5% | 55 4% | 56 5% | 57 5% | 57 4% | 57 4% | 54 4% | 56 5% | 55 4% | 57 5% | 57 4% | 54 5% | 54 4% | 57 5% | 55 5% | 55 4% | 53 4% | + // | 64 | 59 4% | 57 5% | 55 4% | 58 5% | 57 4% | 58 5% | 59 5% | 59 4% | 59 4% | 56 4% | 58 5% | 57 4% | 59 5% | 59 4% | 56 5% | 56 4% | 59 5% | 57 5% | 57 4% | 55 5% | + // | 66 | 61 4% | 59 5% | 57 4% | 60 5% | 59 4% | 60 5% | 61 5% | 61 4% | 61 4% | 58 4% | 60 5% | 59 4% | 61 5% | 61 4% | 58 5% | 58 4% | 61 5% | 59 5% | 59 4% | 57 5% | + // | 68 | 63 4% | 61 5% | 59 4% | 62 5% | 61 4% | 62 5% | 63 5% | 63 4% | 63 4% | 60 4% | 62 5% | 61 4% | 63 5% | 63 4% | 60 5% | 60 4% | 63 5% | 61 5% | 61 4% | 59 5% | + // | 70 | 65 4% | 63 5% | 61 4% | 64 5% | 63 4% | 64 5% | 65 5% | 65 4% | 65 5% | 62 4% | 64 5% | 63 4% | 65 5% | 65 4% | 62 5% | 62 5% | 65 5% | 63 5% | 63 4% | 61 5% | + // | 72 | 67 4% | 65 5% | 63 4% | 66 5% | 65 4% | 66 5% | 67 5% | 67 4% | 67 5% | 64 4% | 66 5% | 65 4% | 67 5% | 67 4% | 64 5% | 64 5% | 67 5% | 65 5% | 65 4% | 63 5% | + // | 74 | 69 4% | 67 5% | 65 4% | 68 5% | 67 4% | 68 5% | 69 5% | 69 4% | 69 4% | 66 4% | 68 5% | 67 5% | 69 5% | 69 4% | 66 5% | 66 5% | 69 5% | 67 5% | 67 4% | 65 5% | + // | 76 | 71 4% | 69 5% | 67 4% | 70 5% | 69 4% | 70 5% | 71 5% | 71 4% | 71 4% | 68 4% | 70 5% | 69 5% | 71 5% | 71 4% | 68 5% | 68 4% | 71 5% | 69 5% | 69 4% | 67 5% | + // | 78 | 73 4% | 71 5% | 69 4% | 72 5% | 71 4% | 72 5% | 73 5% | 73 4% | 73 4% | 70 4% | 72 5% | 71 5% | 73 5% | 73 4% | 70 5% | 70 4% | 73 5% | 71 5% | 71 4% | 69 5% | + // | 80 | 75 4% | 73 4% | 71 4% | 74 5% | 73 4% | 74 5% | 75 5% | 75 4% | 75 4% | 72 5% | 74 5% | 73 4% | 75 5% | 75 4% | 72 5% | 72 4% | 75 5% | 73 5% | 73 4% | 71 5% | + // | 82 | 77 4% | 75 4% | 73 4% | 76 5% | 75 4% | 76 5% | 77 5% | 77 4% | 77 4% | 74 5% | 76 4% | 75 5% | 77 5% | 77 4% | 74 5% | 74 4% | 77 5% | 75 4% | 75 4% | 73 5% | + // | 84 | 79 4% | 77 4% | 75 4% | 78 5% | 77 4% | 78 5% | 79 5% | 79 4% | 79 4% | 76 5% | 78 5% | 77 4% | 79 5% | 79 4% | 76 5% | 76 4% | 79 5% | 77 5% | 77 5% | 75 5% | + // | 86 | 81 4% | 79 4% | 77 4% | 80 5% | 79 4% | 80 5% | 81 5% | 81 4% | 81 4% | 78 5% | 80 5% | 79 5% | 81 5% | 81 4% | 78 5% | 78 4% | 81 5% | 79 4% | 79 4% | 77 5% | + // | 88 | 83 4% | 81 4% | 79 4% | 82 5% | 81 5% | 82 5% | 83 5% | 83 4% | 83 4% | 80 5% | 82 4% | 81 5% | 83 5% | 83 4% | 80 5% | 80 4% | 83 5% | 81 4% | 81 4% | 79 5% | + // | 90 | 85 4% | 83 4% | 81 4% | 84 5% | 83 5% | 84 5% | 85 5% | 85 4% | 85 4% | 82 5% | 84 4% | 83 5% | 85 5% | 85 5% | 82 5% | 82 4% | 85 5% | 83 4% | 83 5% | 81 5% | + // | 92 | 87 4% | 85 4% | 83 4% | 86 5% | 85 5% | 86 5% | 87 5% | 87 4% | 87 4% | 84 5% | 86 5% | 85 5% | 87 5% | 87 5% | 84 5% | 84 4% | 87 5% | 85 4% | 85 4% | 83 5% | + // | 94 | 89 4% | 87 4% | 85 4% | 88 5% | 87 5% | 88 5% | 89 5% | 89 4% | 89 4% | 86 5% | 88 5% | 87 4% | 89 5% | 89 5% | 86 5% | 86 4% | 89 5% | 87 4% | 87 4% | 85 5% | + // | 96 | 91 4% | 89 4% | 87 4% | 90 5% | 89 5% | 90 5% | 91 5% | 91 4% | 91 4% | 88 5% | 90 4% | 89 4% | 91 5% | 91 5% | 88 5% | 88 4% | 91 5% | 89 4% | 89 4% | 87 5% | + // | 98 | 93 4% | 91 4% | 89 4% | 92 5% | 91 5% | 92 5% | 93 5% | 93 4% | 93 4% | 90 5% | 92 4% | 91 4% | 93 5% | 93 5% | 90 5% | 90 4% | 93 5% | 91 5% | 91 4% | 89 5% | // +-----+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ - // Total bytes=962012605, ranges=1838 + // Total bytes=968307769, ranges=1850 } diff --git a/pkg/kv/kvserver/store_rebalancer_test.go b/pkg/kv/kvserver/store_rebalancer_test.go index 4970c2ac95b3..b3de2c1fbe4d 100644 --- a/pkg/kv/kvserver/store_rebalancer_test.go +++ b/pkg/kv/kvserver/store_rebalancer_test.go @@ -738,17 +738,19 @@ func TestChooseRangeToRebalanceAcrossHeterogeneousZones(t *testing.T) { // the same region. { // Within the hottest region, expect rebalance from the hottest node (n1) - // to the coolest node (n3). + // to the coolest node (n3). Within the lease hot region, expect movement + // from n8 to n9. name: "QPS balance without constraints", voters: []roachpb.StoreID{1, 5, 8}, - expRebalancedVoters: []roachpb.StoreID{8, 5, 3}, + expRebalancedVoters: []roachpb.StoreID{9, 5, 3}, }, { // Within the second hottest region, expect rebalance from the hottest - // node (n4) to the coolest node (n6). + // node (n4) to the coolest node (n6). Within the lease hot region, expect + // movement from n8 to n9. name: "QPS balance without constraints", voters: []roachpb.StoreID{8, 4, 3}, - expRebalancedVoters: []roachpb.StoreID{8, 6, 3}, + expRebalancedVoters: []roachpb.StoreID{9, 6, 3}, }, // Multi-region database configurations.