From 1ef40f323b94005dd84378669da8296c9fec8e7c Mon Sep 17 00:00:00 2001 From: Bram Gruneir Date: Wed, 26 Oct 2016 12:44:24 -0400 Subject: [PATCH 1/2] storage: reapply the rule solver This adds back in 3 commits that were removed to facilitate the merge of develop back to master. One other commit, is no longer required. Follow up fixes are tracked in #10275. Closes #9336 1) 4446345bf7a9e1a6eb7c7a3c71ec9c8e3b3d17fc storage: add constraint rule solver for allocation Rules are represented as a single function that returns the candidacy of the store as well as a float value representing the score. These scores are then aggregated from all rules and returns the stores sorted by them. Current rules: - ruleReplicasUniqueNodes ensures that no two replicas are put on the same node. - ruleConstraints enforces that required and prohibited constraints are followed, and that stores with more positive constraints are ranked higher. - ruleDiversity ensures that nodes that have the fewest locality tiers in common are given higher priority. - ruleCapacity prioritizes placing data on empty nodes when the choice is available and prevents data from going onto mostly full nodes. 2) dd3229a718d5e3ec25a784f2f3ff5c9ffd85a3b0 storage: implemented RuleSolver into allocator 3) 27353a8d4d90406cb5ff21d4f8198b7e5b5e2325 storage: removed unused rangeCountBalancer There was a 4th commit that is no longer required. The simulation was already converging since adding a rebalance threshold. 4e29a36e7806a26ad4d7b5c1b3c06037d328340b storage/simulation: only rebalance 50% of ranges on each iteration so it will converge --- pkg/roachpb/metadata.go | 9 - pkg/storage/allocator.go | 266 ++++++++++++----------- pkg/storage/allocator_test.go | 340 +++++++++++++++++++----------- pkg/storage/balancer.go | 218 ------------------- pkg/storage/client_raft_test.go | 4 +- pkg/storage/client_test.go | 7 +- pkg/storage/helpers_test.go | 7 +- pkg/storage/replicate_queue.go | 22 +- pkg/storage/rule_solver.go | 269 +++++++++++++++++++++++ pkg/storage/rule_solver_test.go | 334 +++++++++++++++++++++++++++++ pkg/storage/simulation/cluster.go | 1 - pkg/storage/simulation/range.go | 8 +- pkg/storage/store_pool.go | 108 +++++----- pkg/storage/store_pool_test.go | 68 ++---- 14 files changed, 1045 insertions(+), 616 deletions(-) delete mode 100644 pkg/storage/balancer.go create mode 100644 pkg/storage/rule_solver.go create mode 100644 pkg/storage/rule_solver_test.go diff --git a/pkg/roachpb/metadata.go b/pkg/roachpb/metadata.go index a54170ba4b30..409c10eb663a 100644 --- a/pkg/roachpb/metadata.go +++ b/pkg/roachpb/metadata.go @@ -193,15 +193,6 @@ func (sc StoreCapacity) FractionUsed() float64 { return float64(sc.Capacity-sc.Available) / float64(sc.Capacity) } -// CombinedAttrs returns the full list of attributes for the store, including -// both the node and store attributes. -func (s StoreDescriptor) CombinedAttrs() *Attributes { - var a []string - a = append(a, s.Node.Attrs.Attrs...) - a = append(a, s.Attrs.Attrs...) - return &Attributes{Attrs: a} -} - // String returns a string representation of the Tier. func (t Tier) String() string { return fmt.Sprintf("%s=%s", t.Key, t.Value) diff --git a/pkg/storage/allocator.go b/pkg/storage/allocator.go index cff4b5d76a3f..6fd4cabbb401 100644 --- a/pkg/storage/allocator.go +++ b/pkg/storage/allocator.go @@ -20,16 +20,16 @@ package storage import ( "fmt" - "math/rand" + "math" + "github.com/pkg/errors" "golang.org/x/net/context" "github.com/cockroachdb/cockroach/pkg/config" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/util" + "github.com/cockroachdb/cockroach/pkg/util/envutil" "github.com/cockroachdb/cockroach/pkg/util/log" - "github.com/cockroachdb/cockroach/pkg/util/syncutil" - "github.com/pkg/errors" ) const ( @@ -99,58 +99,28 @@ func (*allocatorError) purgatoryErrorMarker() {} var _ purgatoryError = &allocatorError{} -// allocatorRand pairs a rand.Rand with a mutex. -// TODO: Allocator is typically only accessed from a single thread (the -// replication queue), but this assumption is broken in tests which force -// replication scans. If those tests can be modified to suspend the normal -// replication queue during the forced scan, then this rand could be used -// without a mutex. -type allocatorRand struct { - *syncutil.Mutex - *rand.Rand -} - -func makeAllocatorRand(source rand.Source) allocatorRand { - return allocatorRand{ - Mutex: &syncutil.Mutex{}, - Rand: rand.New(source), - } -} - // AllocatorOptions are configurable options which effect the way that the // replicate queue will handle rebalancing opportunities. type AllocatorOptions struct { // AllowRebalance allows this store to attempt to rebalance its own // replicas to other stores. AllowRebalance bool - - // Deterministic makes allocation decisions deterministic, based on - // current cluster statistics. If this flag is not set, allocation operations - // will have random behavior. This flag is intended to be set for testing - // purposes only. - Deterministic bool } // Allocator tries to spread replicas as evenly as possible across the stores // in the cluster. type Allocator struct { - storePool *StorePool - randGen allocatorRand - options AllocatorOptions + storePool *StorePool + options AllocatorOptions + ruleSolver ruleSolver } // MakeAllocator creates a new allocator using the specified StorePool. func MakeAllocator(storePool *StorePool, options AllocatorOptions) Allocator { - var randSource rand.Source - if options.Deterministic { - randSource = rand.NewSource(777) - } else { - randSource = rand.NewSource(rand.Int63()) - } return Allocator{ - storePool: storePool, - options: options, - randGen: makeAllocatorRand(randSource), + storePool: storePool, + options: options, + ruleSolver: makeDefaultRuleSolver(storePool), } } @@ -197,69 +167,55 @@ func (a *Allocator) ComputeAction( } // AllocateTarget returns a suitable store for a new allocation with the -// required attributes. Nodes already accommodating existing replicas are ruled -// out as targets. The range ID of the replica being allocated for is also -// passed in to ensure that we don't try to replace an existing dead replica on -// a store. If relaxConstraints is true, then the required attributes will be -// relaxed as necessary, from least specific to most specific, in order to -// allocate a target. +// required attributes. Nodes already accommodating existing or dead replicas +// are ruled out as targets. func (a *Allocator) AllocateTarget( - constraints config.Constraints, - existing []roachpb.ReplicaDescriptor, - rangeID roachpb.RangeID, - relaxConstraints bool, + constraints config.Constraints, existing []roachpb.ReplicaDescriptor, rangeID roachpb.RangeID, ) (*roachpb.StoreDescriptor, error) { - existingNodes := make(nodeIDSet, len(existing)) - for _, repl := range existing { - existingNodes[repl.NodeID] = struct{}{} + + sl, _, throttledStoreCount := a.storePool.getStoreList(rangeID) + // When there are throttled stores that do match, we shouldn't send + // the replica to purgatory. + if throttledStoreCount > 0 { + return nil, errors.Errorf("%d matching stores are currently throttled", throttledStoreCount) } - // Because more redundancy is better than less, if relaxConstraints, the - // matching here is lenient, and tries to find a target by relaxing an - // attribute constraint, from last attribute to first. - for attrs := append([]config.Constraint(nil), constraints.Constraints...); ; attrs = attrs[:len(attrs)-1] { - sl, aliveStoreCount, throttledStoreCount := a.storePool.getStoreList( - config.Constraints{Constraints: attrs}, - rangeID, - a.options.Deterministic, - ) - if target := a.selectGood(sl, existingNodes); target != nil { - return target, nil - } + candidates, err := a.ruleSolver.Solve(sl, constraints, existing) + if err != nil { + return nil, err + } - // When there are throttled stores that do match, we shouldn't send - // the replica to purgatory or even consider relaxing the constraints. - if throttledStoreCount > 0 { - return nil, errors.Errorf("%d matching stores are currently throttled", throttledStoreCount) - } - if len(attrs) == 0 || !relaxConstraints { - return nil, &allocatorError{ - required: constraints.Constraints, - relaxConstraints: relaxConstraints, - aliveStoreCount: aliveStoreCount, - } + if len(candidates) == 0 { + return nil, &allocatorError{ + required: constraints.Constraints, } } + // TODO(bram): #10275 Need some randomness here! + return &candidates[0].store, nil } // RemoveTarget returns a suitable replica to remove from the provided replica // set. It attempts to consider which of the provided replicas would be the best // candidate for removal. It also will exclude any replica that belongs to the // range lease holder's store ID. -// -// TODO(mrtracy): removeTarget eventually needs to accept the attributes from -// the zone config associated with the provided replicas. This will allow it to -// make correct decisions in the case of ranges with heterogeneous replica -// requirements (i.e. multiple data centers). func (a Allocator) RemoveTarget( - existing []roachpb.ReplicaDescriptor, leaseStoreID roachpb.StoreID, + constraints config.Constraints, + existing []roachpb.ReplicaDescriptor, + leaseStoreID roachpb.StoreID, ) (roachpb.ReplicaDescriptor, error) { if len(existing) == 0 { return roachpb.ReplicaDescriptor{}, errors.Errorf("must supply at least one replica to allocator.RemoveTarget()") } - // Retrieve store descriptors for the provided replicas from the StorePool. - sl := StoreList{} + // TODO(bram): #10275 Is this getStoreList call required? Compute candidate + // requires a store list, but we should be able to create one using only + // the stores that belong to the range. + // Use an invalid range ID as we don't care about a corrupt replicas since + // as we are removing a replica and not trying to add one. + sl, _, _ := a.storePool.getStoreList(roachpb.RangeID(0)) + + var worst roachpb.ReplicaDescriptor + worstScore := math.MaxFloat64 for _, exist := range existing { if exist.StoreID == leaseStoreID { continue @@ -268,16 +224,25 @@ func (a Allocator) RemoveTarget( if !ok { continue } - sl.add(desc) + // If it's not a valid candidate, score will be zero. + candidate, _ := a.ruleSolver.computeCandidate(solveState{ + constraints: constraints, + store: desc, + existing: nil, + sl: sl, + tierOrder: canonicalTierOrder(sl), + tiers: storeTierMap(sl), + }) + if candidate.score < worstScore { + worstScore = candidate.score + worst = exist + } } - if bad := a.selectBad(sl); bad != nil { - for _, exist := range existing { - if exist.StoreID == bad.StoreID { - return exist, nil - } - } + if worstScore < math.MaxFloat64 { + return worst, nil } + return roachpb.ReplicaDescriptor{}, errors.Errorf("RemoveTarget() could not select an appropriate replica to be remove") } @@ -306,16 +271,12 @@ func (a Allocator) RebalanceTarget( existing []roachpb.ReplicaDescriptor, leaseStoreID roachpb.StoreID, rangeID roachpb.RangeID, -) *roachpb.StoreDescriptor { +) (*roachpb.StoreDescriptor, error) { if !a.options.AllowRebalance { - return nil + return nil, nil } - sl, _, _ := a.storePool.getStoreList( - constraints, - rangeID, - a.options.Deterministic, - ) + sl, _, _ := a.storePool.getStoreList(rangeID) if log.V(3) { log.Infof(context.TODO(), "rebalance-target (lease-holder=%d):\n%s", leaseStoreID, sl) } @@ -332,47 +293,100 @@ func (a Allocator) RebalanceTarget( } } if !shouldRebalance { - return nil + return nil, nil } - existingNodes := make(nodeIDSet, len(existing)) + // Load the exiting storesIDs into a map. + existingStoreIDs := make(map[roachpb.StoreID]struct{}) for _, repl := range existing { - existingNodes[repl.NodeID] = struct{}{} + existingStoreIDs[repl.StoreID] = struct{}{} } - return a.improve(sl, existingNodes) -} -// selectGood attempts to select a store from the supplied store list that it -// considers to be 'Good' relative to the other stores in the list. Any nodes -// in the supplied 'exclude' list will be disqualified from selection. Returns -// the selected store or nil if no such store can be found. -func (a Allocator) selectGood(sl StoreList, excluded nodeIDSet) *roachpb.StoreDescriptor { - rcb := rangeCountBalancer{a.randGen} - return rcb.selectGood(sl, excluded) -} + // Split the store list into existing and candidate stores lists. + existingStoreList := StoreList{} + candidateStoreList := StoreList{} + for _, store := range sl.stores { + if _, ok := existingStoreIDs[store.StoreID]; ok { + existingStoreList.add(store) + } else { + candidateStoreList.add(store) + } + } -// selectBad attempts to select a store from the supplied store list that it -// considers to be 'Bad' relative to the other stores in the list. Returns the -// selected store or nil if no such store can be found. -func (a Allocator) selectBad(sl StoreList) *roachpb.StoreDescriptor { - rcb := rangeCountBalancer{a.randGen} - return rcb.selectBad(sl) -} + existingCandidates, err := a.ruleSolver.Solve(existingStoreList, constraints, nil) + if err != nil { + return nil, err + } + candidates, err := a.ruleSolver.Solve(candidateStoreList, constraints, nil) + if err != nil { + return nil, err + } -// improve attempts to select an improvement over the given store from the -// stores in the given store list. Any nodes in the supplied 'exclude' list -// will be disqualified from selection. Returns the selected store, or nil if -// no such store can be found. -func (a Allocator) improve(sl StoreList, excluded nodeIDSet) *roachpb.StoreDescriptor { - rcb := rangeCountBalancer{a.randGen} - return rcb.improve(sl, excluded) + // Find all candidates that are better than the worst existing store. + var worstCandidateStore float64 + // If any store from existing is not included in existingCandidates, it was + // because it no longer meets the Constraints. So its score would be 0. + if len(existingCandidates) == len(existing) { + worstCandidateStore = existingCandidates[len(existingCandidates)-1].score + } + + // TODO(bram): #10275 Need some randomness here! + for _, cand := range candidates { + if cand.score > worstCandidateStore { + return &(candidates[0].store), nil + } + } + + return nil, nil } +// RebalanceThreshold is the minimum ratio of a store's range surplus to the +// mean range count that permits rebalances away from that store. +var RebalanceThreshold = envutil.EnvOrDefaultFloat("COCKROACH_REBALANCE_THRESHOLD", 0.05) + // shouldRebalance returns whether the specified store is a candidate for // having a replica removed from it given the candidate store list. func (a Allocator) shouldRebalance(store roachpb.StoreDescriptor, sl StoreList) bool { - rcb := rangeCountBalancer{a.randGen} - return rcb.shouldRebalance(store, sl) + // TODO(peter,bram,cuong): The FractionUsed check seems suspicious. When a + // node becomes fuller than maxFractionUsedThreshold we will always select it + // for rebalancing. This is currently utilized by tests. + maxCapacityUsed := store.Capacity.FractionUsed() >= maxFractionUsedThreshold + + // Rebalance if we're above the rebalance target, which is + // mean*(1+RebalanceThreshold). + target := int32(math.Ceil(sl.candidateCount.mean * (1 + RebalanceThreshold))) + rangeCountAboveTarget := store.Capacity.RangeCount > target + + // Rebalance if the candidate store has a range count above the mean, and + // there exists another store that is underfull: its range count is smaller + // than mean*(1-RebalanceThreshold). + var rebalanceToUnderfullStore bool + if float64(store.Capacity.RangeCount) > sl.candidateCount.mean { + underfullThreshold := int32(math.Floor(sl.candidateCount.mean * (1 - RebalanceThreshold))) + for _, desc := range sl.stores { + if desc.Capacity.RangeCount < underfullThreshold { + rebalanceToUnderfullStore = true + break + } + } + } + + // Require that moving a replica from the given store makes its range count + // converge on the mean range count. This only affects clusters with a + // small number of ranges. + rebalanceConvergesOnMean := float64(store.Capacity.RangeCount) > sl.candidateCount.mean+0.5 + + shouldRebalance := + (maxCapacityUsed || rangeCountAboveTarget || rebalanceToUnderfullStore) && rebalanceConvergesOnMean + if log.V(2) { + log.Infof(context.TODO(), + "%d: should-rebalance=%t: fraction-used=%.2f range-count=%d "+ + "(mean=%.1f, target=%d, fraction-used=%t, above-target=%t, underfull=%t, converges=%t)", + store.StoreID, shouldRebalance, store.Capacity.FractionUsed(), + store.Capacity.RangeCount, sl.candidateCount.mean, target, + maxCapacityUsed, rangeCountAboveTarget, rebalanceToUnderfullStore, rebalanceConvergesOnMean) + } + return shouldRebalance } // computeQuorum computes the quorum value for the given number of nodes. diff --git a/pkg/storage/allocator_test.go b/pkg/storage/allocator_test.go index 0405cb92a530..bc796b8dc238 100644 --- a/pkg/storage/allocator_test.go +++ b/pkg/storage/allocator_test.go @@ -21,6 +21,7 @@ package storage import ( "fmt" "math" + "math/rand" "reflect" "sort" "sync" @@ -190,14 +191,20 @@ func mockStorePool( storePool.mu.storeDetails = make(map[roachpb.StoreID]*storeDetail) for _, storeID := range aliveStoreIDs { - detail := newStoreDetail(context.TODO()) - detail.desc = &roachpb.StoreDescriptor{StoreID: storeID} + detail := newStoreDetail(context.Background()) + detail.desc = &roachpb.StoreDescriptor{ + StoreID: storeID, + Node: roachpb.NodeDescriptor{NodeID: roachpb.NodeID(storeID)}, + } storePool.mu.storeDetails[storeID] = detail } for _, storeID := range deadStoreIDs { detail := newStoreDetail(context.TODO()) detail.dead = true - detail.desc = &roachpb.StoreDescriptor{StoreID: storeID} + detail.desc = &roachpb.StoreDescriptor{ + StoreID: storeID, + Node: roachpb.NodeDescriptor{NodeID: roachpb.NodeID(storeID)}, + } storePool.mu.storeDetails[storeID] = detail } for storeID, detail := range storePool.mu.storeDetails { @@ -219,7 +226,6 @@ func TestAllocatorSimpleRetrieval(t *testing.T) { simpleZoneConfig.Constraints, []roachpb.ReplicaDescriptor{}, firstRange, - false, ) if err != nil { t.Fatalf("Unable to perform allocation: %v", err) @@ -251,7 +257,6 @@ func TestAllocatorCorruptReplica(t *testing.T) { simpleZoneConfig.Constraints, []roachpb.ReplicaDescriptor{}, firstRange, - true, ) if err != nil { t.Fatal(err) @@ -269,7 +274,6 @@ func TestAllocatorNoAvailableDisks(t *testing.T) { simpleZoneConfig.Constraints, []roachpb.ReplicaDescriptor{}, firstRange, - false, ) if result != nil { t.Errorf("expected nil result: %+v", result) @@ -288,7 +292,6 @@ func TestAllocatorTwoDatacenters(t *testing.T) { multiDCConfig.Constraints, []roachpb.ReplicaDescriptor{}, firstRange, - false, ) if err != nil { t.Fatalf("Unable to perform allocation: %v", err) @@ -300,7 +303,6 @@ func TestAllocatorTwoDatacenters(t *testing.T) { StoreID: result1.StoreID, }}, firstRange, - false, ) if err != nil { t.Fatalf("Unable to perform allocation: %v", err) @@ -324,7 +326,6 @@ func TestAllocatorTwoDatacenters(t *testing.T) { }, }, firstRange, - false, ) if err == nil { t.Errorf("expected error on allocation without available stores: %+v", result3) @@ -350,7 +351,6 @@ func TestAllocatorExistingReplica(t *testing.T) { }, }, firstRange, - false, ) if err != nil { t.Fatalf("Unable to perform allocation: %v", err) @@ -370,30 +370,121 @@ func TestAllocatorRelaxConstraints(t *testing.T) { gossiputil.NewStoreGossiper(g).GossipStores(multiDCStores, t) testCases := []struct { - required []config.Constraint // attribute strings - existing []int // existing store/node ID - relaxConstraints bool // allow constraints to be relaxed? - expID int // expected store/node ID on allocate - expErr bool + required []config.Constraint // attribute strings + existing []int // existing store/node ID + expID int // expected store/node ID on allocate + expErr bool }{ // The two stores in the system have attributes: // storeID=1 {"a", "ssd"} // storeID=2 {"b", "ssd"} - {[]config.Constraint{{Value: "a"}, {Value: "ssd"}}, []int{}, true, 1, false}, - {[]config.Constraint{{Value: "a"}, {Value: "ssd"}}, []int{1}, true, 2, false}, - {[]config.Constraint{{Value: "a"}, {Value: "ssd"}}, []int{1}, false, 0, true}, - {[]config.Constraint{{Value: "a"}, {Value: "ssd"}}, []int{1, 2}, true, 0, true}, - {[]config.Constraint{{Value: "b"}, {Value: "ssd"}}, []int{}, true, 2, false}, - {[]config.Constraint{{Value: "b"}, {Value: "ssd"}}, []int{1}, true, 2, false}, - {[]config.Constraint{{Value: "b"}, {Value: "ssd"}}, []int{2}, false, 0, true}, - {[]config.Constraint{{Value: "b"}, {Value: "ssd"}}, []int{2}, true, 1, false}, - {[]config.Constraint{{Value: "b"}, {Value: "ssd"}}, []int{1, 2}, true, 0, true}, - {[]config.Constraint{{Value: "b"}, {Value: "hdd"}}, []int{}, true, 2, false}, - {[]config.Constraint{{Value: "b"}, {Value: "hdd"}}, []int{2}, true, 1, false}, - {[]config.Constraint{{Value: "b"}, {Value: "hdd"}}, []int{2}, false, 0, true}, - {[]config.Constraint{{Value: "b"}, {Value: "hdd"}}, []int{1, 2}, true, 0, true}, - {[]config.Constraint{{Value: "b"}, {Value: "ssd"}, {Value: "gpu"}}, []int{}, true, 2, false}, - {[]config.Constraint{{Value: "b"}, {Value: "hdd"}, {Value: "gpu"}}, []int{}, true, 2, false}, + { + []config.Constraint{ + {Value: "a"}, + {Value: "ssd"}, + }, + []int{}, 1, false, + }, + { + []config.Constraint{ + {Value: "a"}, + {Value: "ssd"}, + }, + []int{1}, 2, false, + }, + { + []config.Constraint{ + {Value: "a", Type: config.Constraint_REQUIRED}, + {Value: "ssd", Type: config.Constraint_REQUIRED}, + }, + []int{1}, 0, true, + }, + { + []config.Constraint{ + {Value: "a"}, + {Value: "ssd"}, + }, + []int{1, 2}, 0, true, + }, + { + []config.Constraint{ + {Value: "b"}, + {Value: "ssd"}, + }, + []int{}, 2, false, + }, + { + []config.Constraint{ + {Value: "b"}, + {Value: "ssd"}, + }, + []int{1}, 2, false, + }, + { + []config.Constraint{ + {Value: "b", Type: config.Constraint_REQUIRED}, + {Value: "ssd", Type: config.Constraint_REQUIRED}, + }, + []int{2}, 0, true, + }, + { + []config.Constraint{ + {Value: "b"}, + {Value: "ssd"}, + }, + []int{2}, 1, false, + }, + { + []config.Constraint{ + {Value: "b"}, + {Value: "ssd"}, + }, + []int{1, 2}, 0, true, + }, + { + []config.Constraint{ + {Value: "b"}, + {Value: "hdd"}, + }, + []int{}, 2, false, + }, + { + []config.Constraint{ + {Value: "b"}, + {Value: "hdd"}, + }, + []int{2}, 1, false, + }, + { + []config.Constraint{ + {Value: "b", Type: config.Constraint_REQUIRED}, + {Value: "hdd", Type: config.Constraint_REQUIRED}, + }, + []int{2}, 0, true, + }, + { + []config.Constraint{ + {Value: "b"}, + {Value: "hdd"}, + }, + []int{1, 2}, 0, true, + }, + { + []config.Constraint{ + {Value: "b"}, + {Value: "ssd"}, + {Value: "gpu"}, + }, + []int{}, 2, false, + }, + { + []config.Constraint{ + {Value: "b"}, + {Value: "hdd"}, + {Value: "gpu"}, + }, + []int{}, 2, false, + }, } for i, test := range testCases { var existing []roachpb.ReplicaDescriptor @@ -401,12 +492,7 @@ func TestAllocatorRelaxConstraints(t *testing.T) { existing = append(existing, roachpb.ReplicaDescriptor{NodeID: roachpb.NodeID(id), StoreID: roachpb.StoreID(id)}) } constraints := config.Constraints{Constraints: test.required} - result, err := a.AllocateTarget( - constraints, - existing, - firstRange, - test.relaxConstraints, - ) + result, err := a.AllocateTarget(constraints, existing, firstRange) if haveErr := (err != nil); haveErr != test.expErr { t.Errorf("%d: expected error %t; got %t: %s", i, test.expErr, haveErr, err) } else if err == nil && roachpb.StoreID(test.expID) != result.StoreID { @@ -468,12 +554,15 @@ func TestAllocatorRebalance(t *testing.T) { // Every rebalance target must be either stores 1 or 2. for i := 0; i < 10; i++ { - result := a.RebalanceTarget( + result, err := a.RebalanceTarget( config.Constraints{}, []roachpb.ReplicaDescriptor{{StoreID: 3}}, noStore, firstRange, ) + if err != nil { + t.Fatal(err) + } if result == nil { t.Fatal("nil result") } @@ -483,17 +572,12 @@ func TestAllocatorRebalance(t *testing.T) { } // Verify shouldRebalance results. - a.options.Deterministic = true for i, store := range stores { desc, ok := a.storePool.getStoreDescriptor(store.StoreID) if !ok { t.Fatalf("%d: unable to get store %d descriptor", i, store.StoreID) } - sl, _, _ := a.storePool.getStoreList( - config.Constraints{}, - firstRange, - true, - ) + sl, _, _ := a.storePool.getStoreList(firstRange) result := a.shouldRebalance(desc, sl) if expResult := (i >= 2); expResult != result { t.Errorf("%d: expected rebalance %t; got %t", i, expResult, result) @@ -584,8 +668,8 @@ func TestAllocatorRebalanceThrashing(t *testing.T) { t.Fatalf("%d: numStores %d < min %d", i, numStores, minStores) } stopper, g, _, a, _ := createTestAllocator() - a.options.Deterministic = true defer stopper.Stop() + a.storePool.TestSetDeterministic(true) // Create stores with the range counts from the test case and gossip them. var stores []*roachpb.StoreDescriptor @@ -600,11 +684,7 @@ func TestAllocatorRebalanceThrashing(t *testing.T) { // Ensure gossiped store descriptor changes have propagated. util.SucceedsSoon(t, func() error { - sl, _, _ := a.storePool.getStoreList( - config.Constraints{}, - firstRange, - true, - ) + sl, _, _ := a.storePool.getStoreList(firstRange) for j, s := range sl.stores { if a, e := s.Capacity.RangeCount, tc[j].rangeCount; a != e { return errors.Errorf("tc %d: range count for %d = %d != expected %d", i, j, a, e) @@ -612,11 +692,7 @@ func TestAllocatorRebalanceThrashing(t *testing.T) { } return nil }) - sl, _, _ := a.storePool.getStoreList( - config.Constraints{}, - firstRange, - true, - ) + sl, _, _ := a.storePool.getStoreList(firstRange) // Verify shouldRebalance returns the expected value. for j, store := range stores { @@ -666,29 +742,27 @@ func TestAllocatorRebalanceByCount(t *testing.T) { // Every rebalance target must be store 4 (or nil for case of missing the only option). for i := 0; i < 10; i++ { - result := a.RebalanceTarget( + result, err := a.RebalanceTarget( config.Constraints{}, - []roachpb.ReplicaDescriptor{{StoreID: stores[0].StoreID}}, + []roachpb.ReplicaDescriptor{{StoreID: 1}}, stores[0].StoreID, firstRange, ) + if err != nil { + t.Fatal(err) + } if result != nil && result.StoreID != 4 { t.Errorf("expected store 4; got %d", result.StoreID) } } // Verify shouldRebalance results. - a.options.Deterministic = true for i, store := range stores { desc, ok := a.storePool.getStoreDescriptor(store.StoreID) if !ok { t.Fatalf("%d: unable to get store %d descriptor", i, store.StoreID) } - sl, _, _ := a.storePool.getStoreList( - config.Constraints{}, - firstRange, - true, - ) + sl, _, _ := a.storePool.getStoreList(firstRange) result := a.shouldRebalance(desc, sl) if expResult := (i < 3); expResult != result { t.Errorf("%d: expected rebalance %t; got %t", i, expResult, result) @@ -754,7 +828,7 @@ func TestAllocatorRemoveTarget(t *testing.T) { sg := gossiputil.NewStoreGossiper(g) sg.GossipStores(stores, t) - targetRepl, err := a.RemoveTarget(replicas, stores[0].StoreID) + targetRepl, err := a.RemoveTarget(config.Constraints{}, replicas, stores[0].StoreID) if err != nil { t.Fatal(err) } @@ -764,7 +838,7 @@ func TestAllocatorRemoveTarget(t *testing.T) { // Now perform the same test, but pass in the store ID of store 3 so it's // excluded. - targetRepl, err = a.RemoveTarget(replicas, stores[2].StoreID) + targetRepl, err = a.RemoveTarget(config.Constraints{}, replicas, stores[2].StoreID) if err != nil { t.Fatal(err) } @@ -1195,7 +1269,6 @@ func TestAllocatorThrottled(t *testing.T) { simpleZoneConfig.Constraints, []roachpb.ReplicaDescriptor{}, firstRange, - false, ) if _, ok := err.(purgatoryError); !ok { t.Fatalf("expected a purgatory error, got: %v", err) @@ -1207,7 +1280,6 @@ func TestAllocatorThrottled(t *testing.T) { simpleZoneConfig.Constraints, []roachpb.ReplicaDescriptor{}, firstRange, - false, ) if err != nil { t.Fatalf("unable to perform allocation: %v", err) @@ -1229,7 +1301,6 @@ func TestAllocatorThrottled(t *testing.T) { simpleZoneConfig.Constraints, []roachpb.ReplicaDescriptor{}, firstRange, - false, ) if _, ok := err.(purgatoryError); ok { t.Fatalf("expected a non purgatory error, got: %v", err) @@ -1272,13 +1343,18 @@ func Example_rebalancing() { TestTimeUntilStoreDeadOff, stopper, ) - alloc := MakeAllocator(sp, AllocatorOptions{AllowRebalance: true, Deterministic: true}) + sp.TestSetDeterministic(true) + alloc := MakeAllocator(sp, AllocatorOptions{AllowRebalance: true}) var wg sync.WaitGroup g.RegisterCallback(gossip.MakePrefixPattern(gossip.KeyStorePrefix), func(_ string, _ roachpb.Value) { wg.Done() }) - const generations = 100 const nodes = 20 + const generations = 100 + const printGenerations = generations / 2 + const generationToStopAdding = generations * 9 / 10 + + randGen := rand.New(rand.NewSource(777)) // Initialize testStores. var testStores [nodes]testStore @@ -1288,16 +1364,21 @@ func Example_rebalancing() { testStores[i].Capacity = roachpb.StoreCapacity{Capacity: 1 << 30, Available: 1 << 30} } // Initialize the cluster with a single range. - testStores[0].add(alloc.randGen.Int63n(1 << 20)) + testStores[0].add(randGen.Int63n(1 << 20)) for i := 0; i < generations; i++ { - // First loop through test stores and add data. + if i < generationToStopAdding { + // First loop through test stores and add data. + for j := 0; j < len(testStores); j++ { + // Add a pretend range to the testStore if there's already one. + if testStores[j].Capacity.RangeCount > 0 { + testStores[j].add(randGen.Int63n(1 << 20)) + } + } + } + // Gossip the new store info. wg.Add(len(testStores)) for j := 0; j < len(testStores); j++ { - // Add a pretend range to the testStore if there's already one. - if testStores[j].Capacity.RangeCount > 0 { - testStores[j].add(alloc.randGen.Int63n(1 << 20)) - } if err := g.AddInfoProto(gossip.MakeStoreKey(roachpb.StoreID(j)), &testStores[j].StoreDescriptor, 0); err != nil { panic(err) } @@ -1307,14 +1388,17 @@ func Example_rebalancing() { // Next loop through test stores and maybe rebalance. for j := 0; j < len(testStores); j++ { ts := &testStores[j] - target := alloc.RebalanceTarget( + target, err := alloc.RebalanceTarget( config.Constraints{}, []roachpb.ReplicaDescriptor{{NodeID: ts.Node.NodeID, StoreID: ts.StoreID}}, noStore, firstRange, ) + if err != nil { + panic(err) + } if target != nil { - testStores[j].rebalance(&testStores[int(target.StoreID)], alloc.randGen.Int63n(1<<20)) + testStores[j].rebalance(&testStores[int(target.StoreID)], randGen.Int63n(1<<20)) } } @@ -1350,55 +1434,55 @@ func Example_rebalancing() { fmt.Printf("Total bytes=%d, ranges=%d\n", totBytes, totRanges) // Output: - // 999 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 - // 999 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 - // 999 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 - // 999 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 - // 999 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 - // 999 000 000 000 014 000 000 118 000 000 000 000 111 000 000 000 000 000 000 000 - // 999 113 095 000 073 064 000 221 003 000 020 178 182 000 057 000 027 000 055 000 - // 999 398 222 000 299 366 000 525 239 135 263 385 424 261 261 000 260 194 207 322 - // 999 423 307 294 401 286 292 648 294 426 388 454 511 445 162 521 179 403 280 581 - // 999 396 446 333 445 481 408 602 351 418 492 578 603 526 193 553 279 444 385 568 - // 999 511 598 392 572 526 515 741 441 500 641 672 802 541 310 698 421 447 466 577 - // 999 611 726 528 721 640 564 804 524 568 721 743 811 558 433 706 541 588 500 678 - // 999 668 764 582 716 696 604 832 594 572 695 690 828 607 497 728 595 682 609 689 - // 999 635 729 536 706 736 596 764 614 561 674 659 831 595 492 740 564 732 592 683 - // 999 726 848 539 794 806 676 750 669 637 675 711 930 684 558 750 654 748 658 764 - // 999 664 847 560 811 757 658 748 674 628 694 660 896 647 561 729 704 754 652 775 - // 999 693 901 587 826 799 671 756 655 649 702 727 923 645 600 712 767 816 738 800 - // 999 712 964 600 820 768 705 762 630 698 708 774 929 636 583 725 835 866 734 819 - // 999 734 996 666 816 765 735 809 612 728 687 800 942 625 562 730 816 922 758 834 - // 999 750 956 647 834 771 761 776 616 759 696 799 952 622 576 732 808 963 732 839 - // 999 780 980 699 792 779 736 827 668 762 672 778 986 608 578 732 849 943 727 861 - // 999 749 929 686 770 754 726 803 671 723 723 774 996 628 592 728 862 945 734 903 - // 999 736 886 669 770 716 714 794 654 710 694 725 985 599 621 732 849 924 692 873 - // 999 740 900 699 801 752 747 815 679 717 715 770 962 612 639 773 882 923 717 882 - // 999 810 923 735 815 776 772 823 703 775 750 818 963 637 667 814 891 949 746 933 - // 999 791 882 723 827 760 774 795 671 756 761 777 941 636 654 809 858 932 714 896 - // 999 804 893 726 836 764 752 806 663 747 778 780 958 622 652 812 861 928 724 908 - // 999 819 898 760 875 804 777 809 669 768 809 799 959 617 682 825 879 939 748 910 - // 999 827 882 740 878 834 779 841 702 784 816 828 950 631 689 810 853 915 757 938 - // 999 835 885 759 882 837 762 835 738 791 832 823 953 648 705 816 872 932 763 958 - // 999 838 878 756 880 843 802 850 749 807 838 813 975 683 735 838 888 944 780 967 - // 999 837 883 759 900 826 814 844 752 795 821 792 944 686 750 832 881 925 754 969 - // 999 880 905 784 920 854 834 883 765 837 835 794 958 726 799 854 885 971 776 971 - // 999 897 906 792 926 849 832 894 785 869 852 799 969 735 805 864 909 949 799 975 - // 999 874 888 781 905 844 833 894 787 867 836 792 962 720 806 856 918 943 783 943 - // 999 891 871 756 907 823 836 896 800 844 843 799 934 725 818 836 925 956 758 943 - // 999 901 888 782 893 842 838 894 806 858 838 801 934 742 821 839 947 938 761 931 - // 999 930 909 811 905 872 846 912 812 887 877 816 965 766 844 864 975 953 782 960 - // 999 917 895 810 903 860 862 927 800 886 881 831 954 753 840 869 983 940 774 948 - // 999 920 910 828 894 853 873 911 801 908 893 821 966 757 850 867 987 944 790 951 - // 994 927 918 835 915 876 888 910 808 907 909 843 966 762 844 884 999 941 799 936 - // 999 930 919 848 910 869 901 921 808 897 888 840 967 780 857 888 980 935 793 917 - // 999 917 918 835 903 870 910 913 800 897 873 830 960 765 866 877 971 937 793 915 - // 999 909 904 831 874 875 883 896 791 896 863 833 926 758 841 879 959 932 772 904 - // 999 940 902 849 882 888 899 920 812 918 859 844 953 776 857 896 981 963 775 901 - // 999 926 887 849 869 876 886 898 813 894 834 823 947 762 839 884 979 955 785 890 - // 999 924 881 855 866 867 870 905 820 888 819 805 948 758 836 877 967 938 782 897 - // 999 929 896 863 895 873 874 917 823 913 831 818 964 783 848 882 974 952 791 892 - // 999 941 908 860 888 869 894 921 835 917 826 830 953 785 872 894 970 963 810 901 - // 999 917 895 848 871 856 896 913 831 910 828 832 951 787 875 873 952 947 800 891 - // Total bytes=915403982, ranges=1748 + // 999 129 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 + // 999 758 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 657 625 + // 432 999 000 000 000 000 000 000 000 000 000 000 000 000 257 036 428 845 930 420 + // 193 185 000 000 000 000 559 109 000 000 000 000 153 999 051 134 248 388 349 218 + // 268 301 000 000 000 000 543 163 484 357 415 000 282 999 193 168 362 573 352 324 + // 320 399 000 000 412 598 564 348 440 443 729 000 465 999 311 385 444 771 454 463 + // 284 241 377 999 231 431 338 266 186 275 508 095 316 519 283 204 283 555 330 384 + // 419 435 599 999 289 477 482 418 328 212 622 516 383 536 426 356 463 700 391 588 + // 636 554 727 999 370 655 596 586 489 335 717 601 512 734 572 528 610 797 499 767 + // 626 648 883 999 451 716 655 596 604 387 853 723 612 753 703 590 728 894 625 850 + // 660 697 898 999 468 850 797 660 620 470 918 704 684 714 793 674 844 929 690 806 + // 694 736 965 989 499 909 805 692 729 593 961 702 723 720 913 721 909 999 854 876 + // 682 724 918 951 464 837 770 652 712 611 905 723 739 665 826 618 835 999 864 796 + // 737 739 924 949 573 840 789 680 706 659 929 819 764 721 843 589 868 999 891 870 + // 777 795 952 998 637 852 837 739 733 737 947 843 832 796 912 664 897 999 906 839 + // 760 809 929 961 658 859 780 776 720 738 900 826 797 833 911 703 883 999 930 834 + // 737 850 942 999 677 907 767 819 738 754 958 832 845 808 906 722 931 986 978 817 + // 762 846 935 999 688 913 773 821 733 815 954 870 866 792 968 764 992 986 974 874 + // 771 828 933 965 681 878 804 814 735 869 936 855 856 752 951 763 999 995 969 877 + // 723 817 862 931 673 824 752 824 707 832 857 793 834 731 887 731 999 934 926 871 + // 706 825 840 925 682 824 747 780 715 818 853 795 827 700 842 703 999 882 877 875 + // 739 835 876 958 731 887 763 823 740 839 881 849 873 737 889 763 999 926 925 929 + // 773 882 904 971 768 926 841 847 822 843 940 887 941 765 898 814 999 960 998 993 + // 764 872 905 999 773 915 859 843 848 865 909 871 944 739 922 780 974 977 992 993 + // 732 876 911 999 765 888 891 856 848 831 901 876 948 753 889 773 989 967 989 997 + // 733 856 936 981 797 901 924 868 876 820 930 889 958 770 867 785 984 963 989 999 + // 737 836 960 991 787 895 912 875 856 822 930 907 974 778 849 792 965 950 999 988 + // 742 839 970 999 803 925 918 894 884 812 952 901 977 809 872 764 985 961 979 974 + // 754 841 969 987 822 934 938 922 904 830 960 911 990 804 879 781 975 938 999 986 + // 735 842 966 980 805 909 920 938 888 842 967 913 999 782 882 783 994 932 985 980 + // 759 857 971 999 826 927 912 951 872 864 976 920 976 781 899 771 998 947 963 959 + // 756 849 965 999 818 924 895 916 856 861 962 898 944 771 906 743 966 942 945 938 + // 743 850 963 999 812 919 870 922 876 844 977 885 934 773 909 725 958 940 929 943 + // 757 831 951 999 798 918 854 933 865 832 979 852 907 793 920 728 956 931 906 934 + // 775 823 927 999 792 907 840 925 871 826 954 837 909 807 909 711 958 909 920 930 + // 772 843 933 999 788 902 841 920 890 843 941 830 895 804 908 735 951 927 921 929 + // 799 851 938 999 805 914 868 920 903 836 937 832 911 811 914 764 949 927 903 930 + // 828 877 952 999 817 945 869 915 907 849 974 856 936 827 925 790 966 944 929 937 + // 843 888 971 999 844 955 892 913 925 856 990 871 968 854 937 795 977 950 937 966 + // 859 911 980 994 863 960 899 925 940 863 999 884 989 855 948 809 989 949 945 966 + // 847 905 980 974 848 955 888 924 940 849 999 877 981 855 938 786 997 940 947 965 + // 844 895 973 966 853 935 876 917 935 851 991 861 978 833 946 788 999 935 942 964 + // 849 886 975 966 865 930 890 916 927 855 988 870 994 841 943 787 999 935 963 963 + // 843 887 973 951 860 933 874 910 909 865 966 875 984 830 933 779 999 922 967 960 + // 836 879 983 945 863 923 890 913 893 867 964 894 982 836 914 783 999 940 969 967 + // 830 879 981 943 853 926 886 900 894 863 947 896 976 832 903 771 999 935 972 967 + // 830 879 981 943 853 926 886 900 894 863 947 896 976 832 903 771 999 935 972 967 + // 830 879 981 943 853 926 886 900 894 863 947 896 976 832 903 771 999 935 972 967 + // 830 879 981 943 853 926 886 900 894 863 947 896 976 832 903 771 999 935 972 967 + // 830 879 981 943 853 926 886 900 894 863 947 896 976 832 903 771 999 935 972 967 + // Total bytes=872399094, ranges=1668 } diff --git a/pkg/storage/balancer.go b/pkg/storage/balancer.go deleted file mode 100644 index 1170a37209c7..000000000000 --- a/pkg/storage/balancer.go +++ /dev/null @@ -1,218 +0,0 @@ -// Copyright 2014 The Cockroach Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. See the License for the specific language governing -// permissions and limitations under the License. -// -// Author: Matt Tracy (matt@cockroachlabs.com) - -package storage - -import ( - "bytes" - "fmt" - "math" - - "golang.org/x/net/context" - - "github.com/cockroachdb/cockroach/pkg/roachpb" - "github.com/cockroachdb/cockroach/pkg/util/envutil" - "github.com/cockroachdb/cockroach/pkg/util/log" -) - -const allocatorRandomCount = 10 - -type nodeIDSet map[roachpb.NodeID]struct{} - -func formatCandidates( - selected *roachpb.StoreDescriptor, candidates []roachpb.StoreDescriptor, -) string { - var buf bytes.Buffer - _, _ = buf.WriteString("[") - for i := range candidates { - candidate := &candidates[i] - if i > 0 { - _, _ = buf.WriteString(" ") - } - fmt.Fprintf(&buf, "%d:%d", candidate.StoreID, candidate.Capacity.RangeCount) - if candidate == selected { - _, _ = buf.WriteString("*") - } - } - _, _ = buf.WriteString("]") - return buf.String() -} - -// rangeCountBalancer attempts to balance ranges across the cluster while -// considering only the number of ranges being serviced each store. -type rangeCountBalancer struct { - rand allocatorRand -} - -func (rangeCountBalancer) selectBest(sl StoreList) *roachpb.StoreDescriptor { - var best *roachpb.StoreDescriptor - for i := range sl.stores { - candidate := &sl.stores[i] - if best == nil { - best = candidate - continue - } - if candidate.Capacity.RangeCount < best.Capacity.RangeCount { - best = candidate - } - } - - // NB: logging of the best candidate is performed by the caller (selectGood - // or improve). - return best -} - -func (rcb rangeCountBalancer) selectGood(sl StoreList, excluded nodeIDSet) *roachpb.StoreDescriptor { - // Consider a random sample of stores from the store list. - sl.stores = selectRandom(rcb.rand, allocatorRandomCount, sl, excluded) - good := rcb.selectBest(sl) - - if log.V(2) { - log.Infof(context.TODO(), "selected good: mean=%.1f %s", - sl.candidateCount.mean, formatCandidates(good, sl.stores)) - } - return good -} - -func (rangeCountBalancer) selectBad(sl StoreList) *roachpb.StoreDescriptor { - var worst *roachpb.StoreDescriptor - for i := range sl.stores { - candidate := &sl.stores[i] - if worst == nil { - worst = candidate - continue - } - if candidate.Capacity.RangeCount > worst.Capacity.RangeCount { - worst = candidate - } - } - - if log.V(2) { - log.Infof(context.TODO(), "selected bad: mean=%.1f %s", - sl.candidateCount.mean, formatCandidates(worst, sl.stores)) - } - return worst -} - -// improve returns a candidate StoreDescriptor to rebalance a replica to. The -// strategy is to always converge on the mean range count. If that isn't -// possible, we don't return any candidate. -func (rcb rangeCountBalancer) improve(sl StoreList, excluded nodeIDSet) *roachpb.StoreDescriptor { - // Attempt to select a better candidate from the supplied list. - sl.stores = selectRandom(rcb.rand, allocatorRandomCount, sl, excluded) - candidate := rcb.selectBest(sl) - if candidate == nil { - if log.V(2) { - log.Infof(context.TODO(), "not rebalancing: no valid candidate targets: %s", - formatCandidates(nil, sl.stores)) - } - return nil - } - - // Adding a replica to the candidate must make its range count converge on the - // mean range count. - rebalanceConvergesOnMean := float64(candidate.Capacity.RangeCount) < sl.candidateCount.mean-0.5 - if !rebalanceConvergesOnMean { - if log.V(2) { - log.Infof(context.TODO(), "not rebalancing: %s wouldn't converge on the mean %.1f", - formatCandidates(candidate, sl.stores), sl.candidateCount.mean) - } - return nil - } - - if log.V(2) { - log.Infof(context.TODO(), "rebalancing: mean=%.1f %s", - sl.candidateCount.mean, formatCandidates(candidate, sl.stores)) - } - return candidate -} - -// RebalanceThreshold is the minimum ratio of a store's range surplus to the -// mean range count that permits rebalances away from that store. -var RebalanceThreshold = envutil.EnvOrDefaultFloat("COCKROACH_REBALANCE_THRESHOLD", 0.05) - -func (rangeCountBalancer) shouldRebalance(store roachpb.StoreDescriptor, sl StoreList) bool { - // TODO(peter,bram,cuong): The FractionUsed check seems suspicious. When a - // node becomes fuller than maxFractionUsedThreshold we will always select it - // for rebalancing. This is currently utilized by tests. - maxCapacityUsed := store.Capacity.FractionUsed() >= maxFractionUsedThreshold - - // Rebalance if we're above the rebalance target, which is - // mean*(1+RebalanceThreshold). - target := int32(math.Ceil(sl.candidateCount.mean * (1 + RebalanceThreshold))) - rangeCountAboveTarget := store.Capacity.RangeCount > target - - // Rebalance if the candidate store has a range count above the mean, and - // there exists another store that is underfull: its range count is smaller - // than mean*(1-RebalanceThreshold). - var rebalanceToUnderfullStore bool - if float64(store.Capacity.RangeCount) > sl.candidateCount.mean { - underfullThreshold := int32(math.Floor(sl.candidateCount.mean * (1 - RebalanceThreshold))) - for _, desc := range sl.stores { - if desc.Capacity.RangeCount < underfullThreshold { - rebalanceToUnderfullStore = true - break - } - } - } - - // Require that moving a replica from the given store makes its range count - // converge on the mean range count. This only affects clusters with a - // small number of ranges. - rebalanceConvergesOnMean := float64(store.Capacity.RangeCount) > sl.candidateCount.mean+0.5 - - shouldRebalance := - (maxCapacityUsed || rangeCountAboveTarget || rebalanceToUnderfullStore) && rebalanceConvergesOnMean - if log.V(2) { - log.Infof(context.TODO(), - "%d: should-rebalance=%t: fraction-used=%.2f range-count=%d "+ - "(mean=%.1f, target=%d, fraction-used=%t, above-target=%t, underfull=%t, converges=%t)", - store.StoreID, shouldRebalance, store.Capacity.FractionUsed(), - store.Capacity.RangeCount, sl.candidateCount.mean, target, - maxCapacityUsed, rangeCountAboveTarget, rebalanceToUnderfullStore, rebalanceConvergesOnMean) - } - return shouldRebalance -} - -// selectRandom chooses up to count random store descriptors from the given -// store list, excluding any stores that are too full to accept more replicas. -func selectRandom( - randGen allocatorRand, count int, sl StoreList, excluded nodeIDSet, -) []roachpb.StoreDescriptor { - var descs []roachpb.StoreDescriptor - // Randomly permute available stores matching the required attributes. - randGen.Lock() - defer randGen.Unlock() - for _, idx := range randGen.Perm(len(sl.stores)) { - desc := sl.stores[idx] - // Skip if store is in excluded set. - if _, ok := excluded[desc.Node.NodeID]; ok { - continue - } - - // Don't overfill stores. - if desc.Capacity.FractionUsed() > maxFractionUsedThreshold { - continue - } - - // Add this store; exit loop if we've satisfied count. - descs = append(descs, sl.stores[idx]) - if len(descs) >= count { - break - } - } - return descs -} diff --git a/pkg/storage/client_raft_test.go b/pkg/storage/client_raft_test.go index f6cea34d39a7..cbcce84ac230 100644 --- a/pkg/storage/client_raft_test.go +++ b/pkg/storage/client_raft_test.go @@ -1831,7 +1831,6 @@ func TestStoreRangeRebalance(t *testing.T) { sc := storage.TestStoreConfig() sc.AllocatorOptions = storage.AllocatorOptions{ AllowRebalance: true, - Deterministic: true, } mtc := &multiTestContext{storeConfig: &sc} @@ -1915,7 +1914,8 @@ func TestStoreRangeRebalance(t *testing.T) { // Exit when all stores have a single replica. actual := countReplicas() if !reflect.DeepEqual(expected, actual) { - return errors.Errorf("replicas are not distributed as expected %s", pretty.Diff(expected, actual)) + return errors.Errorf("replicas are not distributed as expected %s", + pretty.Diff(expected, actual)) } return nil }) diff --git a/pkg/storage/client_test.go b/pkg/storage/client_test.go index 85ce8e15452d..f38f311bba7c 100644 --- a/pkg/storage/client_test.go +++ b/pkg/storage/client_test.go @@ -44,7 +44,6 @@ import ( "google.golang.org/grpc" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/config" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/gossip/resolver" "github.com/cockroachdb/cockroach/pkg/internal/client" @@ -405,11 +404,7 @@ func (m *multiTestContext) initGossipNetwork() { m.gossipStores() util.SucceedsSoon(m.t, func() error { for i := 0; i < len(m.stores); i++ { - _, alive, _ := m.storePools[i].GetStoreList( - config.Constraints{}, - roachpb.RangeID(0), - /* deterministic */ false, - ) + _, alive, _ := m.storePools[i].GetStoreList(roachpb.RangeID(0)) if alive != len(m.stores) { return errors.Errorf("node %d's store pool only has %d alive stores, expected %d", m.stores[i].Ident.NodeID, alive, len(m.stores)) diff --git a/pkg/storage/helpers_test.go b/pkg/storage/helpers_test.go index 2ff71bfcac03..4c97e49c27cf 100644 --- a/pkg/storage/helpers_test.go +++ b/pkg/storage/helpers_test.go @@ -27,7 +27,6 @@ import ( "golang.org/x/net/context" - "github.com/cockroachdb/cockroach/pkg/config" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/storage/engine/enginepb" @@ -200,10 +199,8 @@ func (r *Replica) GetTimestampCacheLowWater() hlc.Timestamp { } // GetStoreList is the same function as GetStoreList exposed for tests only. -func (sp *StorePool) GetStoreList( - constraints config.Constraints, rangeID roachpb.RangeID, deterministic bool, -) (StoreList, int, int) { - return sp.getStoreList(constraints, rangeID, deterministic) +func (sp *StorePool) GetStoreList(rangeID roachpb.RangeID) (StoreList, int, int) { + return sp.getStoreList(rangeID) } // IsQuiescent returns whether the replica is quiescent or not. diff --git a/pkg/storage/replicate_queue.go b/pkg/storage/replicate_queue.go index 0d1792c806b8..e2c99bd99bba 100644 --- a/pkg/storage/replicate_queue.go +++ b/pkg/storage/replicate_queue.go @@ -119,12 +119,15 @@ func (rq *replicateQueue) shouldQueue( if lease, _ := repl.getLease(); lease != nil { leaseStoreID = lease.Replica.StoreID } - target := rq.allocator.RebalanceTarget( + target, err := rq.allocator.RebalanceTarget( zone.Constraints, desc.Replicas, leaseStoreID, desc.RangeID, ) + if err != nil { + return false, 0 + } if log.V(2) { if target != nil { log.Infof(ctx, "%s rebalance target found, enqueuing", repl) @@ -158,12 +161,7 @@ func (rq *replicateQueue) process( switch action { case AllocatorAdd: log.Event(ctx, "adding a new replica") - newStore, err := rq.allocator.AllocateTarget( - zone.Constraints, - desc.Replicas, - desc.RangeID, - true, - ) + newStore, err := rq.allocator.AllocateTarget(zone.Constraints, desc.Replicas, desc.RangeID) if err != nil { return err } @@ -180,7 +178,7 @@ func (rq *replicateQueue) process( log.Event(ctx, "removing a replica") // We require the lease in order to process replicas, so // repl.store.StoreID() corresponds to the lease-holder's store ID. - removeReplica, err := rq.allocator.RemoveTarget(desc.Replicas, repl.store.StoreID()) + removeReplica, err := rq.allocator.RemoveTarget(zone.Constraints, desc.Replicas, repl.store.StoreID()) if err != nil { return err } @@ -212,17 +210,17 @@ func (rq *replicateQueue) process( // // We require the lease in order to process replicas, so // repl.store.StoreID() corresponds to the lease-holder's store ID. - rebalanceStore := rq.allocator.RebalanceTarget( + rebalanceStore, err := rq.allocator.RebalanceTarget( zone.Constraints, desc.Replicas, repl.store.StoreID(), desc.RangeID, ) - if rebalanceStore == nil { - log.VEventf(ctx, 1, "no suitable rebalance target") + if rebalanceStore == nil || err != nil { + log.VEventf(ctx, 1, "%s: no suitable rebalance target", repl) // No action was necessary and no rebalance target was found. Return // without re-queuing this replica. - return nil + return err } rebalanceReplica := roachpb.ReplicaDescriptor{ NodeID: rebalanceStore.Node.NodeID, diff --git a/pkg/storage/rule_solver.go b/pkg/storage/rule_solver.go new file mode 100644 index 000000000000..73369b34b886 --- /dev/null +++ b/pkg/storage/rule_solver.go @@ -0,0 +1,269 @@ +// Copyright 2016 The Cockroach Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. +// +// Author: Tristan Rice (rice@fn.lc) + +package storage + +import ( + "math" + "sort" + + "github.com/cockroachdb/cockroach/pkg/config" + "github.com/cockroachdb/cockroach/pkg/roachpb" +) + +// candidate store for allocation. +type candidate struct { + store roachpb.StoreDescriptor + score float64 +} + +// solveState is used to pass solution state information into a rule. +type solveState struct { + constraints config.Constraints + store roachpb.StoreDescriptor + existing []roachpb.ReplicaDescriptor + sl StoreList + tiers map[roachpb.StoreID]map[string]roachpb.Tier + tierOrder []roachpb.Tier +} + +// rule is a generic rule that can be used to solve a constraint problem. +// Returning false will remove the store from the list of candidate stores. The +// score will be weighted and then summed together with the other rule scores to +// create a store ranking (higher is better). +type rule struct { + weight float64 + run func(state solveState) (float64, bool) +} + +// defaultRules is the default rule set to use. +var defaultRules = []rule{ + { + weight: 1.0, + run: ruleReplicasUniqueNodes, + }, + { + weight: 1.0, + run: ruleConstraints, + }, + { + weight: 0.01, + run: ruleCapacity, + }, + { + weight: 0.1, + run: ruleDiversity, + }, +} + +// makeDefaultRuleSolver returns a ruleSolver with defaultRules. +func makeDefaultRuleSolver(storePool *StorePool) ruleSolver { + return makeRuleSolver(storePool, defaultRules) +} + +// makeRuleSolver makes a new ruleSolver. The order of the rules is the order in +// which they are run. For optimization purposes, less computationally intense +// rules should run first to eliminate candidates. +func makeRuleSolver(storePool *StorePool, rules []rule) ruleSolver { + return ruleSolver{ + rules: rules, + } +} + +// ruleSolver solves a set of rules for a store. +type ruleSolver struct { + rules []rule +} + +// solve given constraints and return the score. +func (rs ruleSolver) Solve( + sl StoreList, c config.Constraints, existing []roachpb.ReplicaDescriptor, +) ([]candidate, error) { + candidates := make([]candidate, 0, len(sl.stores)) + state := solveState{ + constraints: c, + existing: existing, + sl: sl, + tierOrder: canonicalTierOrder(sl), + tiers: storeTierMap(sl), + } + + for _, store := range sl.stores { + state.store = store + if cand, ok := rs.computeCandidate(state); ok { + candidates = append(candidates, cand) + } + } + sort.Sort(byScore(candidates)) + return candidates, nil +} + +// computeCandidate runs all the rules for the store and returns the candidacy +// information. Returns false if not a candidate. +func (rs ruleSolver) computeCandidate(state solveState) (candidate, bool) { + var totalScore float64 + for _, rule := range rs.rules { + score, valid := rule.run(state) + if !valid { + return candidate{}, false + } + if !math.IsNaN(score) { + totalScore += score * rule.weight + } + } + return candidate{store: state.store, score: totalScore}, true +} + +// ruleReplicasUniqueNodes ensures that no two replicas are put on the same +// node. +func ruleReplicasUniqueNodes(state solveState) (float64, bool) { + for _, r := range state.existing { + if r.NodeID == state.store.Node.NodeID { + return 0, false + } + } + return 0, true +} + +// storeHasConstraint returns whether a store descriptor attributes or locality +// matches the key value pair in the constraint. +func storeHasConstraint(store roachpb.StoreDescriptor, c config.Constraint) bool { + var found bool + if c.Key == "" { + for _, attrs := range []roachpb.Attributes{store.Attrs, store.Node.Attrs} { + for _, attr := range attrs.Attrs { + if attr == c.Value { + return true + } + } + } + } else { + for _, tier := range store.Node.Locality.Tiers { + if c.Key == tier.Key && c.Value == tier.Value { + return true + } + } + } + return found +} + +// ruleConstraints enforces that required and prohibited constraints are +// followed, and that stores with more positive constraints are ranked higher. +func ruleConstraints(state solveState) (float64, bool) { + matched := 0 + for _, c := range state.constraints.Constraints { + hasConstraint := storeHasConstraint(state.store, c) + switch { + case c.Type == config.Constraint_POSITIVE && hasConstraint: + matched++ + case c.Type == config.Constraint_REQUIRED && !hasConstraint: + return 0, false + case c.Type == config.Constraint_PROHIBITED && hasConstraint: + return 0, false + } + } + + return float64(matched) / float64(len(state.constraints.Constraints)), true +} + +// ruleDiversity ensures that nodes that have the fewest locality tiers in +// common are given higher priority. +func ruleDiversity(state solveState) (float64, bool) { + storeTiers := state.tiers[state.store.StoreID] + var maxScore, score float64 + for i, tier := range state.tierOrder { + storeTier, ok := storeTiers[tier.Key] + if !ok { + continue + } + tierScore := 1 / (float64(i) + 1) + for _, existing := range state.existing { + existingTier, ok := state.tiers[existing.StoreID][tier.Key] + if ok && existingTier.Value != storeTier.Value { + score += tierScore + } + maxScore += tierScore + } + } + return score / maxScore, true +} + +// ruleCapacity prioritizes placing data on empty nodes when the choice is +// available and prevents data from going onto mostly full nodes. +func ruleCapacity(state solveState) (float64, bool) { + // Don't overfill stores. + if state.store.Capacity.FractionUsed() > maxFractionUsedThreshold { + return 0, false + } + + return 1 / float64(state.store.Capacity.RangeCount+1), true +} + +// canonicalTierOrder returns the most common key at each tier level. +func canonicalTierOrder(sl StoreList) []roachpb.Tier { + maxTierCount := 0 + for _, store := range sl.stores { + if count := len(store.Node.Locality.Tiers); maxTierCount < count { + maxTierCount = count + } + } + + // Might have up to maxTierCount of tiers. + tiers := make([]roachpb.Tier, 0, maxTierCount) + for i := 0; i < maxTierCount; i++ { + // At each tier, count the number of occurrences of each key. + counts := map[string]int{} + maxKey := "" + for _, store := range sl.stores { + key := "" + if i < len(store.Node.Locality.Tiers) { + key = store.Node.Locality.Tiers[i].Key + } + counts[key]++ + if counts[key] > counts[maxKey] { + maxKey = key + } + } + // Don't add the tier if most nodes don't have that many tiers. + if maxKey != "" { + tiers = append(tiers, roachpb.Tier{Key: maxKey}) + } + } + return tiers +} + +// storeTierMap indexes a store list so you can look up the locality tier +// value from store ID and tier key. +func storeTierMap(sl StoreList) map[roachpb.StoreID]map[string]roachpb.Tier { + m := map[roachpb.StoreID]map[string]roachpb.Tier{} + for _, store := range sl.stores { + sm := map[string]roachpb.Tier{} + m[store.StoreID] = sm + for _, tier := range store.Node.Locality.Tiers { + sm[tier.Key] = tier + } + } + return m +} + +// byScore implements sort.Interface for candidate slices. +type byScore []candidate + +var _ sort.Interface = byScore(nil) + +func (c byScore) Len() int { return len(c) } +func (c byScore) Less(i, j int) bool { return c[i].score > c[j].score } +func (c byScore) Swap(i, j int) { c[i], c[j] = c[j], c[i] } diff --git a/pkg/storage/rule_solver_test.go b/pkg/storage/rule_solver_test.go new file mode 100644 index 000000000000..2df6dd393362 --- /dev/null +++ b/pkg/storage/rule_solver_test.go @@ -0,0 +1,334 @@ +// Copyright 2016 The Cockroach Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. +// +// Author: Tristan Rice (rice@fn.lc) + +package storage + +import ( + "reflect" + "sort" + "testing" + + "github.com/cockroachdb/cockroach/pkg/config" + "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/util/leaktest" +) + +type byScoreAndID []candidate + +func (c byScoreAndID) Len() int { return len(c) } +func (c byScoreAndID) Less(i, j int) bool { + if c[i].score == c[j].score { + return c[i].store.StoreID < c[j].store.StoreID + } + return c[i].score > c[j].score +} +func (c byScoreAndID) Swap(i, j int) { c[i], c[j] = c[j], c[i] } + +// TestRuleSolver tests the mechanics of ruleSolver. +func TestRuleSolver(t *testing.T) { + defer leaktest.AfterTest(t)() + stopper, _, _, storePool := createTestStorePool(TestTimeUntilStoreDeadOff) + defer stopper.Stop() + // 3 alive replicas, 1 dead + mockStorePool(storePool, []roachpb.StoreID{1, 2, 3, 5}, []roachpb.StoreID{4}, nil) + + storePool.mu.Lock() + storePool.mu.storeDetails[1].desc.Attrs.Attrs = []string{"a"} + storePool.mu.storeDetails[2].desc.Attrs.Attrs = []string{"a", "b"} + storePool.mu.storeDetails[3].desc.Attrs.Attrs = []string{"a", "b", "c"} + + storePool.mu.storeDetails[1].desc.Node.Locality.Tiers = []roachpb.Tier{ + {Key: "datacenter", Value: "us"}, + {Key: "rack", Value: "1"}, + {Key: "slot", Value: "5"}, + } + storePool.mu.storeDetails[2].desc.Node.Locality.Tiers = []roachpb.Tier{ + {Key: "datacenter", Value: "us"}, + {Key: "rack", Value: "1"}, + } + storePool.mu.storeDetails[3].desc.Node.Locality.Tiers = []roachpb.Tier{ + {Key: "datacenter", Value: "us"}, + {Key: "floor", Value: "1"}, + {Key: "rack", Value: "2"}, + } + storePool.mu.storeDetails[5].desc.Node.Locality.Tiers = []roachpb.Tier{ + {Key: "datacenter", Value: "eur"}, + {Key: "rack", Value: "1"}, + } + + storePool.mu.storeDetails[1].desc.Capacity = roachpb.StoreCapacity{ + Capacity: 100, + Available: 1, + RangeCount: 99, + } + storePool.mu.storeDetails[2].desc.Capacity = roachpb.StoreCapacity{ + Capacity: 100, + Available: 100, + RangeCount: 0, + } + storePool.mu.storeDetails[3].desc.Capacity = roachpb.StoreCapacity{ + Capacity: 100, + Available: 50, + RangeCount: 50, + } + storePool.mu.storeDetails[5].desc.Capacity = roachpb.StoreCapacity{ + Capacity: 100, + Available: 60, + RangeCount: 40, + } + storePool.mu.Unlock() + + testCases := []struct { + rules []rule + c config.Constraints + existing []roachpb.ReplicaDescriptor + expected []roachpb.StoreID + }{ + // No constraints or rules. + { + expected: []roachpb.StoreID{1, 2, 3, 5}, + }, + // Store 1: score 0; Store 3: score 1; everything else fails. + { + rules: []rule{ + { + weight: 1, + run: func(state solveState) (float64, bool) { + switch state.store.StoreID { + case 1: + return 0, true + case 3: + return 1, true + default: + return 0, false + } + }, + }, + }, + expected: []roachpb.StoreID{3, 1}, + }, + // Don't put a replica on the same node as another. + { + rules: []rule{{weight: 1, run: ruleReplicasUniqueNodes}}, + existing: []roachpb.ReplicaDescriptor{ + {NodeID: 1}, + {NodeID: 3}, + }, + expected: []roachpb.StoreID{2, 5}, + }, + { + rules: []rule{{weight: 1, run: ruleReplicasUniqueNodes}}, + existing: []roachpb.ReplicaDescriptor{ + {NodeID: 1}, + {NodeID: 2}, + {NodeID: 3}, + {NodeID: 5}, + }, + expected: nil, + }, + // Only put replicas on nodes with required constraints. + { + rules: []rule{{weight: 1, run: ruleConstraints}}, + c: config.Constraints{ + Constraints: []config.Constraint{ + {Value: "b", Type: config.Constraint_REQUIRED}, + }, + }, + expected: []roachpb.StoreID{2, 3}, + }, + // Required locality constraints. + { + rules: []rule{{weight: 1, run: ruleConstraints}}, + c: config.Constraints{ + Constraints: []config.Constraint{ + {Key: "datacenter", Value: "us", Type: config.Constraint_REQUIRED}, + }, + }, + expected: []roachpb.StoreID{1, 2, 3}, + }, + // Don't put a replica on a node with a prohibited constraint. + { + rules: []rule{{weight: 1, run: ruleConstraints}}, + c: config.Constraints{ + Constraints: []config.Constraint{ + {Value: "b", Type: config.Constraint_PROHIBITED}, + }, + }, + expected: []roachpb.StoreID{1, 5}, + }, + // Prohibited locality constraints. + { + rules: []rule{{weight: 1, run: ruleConstraints}}, + c: config.Constraints{ + Constraints: []config.Constraint{ + {Key: "datacenter", Value: "us", Type: config.Constraint_PROHIBITED}, + }, + }, + expected: []roachpb.StoreID{5}, + }, + // Positive constraints ordered by number of matches. + { + rules: []rule{{weight: 1, run: ruleConstraints}}, + c: config.Constraints{ + Constraints: []config.Constraint{ + {Value: "a"}, + {Value: "b"}, + {Value: "c"}, + }, + }, + expected: []roachpb.StoreID{3, 2, 1, 5}, + }, + // Positive locality constraints. + { + rules: []rule{{weight: 1, run: ruleConstraints}}, + c: config.Constraints{ + Constraints: []config.Constraint{ + {Key: "datacenter", Value: "eur"}, + }, + }, + expected: []roachpb.StoreID{5, 1, 2, 3}, + }, + // Diversity with no existing. + { + rules: []rule{{weight: 1, run: ruleDiversity}}, + existing: nil, + expected: []roachpb.StoreID{1, 2, 3, 5}, + }, + // Diversity with one existing. + { + rules: []rule{{weight: 1, run: ruleDiversity}}, + existing: []roachpb.ReplicaDescriptor{ + {StoreID: 1}, + }, + expected: []roachpb.StoreID{5, 3, 1, 2}, + }, + // Prioritize lower capacity nodes, and don't overfill. + { + rules: []rule{{weight: 1, run: ruleCapacity}}, + expected: []roachpb.StoreID{2, 5, 3}, + }, + } + + for i, tc := range testCases { + solver := makeRuleSolver(storePool, tc.rules) + // TODO(bram): add a corrupt replica test and remove the 0 range ID. + sl, _, _ := storePool.getStoreList(roachpb.RangeID(0)) + candidates, err := solver.Solve(sl, tc.c, tc.existing) + if err != nil { + t.Fatal(err) + } + sort.Sort(byScoreAndID(candidates)) + if len(candidates) != len(tc.expected) { + t.Errorf("%d: length of %+v should match %+v", i, candidates, tc.expected) + continue + } + for j, expected := range tc.expected { + if out := candidates[j].store.StoreID; out != expected { + t.Errorf("%d: candidates[%d].store.StoreID = %d; not %d; %+v", i, j, out, expected, candidates) + } + } + } +} + +func TestCanonicalTierOrder(t *testing.T) { + defer leaktest.AfterTest(t)() + + testCases := []struct { + stores [][]roachpb.Tier + want []roachpb.Tier + }{ + { + nil, + []roachpb.Tier{}, + }, + { + [][]roachpb.Tier{nil, nil}, + []roachpb.Tier{}, + }, + { + [][]roachpb.Tier{ + { + {Key: "a"}, + {Key: "b"}, + {Key: "c"}, + }, + }, + []roachpb.Tier{ + {Key: "a"}, + {Key: "b"}, + {Key: "c"}, + }, + }, + { + [][]roachpb.Tier{ + {{Key: "a"}, + {Key: "b"}, + {Key: "c"}, + }, + { + {Key: "a"}, + {Key: "b"}, + {Key: "c"}, + }, + { + {Key: "b"}, + {Key: "c"}, + {Key: "a"}, + {Key: "d"}, + }, + }, + []roachpb.Tier{ + {Key: "a"}, + {Key: "b"}, + {Key: "c"}, + }, + }, + { + [][]roachpb.Tier{ + { + {Key: "a"}, + {Key: "b"}, + {Key: "c"}, + }, + { + {Key: "e"}, + {Key: "f"}, + {Key: "g"}, + }, + }, + []roachpb.Tier{ + {Key: "a"}, + {Key: "b"}, + {Key: "c"}, + }, + }, + } + + for i, tc := range testCases { + sl := StoreList{} + for _, tiers := range tc.stores { + sl.stores = append(sl.stores, roachpb.StoreDescriptor{ + Node: roachpb.NodeDescriptor{ + Locality: roachpb.Locality{Tiers: tiers}, + }, + }) + } + + if out := canonicalTierOrder(sl); !reflect.DeepEqual(out, tc.want) { + t.Errorf("%d: canonicalTierOrder(%+v) = %+v; not %+v", i, tc.stores, out, tc.want) + } + } +} diff --git a/pkg/storage/simulation/cluster.go b/pkg/storage/simulation/cluster.go index 7095db40abd3..d305dbd0dae9 100644 --- a/pkg/storage/simulation/cluster.go +++ b/pkg/storage/simulation/cluster.go @@ -98,7 +98,6 @@ func createCluster( storePool: storePool, allocator: storage.MakeAllocator(storePool, storage.AllocatorOptions{ AllowRebalance: true, - Deterministic: true, }), storeGossiper: gossiputil.NewStoreGossiper(g), nodes: make(map[roachpb.NodeID]*Node), diff --git a/pkg/storage/simulation/range.go b/pkg/storage/simulation/range.go index 94be037c5948..900affa32a2a 100644 --- a/pkg/storage/simulation/range.go +++ b/pkg/storage/simulation/range.go @@ -118,7 +118,6 @@ func (r *Range) getAllocateTarget() (roachpb.StoreID, error) { r.zone.Constraints, r.desc.Replicas, r.desc.RangeID, - true, ) if err != nil { return 0, err @@ -131,7 +130,7 @@ func (r *Range) getAllocateTarget() (roachpb.StoreID, error) { func (r *Range) getRemoveTarget() (roachpb.StoreID, error) { // Pass in an invalid store ID since we don't consider range leases as part // of the simulator. - removeStore, err := r.allocator.RemoveTarget(r.desc.Replicas, roachpb.StoreID(-1)) + removeStore, err := r.allocator.RemoveTarget(r.zone.Constraints, r.desc.Replicas, roachpb.StoreID(-1)) if err != nil { return 0, err } @@ -142,12 +141,15 @@ func (r *Range) getRemoveTarget() (roachpb.StoreID, error) { // candidate to add a replica for rebalancing. Returns true only if a target is // found. func (r *Range) getRebalanceTarget(storeID roachpb.StoreID) (roachpb.StoreID, bool) { - rebalanceTarget := r.allocator.RebalanceTarget( + rebalanceTarget, err := r.allocator.RebalanceTarget( r.zone.Constraints, r.desc.Replicas, storeID, r.desc.RangeID, ) + if err != nil { + panic(err) + } if rebalanceTarget == nil { return 0, false } diff --git a/pkg/storage/store_pool.go b/pkg/storage/store_pool.go index b5827d0a3a5c..f67e7056841c 100644 --- a/pkg/storage/store_pool.go +++ b/pkg/storage/store_pool.go @@ -25,7 +25,6 @@ import ( "golang.org/x/net/context" - "github.com/cockroachdb/cockroach/pkg/config" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/rpc" @@ -91,57 +90,46 @@ func (sd *storeDetail) markAlive(foundAliveOn hlc.Timestamp, storeDesc *roachpb. sd.lastUpdatedTime = foundAliveOn } -// storeMatch is the return value for match(). -type storeMatch int +// isThrottled returns whether the store is currently throttled. +func (sd storeDetail) isThrottled(now time.Time) bool { + return sd.throttledUntil.After(now) +} + +// storeStatus is the current status of a store. +type storeStatus int -// These are the possible values for a storeMatch. +// These are the possible values for a storeStatus. const ( // The store is not yet available or has been timed out. - storeMatchDead storeMatch = iota - // The store is alive, but its attributes didn't match the required ones. - storeMatchAlive + storeStatusDead storeStatus = iota // The store is alive and its attributes matched, but it is throttled. - storeMatchThrottled + storeStatusThrottled // The store is alive and its attributes matched, but a replica for the // same rangeID was recently discovered to be corrupt. - storeMatchReplicaCorrupted + storeStatusReplicaCorrupted // The store is alive, available and its attributes matched. - storeMatchAvailable + storeStatusAvailable ) -// match checks the store against the attributes and returns a storeMatch. -func (sd *storeDetail) match( - now time.Time, constraints config.Constraints, rangeID roachpb.RangeID, -) storeMatch { +// status returns the current status of the store. +func (sd *storeDetail) status(now time.Time, rangeID roachpb.RangeID) storeStatus { // The store must be alive and it must have a descriptor to be considered // alive. if sd.dead || sd.desc == nil { - return storeMatchDead - } - - // Does the store match the attributes? - m := map[string]struct{}{} - for _, s := range sd.desc.CombinedAttrs().Attrs { - m[s] = struct{}{} - } - for _, c := range constraints.Constraints { - // TODO(d4l3k): Locality constraints, number of matches. - if _, ok := m[c.Value]; !ok { - return storeMatchAlive - } + return storeStatusDead } // The store must not have a recent declined reservation to be available. - if sd.throttledUntil.After(now) { - return storeMatchThrottled + if sd.isThrottled(now) { + return storeStatusThrottled } // The store must not have a corrupt replica on it. if len(sd.deadReplicas[rangeID]) > 0 { - return storeMatchReplicaCorrupted + return storeStatusReplicaCorrupted } - return storeMatchAvailable + return storeStatusAvailable } // storePoolPQ implements the heap.Interface (which includes sort.Interface) @@ -223,8 +211,9 @@ type StorePool struct { syncutil.RWMutex // Each storeDetail is contained in both a map and a priorityQueue; // pointers are used so that data can be kept in sync. - storeDetails map[roachpb.StoreID]*storeDetail - queue storePoolPQ + storeDetails map[roachpb.StoreID]*storeDetail + queue storePoolPQ + deterministic bool } } @@ -487,46 +476,47 @@ func (sl *StoreList) add(s roachpb.StoreDescriptor) { } } -// getStoreList returns a storeList that contains all active stores that -// contain the required attributes and their associated stats. It also returns -// the total number of alive and throttled stores. -// TODO(embark, spencer): consider using a reverse index map from -// Attr->stores, for efficiency. Ensure that entries in this map still -// have an opportunity to be garbage collected. -func (sp *StorePool) getStoreList( - constraints config.Constraints, rangeID roachpb.RangeID, deterministic bool, -) (StoreList, int, int) { +var _ sort.Interface = StoreList{} + +// Len implements sort.Interface. +func (sl StoreList) Len() int { return len(sl.stores) } + +// Less implements sort.Interface. +func (sl StoreList) Less(i, j int) bool { return sl.stores[i].StoreID < sl.stores[j].StoreID } + +// Swap implements sort.Interface. +func (sl StoreList) Swap(i, j int) { sl.stores[i], sl.stores[j] = sl.stores[j], sl.stores[i] } + +// getStoreList returns a storeList that contains all active stores and their +// associated stats. It also returns the total number of alive and throttled +// stores. +func (sp *StorePool) getStoreList(rangeID roachpb.RangeID) (StoreList, int, int) { sp.mu.RLock() defer sp.mu.RUnlock() - var storeIDs roachpb.StoreIDSlice - for storeID := range sp.mu.storeDetails { - storeIDs = append(storeIDs, storeID) - } - // Sort the stores by key if deterministic is requested. This is only for - // unit testing. - if deterministic { - sort.Sort(storeIDs) - } now := sp.clock.Now().GoTime() sl := StoreList{} var aliveStoreCount int var throttledStoreCount int - for _, storeID := range storeIDs { + + for storeID := range sp.mu.storeDetails { detail := sp.mu.storeDetails[storeID] - // TODO(d4l3k): Sort by number of matches. - matched := detail.match(now, constraints, rangeID) - switch matched { - case storeMatchAlive, storeMatchReplicaCorrupted: - aliveStoreCount++ - case storeMatchThrottled: + switch detail.status(now, rangeID) { + case storeStatusThrottled: aliveStoreCount++ throttledStoreCount++ - case storeMatchAvailable: + case storeStatusReplicaCorrupted: + aliveStoreCount++ + case storeStatusAvailable: aliveStoreCount++ sl.add(*detail.desc) } } + + if sp.mu.deterministic { + sort.Sort(sl) + } + return sl, aliveStoreCount, throttledStoreCount } diff --git a/pkg/storage/store_pool_test.go b/pkg/storage/store_pool_test.go index 04c95c93d5a8..ae9b178eaf08 100644 --- a/pkg/storage/store_pool_test.go +++ b/pkg/storage/store_pool_test.go @@ -22,8 +22,9 @@ import ( "testing" "time" + "github.com/pkg/errors" + "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/config" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/rpc" @@ -35,9 +36,15 @@ import ( "github.com/cockroachdb/cockroach/pkg/util/metric" "github.com/cockroachdb/cockroach/pkg/util/stop" "github.com/cockroachdb/cockroach/pkg/util/timeutil" - "github.com/pkg/errors" ) +// TestSetDeterministic makes StorePool return results in a deterministic way. +func (sp *StorePool) TestSetDeterministic(deterministic bool) { + sp.mu.Lock() + defer sp.mu.Unlock() + sp.mu.deterministic = deterministic +} + var uniqueStore = []*roachpb.StoreDescriptor{ { StoreID: 2, @@ -212,18 +219,13 @@ func TestStorePoolDies(t *testing.T) { // verifyStoreList ensures that the returned list of stores is correct. func verifyStoreList( sp *StorePool, - constraints config.Constraints, rangeID roachpb.RangeID, expected []int, expectedAliveStoreCount int, expectedThrottledStoreCount int, ) error { var actual []int - sl, aliveStoreCount, throttledStoreCount := sp.getStoreList( - constraints, - rangeID, - false, - ) + sl, aliveStoreCount, throttledStoreCount := sp.getStoreList(rangeID) if aliveStoreCount != expectedAliveStoreCount { return errors.Errorf("expected AliveStoreCount %d does not match actual %d", expectedAliveStoreCount, aliveStoreCount) @@ -244,73 +246,51 @@ func verifyStoreList( } // TestStorePoolGetStoreList ensures that the store list returns only stores -// that are alive and match the attribute criteria. +// that are alive. func TestStorePoolGetStoreList(t *testing.T) { defer leaktest.AfterTest(t)() // We're going to manually mark stores dead in this test. stopper, g, _, sp := createTestStorePool(TestTimeUntilStoreDeadOff) defer stopper.Stop() sg := gossiputil.NewStoreGossiper(g) - constraints := config.Constraints{Constraints: []config.Constraint{{Value: "ssd"}, {Value: "dc"}}} - required := []string{"ssd", "dc"} // Nothing yet. - if sl, _, _ := sp.getStoreList( - constraints, - roachpb.RangeID(0), - false, - ); len(sl.stores) != 0 { + if sl, _, _ := sp.getStoreList(roachpb.RangeID(0)); len(sl.stores) != 0 { t.Errorf("expected no stores, instead %+v", sl.stores) } matchingStore := roachpb.StoreDescriptor{ StoreID: 1, Node: roachpb.NodeDescriptor{NodeID: 1}, - Attrs: roachpb.Attributes{Attrs: required}, } supersetStore := roachpb.StoreDescriptor{ StoreID: 2, Node: roachpb.NodeDescriptor{NodeID: 1}, - Attrs: roachpb.Attributes{Attrs: append(required, "db")}, - } - unmatchingStore := roachpb.StoreDescriptor{ - StoreID: 3, - Node: roachpb.NodeDescriptor{NodeID: 1}, - Attrs: roachpb.Attributes{Attrs: []string{"ssd", "otherdc"}}, - } - emptyStore := roachpb.StoreDescriptor{ - StoreID: 4, - Node: roachpb.NodeDescriptor{NodeID: 1}, - Attrs: roachpb.Attributes{}, } deadStore := roachpb.StoreDescriptor{ - StoreID: 5, + StoreID: 3, Node: roachpb.NodeDescriptor{NodeID: 1}, - Attrs: roachpb.Attributes{Attrs: required}, } declinedStore := roachpb.StoreDescriptor{ - StoreID: 6, + StoreID: 4, Node: roachpb.NodeDescriptor{NodeID: 1}, - Attrs: roachpb.Attributes{Attrs: required}, } corruptReplicaStore := roachpb.StoreDescriptor{ StoreID: 7, Node: roachpb.NodeDescriptor{NodeID: 1}, - Attrs: roachpb.Attributes{Attrs: required}, } corruptedRangeID := roachpb.RangeID(1) - // Mark all alive initially. - sg.GossipStores([]*roachpb.StoreDescriptor{ + allStores := []*roachpb.StoreDescriptor{ &matchingStore, &supersetStore, - &unmatchingStore, - &emptyStore, &deadStore, &declinedStore, &corruptReplicaStore, - }, t) + } + // Mark all alive initially. + sg.GossipStores(allStores, t) // Add some corrupt replicas that should not affect getStoreList(). sp.mu.Lock() sp.mu.storeDetails[matchingStore.StoreID].deadReplicas[roachpb.RangeID(10)] = @@ -332,7 +312,6 @@ func TestStorePoolGetStoreList(t *testing.T) { if err := verifyStoreList( sp, - constraints, corruptedRangeID, []int{ int(matchingStore.StoreID), @@ -341,7 +320,7 @@ func TestStorePoolGetStoreList(t *testing.T) { int(declinedStore.StoreID), int(corruptReplicaStore.StoreID), }, - /* expectedAliveStoreCount */ 7, + /* expectedAliveStoreCount */ len(allStores), /* expectedThrottledStoreCount */ 0, ); err != nil { t.Error(err) @@ -362,13 +341,12 @@ func TestStorePoolGetStoreList(t *testing.T) { if err := verifyStoreList( sp, - constraints, corruptedRangeID, []int{ int(matchingStore.StoreID), int(supersetStore.StoreID), }, - /* expectedAliveStoreCount */ 6, + /* expectedAliveStoreCount */ len(allStores)-1, /* expectedThrottledStoreCount */ 1, ); err != nil { t.Error(err) @@ -484,11 +462,7 @@ func TestStorePoolDefaultState(t *testing.T) { t.Errorf("expected 0 dead replicas; got %v", dead) } - sl, alive, throttled := sp.getStoreList( - config.Constraints{}, - roachpb.RangeID(0), - true, - ) + sl, alive, throttled := sp.getStoreList(roachpb.RangeID(0)) if len(sl.stores) > 0 { t.Errorf("expected no live stores; got list of %v", sl) } From 1f4cac27b2c4e3ca251aaa94ecda020f755cf68f Mon Sep 17 00:00:00 2001 From: Bram Gruneir Date: Thu, 27 Oct 2016 17:13:10 -0400 Subject: [PATCH 2/2] storage: update output for allocator_test's Example_rebalancing --- pkg/storage/allocator_test.go | 127 ++++++++++++++++------------------ 1 file changed, 61 insertions(+), 66 deletions(-) diff --git a/pkg/storage/allocator_test.go b/pkg/storage/allocator_test.go index bc796b8dc238..40aa279963fc 100644 --- a/pkg/storage/allocator_test.go +++ b/pkg/storage/allocator_test.go @@ -1402,26 +1402,21 @@ func Example_rebalancing() { } } - // Output store capacities as hexadecimal 2-character values. - if i%(generations/50) == 0 { - var maxBytes int64 + if i%(generations/printGenerations) == 0 { + var totalBytes int64 for j := 0; j < len(testStores); j++ { - bytes := testStores[j].Capacity.Capacity - testStores[j].Capacity.Available - if bytes > maxBytes { - maxBytes = bytes - } + totalBytes += testStores[j].Capacity.Capacity - testStores[j].Capacity.Available } - if maxBytes > 0 { - for j := 0; j < len(testStores); j++ { - endStr := " " - if j == len(testStores)-1 { - endStr = "" - } - bytes := testStores[j].Capacity.Capacity - testStores[j].Capacity.Available - fmt.Printf("%03d%s", (999*bytes)/maxBytes, endStr) + fmt.Printf("generation %4d: ", i) + for j := 0; j < len(testStores); j++ { + if j != 0 && j != len(testStores)-1 { + fmt.Printf(",") } - fmt.Printf("\n") + ts := testStores[j] + bytes := ts.Capacity.Capacity - ts.Capacity.Available + fmt.Printf("%3d %2d%%", ts.Capacity.RangeCount, (100*bytes)/totalBytes) } + fmt.Printf("\n") } } @@ -1434,55 +1429,55 @@ func Example_rebalancing() { fmt.Printf("Total bytes=%d, ranges=%d\n", totBytes, totRanges) // Output: - // 999 129 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 - // 999 758 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 000 657 625 - // 432 999 000 000 000 000 000 000 000 000 000 000 000 000 257 036 428 845 930 420 - // 193 185 000 000 000 000 559 109 000 000 000 000 153 999 051 134 248 388 349 218 - // 268 301 000 000 000 000 543 163 484 357 415 000 282 999 193 168 362 573 352 324 - // 320 399 000 000 412 598 564 348 440 443 729 000 465 999 311 385 444 771 454 463 - // 284 241 377 999 231 431 338 266 186 275 508 095 316 519 283 204 283 555 330 384 - // 419 435 599 999 289 477 482 418 328 212 622 516 383 536 426 356 463 700 391 588 - // 636 554 727 999 370 655 596 586 489 335 717 601 512 734 572 528 610 797 499 767 - // 626 648 883 999 451 716 655 596 604 387 853 723 612 753 703 590 728 894 625 850 - // 660 697 898 999 468 850 797 660 620 470 918 704 684 714 793 674 844 929 690 806 - // 694 736 965 989 499 909 805 692 729 593 961 702 723 720 913 721 909 999 854 876 - // 682 724 918 951 464 837 770 652 712 611 905 723 739 665 826 618 835 999 864 796 - // 737 739 924 949 573 840 789 680 706 659 929 819 764 721 843 589 868 999 891 870 - // 777 795 952 998 637 852 837 739 733 737 947 843 832 796 912 664 897 999 906 839 - // 760 809 929 961 658 859 780 776 720 738 900 826 797 833 911 703 883 999 930 834 - // 737 850 942 999 677 907 767 819 738 754 958 832 845 808 906 722 931 986 978 817 - // 762 846 935 999 688 913 773 821 733 815 954 870 866 792 968 764 992 986 974 874 - // 771 828 933 965 681 878 804 814 735 869 936 855 856 752 951 763 999 995 969 877 - // 723 817 862 931 673 824 752 824 707 832 857 793 834 731 887 731 999 934 926 871 - // 706 825 840 925 682 824 747 780 715 818 853 795 827 700 842 703 999 882 877 875 - // 739 835 876 958 731 887 763 823 740 839 881 849 873 737 889 763 999 926 925 929 - // 773 882 904 971 768 926 841 847 822 843 940 887 941 765 898 814 999 960 998 993 - // 764 872 905 999 773 915 859 843 848 865 909 871 944 739 922 780 974 977 992 993 - // 732 876 911 999 765 888 891 856 848 831 901 876 948 753 889 773 989 967 989 997 - // 733 856 936 981 797 901 924 868 876 820 930 889 958 770 867 785 984 963 989 999 - // 737 836 960 991 787 895 912 875 856 822 930 907 974 778 849 792 965 950 999 988 - // 742 839 970 999 803 925 918 894 884 812 952 901 977 809 872 764 985 961 979 974 - // 754 841 969 987 822 934 938 922 904 830 960 911 990 804 879 781 975 938 999 986 - // 735 842 966 980 805 909 920 938 888 842 967 913 999 782 882 783 994 932 985 980 - // 759 857 971 999 826 927 912 951 872 864 976 920 976 781 899 771 998 947 963 959 - // 756 849 965 999 818 924 895 916 856 861 962 898 944 771 906 743 966 942 945 938 - // 743 850 963 999 812 919 870 922 876 844 977 885 934 773 909 725 958 940 929 943 - // 757 831 951 999 798 918 854 933 865 832 979 852 907 793 920 728 956 931 906 934 - // 775 823 927 999 792 907 840 925 871 826 954 837 909 807 909 711 958 909 920 930 - // 772 843 933 999 788 902 841 920 890 843 941 830 895 804 908 735 951 927 921 929 - // 799 851 938 999 805 914 868 920 903 836 937 832 911 811 914 764 949 927 903 930 - // 828 877 952 999 817 945 869 915 907 849 974 856 936 827 925 790 966 944 929 937 - // 843 888 971 999 844 955 892 913 925 856 990 871 968 854 937 795 977 950 937 966 - // 859 911 980 994 863 960 899 925 940 863 999 884 989 855 948 809 989 949 945 966 - // 847 905 980 974 848 955 888 924 940 849 999 877 981 855 938 786 997 940 947 965 - // 844 895 973 966 853 935 876 917 935 851 991 861 978 833 946 788 999 935 942 964 - // 849 886 975 966 865 930 890 916 927 855 988 870 994 841 943 787 999 935 963 963 - // 843 887 973 951 860 933 874 910 909 865 966 875 984 830 933 779 999 922 967 960 - // 836 879 983 945 863 923 890 913 893 867 964 894 982 836 914 783 999 940 969 967 - // 830 879 981 943 853 926 886 900 894 863 947 896 976 832 903 771 999 935 972 967 - // 830 879 981 943 853 926 886 900 894 863 947 896 976 832 903 771 999 935 972 967 - // 830 879 981 943 853 926 886 900 894 863 947 896 976 832 903 771 999 935 972 967 - // 830 879 981 943 853 926 886 900 894 863 947 896 976 832 903 771 999 935 972 967 - // 830 879 981 943 853 926 886 900 894 863 947 896 976 832 903 771 999 935 972 967 + // generation 0: 1 88%, 1 11%, 0 0%, 0 0%, 0 0%, 0 0%, 0 0%, 0 0%, 0 0%, 0 0%, 0 0%, 0 0%, 0 0%, 0 0%, 0 0%, 0 0%, 0 0%, 0 0%, 0 0% 0 0% + // generation 2: 1 32%, 2 24%, 0 0%, 0 0%, 0 0%, 0 0%, 0 0%, 0 0%, 0 0%, 0 0%, 0 0%, 0 0%, 0 0%, 0 0%, 0 0%, 0 0%, 0 0%, 0 0%, 2 21% 2 20% + // generation 4: 2 9%, 2 22%, 0 0%, 0 0%, 0 0%, 0 0%, 0 0%, 0 0%, 0 0%, 0 0%, 0 0%, 0 0%, 0 0%, 0 0%, 3 5%, 1 0%, 2 9%, 2 19%, 2 21% 3 9% + // generation 6: 3 5%, 2 5%, 0 0%, 0 0%, 0 0%, 0 0%, 6 15%, 1 3%, 0 0%, 0 0%, 0 0%, 0 0%, 2 4%, 7 27%, 3 1%, 2 3%, 2 6%, 2 10%, 2 9% 3 6% + // generation 8: 3 4%, 3 5%, 0 0%, 0 0%, 0 0%, 0 0%, 6 9%, 3 2%, 7 8%, 7 6%, 3 7%, 0 0%, 3 4%, 7 17%, 4 3%, 3 2%, 3 6%, 3 9%, 3 6% 3 5% + // generation 10: 5 3%, 5 4%, 0 0%, 0 0%, 5 4%, 5 6%, 6 6%, 5 4%, 7 5%, 7 5%, 5 8%, 0 0%, 5 5%, 7 11%, 5 3%, 5 4%, 5 5%, 5 9%, 5 5% 5 5% + // generation 12: 6 3%, 6 3%, 5 5%, 15 14%, 6 3%, 6 6%, 6 4%, 6 3%, 7 2%, 7 3%, 6 7%, 3 1%, 6 4%, 7 7%, 6 3%, 6 2%, 6 3%, 6 7%, 6 4% 6 5% + // generation 14: 8 4%, 8 4%, 8 6%, 15 10%, 8 2%, 8 4%, 8 5%, 8 4%, 8 3%, 8 2%, 8 6%, 9 5%, 8 3%, 8 5%, 8 4%, 8 3%, 8 4%, 8 7%, 8 4% 8 6% + // generation 16: 11 5%, 10 4%, 10 5%, 15 8%, 10 3%, 10 5%, 10 4%, 10 4%, 10 3%, 10 2%, 10 5%, 11 4%, 10 4%, 10 5%, 10 4%, 10 4%, 10 4%, 10 6%, 10 4% 11 6% + // generation 18: 13 4%, 12 4%, 12 6%, 15 7%, 12 3%, 12 5%, 12 4%, 12 4%, 12 4%, 12 2%, 12 6%, 13 5%, 12 4%, 12 5%, 12 5%, 12 4%, 13 5%, 13 6%, 12 4% 13 6% + // generation 20: 15 4%, 14 4%, 14 6%, 16 6%, 14 3%, 14 5%, 14 5%, 14 4%, 14 4%, 14 3%, 14 6%, 15 4%, 14 4%, 14 4%, 14 5%, 15 4%, 15 5%, 15 6%, 14 4% 15 5% + // generation 22: 17 4%, 16 4%, 16 6%, 18 6%, 16 3%, 16 5%, 16 5%, 16 4%, 16 4%, 16 3%, 16 6%, 17 4%, 16 4%, 16 4%, 16 5%, 17 4%, 17 5%, 17 6%, 16 5% 17 5% + // generation 24: 19 4%, 18 4%, 18 6%, 20 6%, 18 3%, 18 5%, 18 5%, 18 4%, 18 4%, 18 3%, 18 5%, 19 4%, 18 4%, 18 4%, 18 5%, 19 4%, 19 5%, 19 6%, 18 5% 19 5% + // generation 26: 21 4%, 20 4%, 20 5%, 22 5%, 20 3%, 20 5%, 20 4%, 20 4%, 20 4%, 20 4%, 20 5%, 21 5%, 20 4%, 20 4%, 20 5%, 21 3%, 21 5%, 21 6%, 20 5% 21 5% + // generation 28: 23 4%, 22 4%, 22 5%, 24 5%, 22 3%, 22 5%, 22 5%, 22 4%, 22 4%, 22 4%, 22 5%, 23 5%, 22 4%, 22 4%, 22 5%, 23 3%, 23 5%, 23 5%, 22 5% 23 5% + // generation 30: 25 4%, 24 4%, 24 5%, 26 5%, 24 3%, 24 5%, 24 4%, 24 4%, 24 4%, 24 4%, 24 5%, 25 4%, 24 4%, 24 5%, 24 5%, 25 4%, 25 5%, 25 6%, 24 5% 25 5% + // generation 32: 27 4%, 26 5%, 26 5%, 28 5%, 26 3%, 26 5%, 26 4%, 26 4%, 26 4%, 26 4%, 26 5%, 27 4%, 26 4%, 26 4%, 26 5%, 27 4%, 27 5%, 27 5%, 26 5% 27 4% + // generation 34: 29 4%, 28 4%, 28 5%, 30 5%, 28 3%, 28 5%, 28 4%, 28 4%, 28 4%, 28 4%, 28 5%, 29 5%, 28 4%, 28 4%, 28 5%, 29 4%, 29 5%, 29 5%, 28 5% 29 5% + // generation 36: 31 4%, 30 4%, 30 5%, 32 5%, 30 3%, 30 5%, 30 4%, 30 4%, 30 4%, 30 5%, 30 5%, 31 4%, 30 4%, 30 4%, 30 5%, 31 4%, 31 5%, 31 5%, 30 5% 31 5% + // generation 38: 33 4%, 32 4%, 32 5%, 34 5%, 32 4%, 32 4%, 32 4%, 32 4%, 32 4%, 32 5%, 32 5%, 33 4%, 32 5%, 32 4%, 32 5%, 33 4%, 33 6%, 33 5%, 32 5% 33 5% + // generation 40: 35 4%, 34 5%, 34 5%, 36 5%, 34 4%, 34 5%, 34 4%, 34 4%, 34 4%, 34 5%, 34 5%, 35 4%, 34 5%, 34 4%, 34 5%, 35 4%, 35 6%, 35 5%, 34 5% 35 5% + // generation 42: 37 4%, 36 4%, 36 5%, 38 5%, 36 4%, 36 5%, 36 4%, 36 4%, 36 4%, 36 4%, 36 5%, 37 5%, 36 5%, 36 4%, 36 5%, 37 4%, 37 5%, 37 5%, 36 5% 37 5% + // generation 44: 39 4%, 38 4%, 38 5%, 40 5%, 38 4%, 38 5%, 38 4%, 38 4%, 38 4%, 38 4%, 38 5%, 39 4%, 38 5%, 38 4%, 38 5%, 39 4%, 39 5%, 39 5%, 38 5% 39 5% + // generation 46: 41 4%, 40 4%, 40 5%, 42 5%, 40 4%, 40 5%, 40 4%, 40 4%, 40 4%, 40 4%, 40 5%, 41 4%, 40 5%, 40 4%, 40 5%, 41 4%, 41 5%, 41 5%, 40 5% 41 5% + // generation 48: 43 4%, 42 4%, 42 5%, 44 5%, 42 4%, 42 5%, 42 5%, 42 4%, 42 4%, 42 4%, 42 5%, 43 4%, 42 5%, 42 4%, 42 5%, 43 4%, 43 5%, 43 5%, 42 5% 43 5% + // generation 50: 45 4%, 44 4%, 44 5%, 46 5%, 44 4%, 44 5%, 44 5%, 44 4%, 44 4%, 44 4%, 44 5%, 45 4%, 44 5%, 44 4%, 44 4%, 45 4%, 45 5%, 45 5%, 44 5% 45 5% + // generation 52: 47 4%, 46 4%, 46 5%, 48 5%, 46 4%, 46 5%, 46 5%, 46 4%, 46 4%, 46 4%, 46 5%, 47 5%, 46 5%, 46 4%, 46 4%, 47 4%, 47 5%, 47 5%, 46 5% 47 5% + // generation 54: 49 4%, 48 4%, 48 5%, 50 5%, 48 4%, 48 5%, 48 5%, 48 4%, 48 4%, 48 4%, 48 5%, 49 5%, 48 5%, 48 4%, 48 4%, 49 4%, 49 5%, 49 5%, 48 5% 49 5% + // generation 56: 51 4%, 50 4%, 50 5%, 52 5%, 50 4%, 50 5%, 50 5%, 50 5%, 50 4%, 50 4%, 50 5%, 51 5%, 50 5%, 50 4%, 50 4%, 51 4%, 51 5%, 51 5%, 50 5% 51 5% + // generation 58: 53 4%, 52 4%, 52 5%, 54 5%, 52 4%, 52 5%, 52 5%, 52 5%, 52 4%, 52 4%, 52 5%, 53 5%, 52 5%, 52 4%, 52 4%, 53 4%, 53 5%, 53 5%, 52 5% 53 5% + // generation 60: 55 4%, 54 4%, 54 5%, 56 5%, 54 4%, 54 5%, 54 5%, 54 5%, 54 4%, 54 4%, 54 5%, 55 5%, 54 5%, 54 4%, 54 4%, 55 4%, 55 5%, 55 5%, 54 5% 55 5% + // generation 62: 57 4%, 56 4%, 56 5%, 58 5%, 56 4%, 56 5%, 56 5%, 56 5%, 56 4%, 56 4%, 56 5%, 57 5%, 56 5%, 56 4%, 56 5%, 57 4%, 57 5%, 57 5%, 56 5% 57 5% + // generation 64: 59 4%, 58 4%, 58 5%, 60 5%, 58 4%, 58 5%, 58 4%, 58 5%, 58 4%, 58 4%, 58 5%, 59 4%, 58 5%, 58 4%, 58 5%, 59 4%, 59 5%, 59 5%, 58 5% 59 5% + // generation 66: 61 4%, 60 4%, 60 5%, 62 5%, 60 4%, 60 5%, 60 4%, 60 5%, 60 4%, 60 4%, 60 5%, 61 4%, 60 5%, 60 4%, 60 5%, 61 4%, 61 5%, 61 5%, 60 5% 61 5% + // generation 68: 63 4%, 62 4%, 62 5%, 64 5%, 62 4%, 62 5%, 62 4%, 62 5%, 62 4%, 62 4%, 62 5%, 63 4%, 62 5%, 62 4%, 62 5%, 63 4%, 63 5%, 63 5%, 62 5% 63 5% + // generation 70: 65 4%, 64 4%, 64 5%, 66 5%, 64 4%, 64 5%, 64 4%, 64 5%, 64 5%, 64 4%, 64 5%, 65 4%, 64 5%, 64 4%, 64 5%, 65 4%, 65 5%, 65 5%, 64 5% 65 5% + // generation 72: 67 4%, 66 4%, 66 5%, 68 5%, 66 4%, 66 5%, 66 4%, 66 5%, 66 5%, 66 4%, 66 5%, 67 4%, 66 5%, 66 4%, 66 5%, 67 4%, 67 5%, 67 5%, 66 5% 67 5% + // generation 74: 69 4%, 68 4%, 68 5%, 70 5%, 68 4%, 68 5%, 68 4%, 68 5%, 68 5%, 68 4%, 68 5%, 69 4%, 68 5%, 68 4%, 68 5%, 69 4%, 69 5%, 69 5%, 68 5% 69 5% + // generation 76: 71 4%, 70 4%, 70 5%, 72 5%, 70 4%, 70 5%, 70 4%, 70 4%, 70 5%, 70 4%, 70 5%, 71 4%, 70 5%, 70 4%, 70 5%, 71 4%, 71 5%, 71 5%, 70 5% 71 5% + // generation 78: 73 4%, 72 4%, 72 5%, 74 5%, 72 4%, 72 5%, 72 4%, 72 4%, 72 5%, 72 4%, 72 5%, 73 4%, 72 5%, 72 4%, 72 5%, 73 4%, 73 5%, 73 5%, 72 5% 73 5% + // generation 80: 75 4%, 74 4%, 74 5%, 76 5%, 74 4%, 74 5%, 74 4%, 74 5%, 74 5%, 74 4%, 74 5%, 75 4%, 74 5%, 74 4%, 74 5%, 75 4%, 75 5%, 75 5%, 74 5% 75 5% + // generation 82: 77 4%, 76 4%, 76 5%, 78 5%, 76 4%, 76 5%, 76 4%, 76 5%, 76 5%, 76 4%, 76 5%, 77 4%, 76 5%, 76 4%, 76 5%, 77 4%, 77 5%, 77 5%, 76 5% 77 5% + // generation 84: 79 4%, 78 4%, 78 5%, 80 5%, 78 4%, 78 5%, 78 4%, 78 4%, 78 5%, 78 4%, 78 5%, 79 4%, 78 5%, 78 4%, 78 5%, 79 4%, 79 5%, 79 5%, 78 5% 79 5% + // generation 86: 81 4%, 80 4%, 80 5%, 82 5%, 80 4%, 80 5%, 80 4%, 80 4%, 80 4%, 80 4%, 80 5%, 81 4%, 80 5%, 80 4%, 80 5%, 81 4%, 81 5%, 81 5%, 80 5% 81 5% + // generation 88: 83 4%, 82 4%, 82 5%, 84 5%, 82 4%, 82 5%, 82 4%, 82 5%, 82 4%, 82 4%, 82 5%, 83 4%, 82 5%, 82 4%, 82 5%, 83 4%, 83 5%, 83 5%, 82 5% 83 5% + // generation 90: 84 4%, 83 4%, 83 5%, 85 5%, 83 4%, 83 5%, 83 4%, 83 4%, 83 4%, 83 4%, 83 5%, 84 4%, 83 5%, 83 4%, 83 4%, 84 4%, 84 5%, 84 5%, 83 5% 84 5% + // generation 92: 84 4%, 83 4%, 83 5%, 85 5%, 83 4%, 83 5%, 83 4%, 83 4%, 83 4%, 83 4%, 83 5%, 84 4%, 83 5%, 83 4%, 83 4%, 84 4%, 84 5%, 84 5%, 83 5% 84 5% + // generation 94: 84 4%, 83 4%, 83 5%, 85 5%, 83 4%, 83 5%, 83 4%, 83 4%, 83 4%, 83 4%, 83 5%, 84 4%, 83 5%, 83 4%, 83 4%, 84 4%, 84 5%, 84 5%, 83 5% 84 5% + // generation 96: 84 4%, 83 4%, 83 5%, 85 5%, 83 4%, 83 5%, 83 4%, 83 4%, 83 4%, 83 4%, 83 5%, 84 4%, 83 5%, 83 4%, 83 4%, 84 4%, 84 5%, 84 5%, 83 5% 84 5% + // generation 98: 84 4%, 83 4%, 83 5%, 85 5%, 83 4%, 83 5%, 83 4%, 83 4%, 83 4%, 83 4%, 83 5%, 84 4%, 83 5%, 83 4%, 83 4%, 84 4%, 84 5%, 84 5%, 83 5% 84 5% // Total bytes=872399094, ranges=1668 }