From f382b471f2a3a51afc68e7fe7c7ad3012bf8543f Mon Sep 17 00:00:00 2001 From: Luffbee Date: Sun, 2 Feb 2020 20:56:35 +0800 Subject: [PATCH 01/29] try more solution --- server/schedulers/hot_region.go | 297 ++++++++++++++------------------ server/schedulers/utils.go | 38 ++++ 2 files changed, 167 insertions(+), 168 deletions(-) diff --git a/server/schedulers/hot_region.go b/server/schedulers/hot_region.go index 98038a6f9c4..bad49460b12 100644 --- a/server/schedulers/hot_region.go +++ b/server/schedulers/hot_region.go @@ -357,23 +357,18 @@ func (h *hotScheduler) balanceHotReadRegions(cluster opt.Cluster) []*operator.Op return nil } -// balanceHotRetryLimit is the limit to retry schedule for selected balance strategy. -const balanceHotRetryLimit = 5 - func (h *hotScheduler) balanceHotWriteRegions(cluster opt.Cluster) []*operator.Operator { - for i := 0; i < balanceHotRetryLimit; i++ { - // prefer to balance by peer - peerSolver := newBalanceSolver(h, cluster, write, movePeer) - ops := peerSolver.solve() - if len(ops) > 0 { - return ops - } + // prefer to balance by peer + peerSolver := newBalanceSolver(h, cluster, write, movePeer) + ops := peerSolver.solve() + if len(ops) > 0 { + return ops + } - leaderSolver := newBalanceSolver(h, cluster, write, transferLeader) - ops = leaderSolver.solve() - if len(ops) > 0 { - return ops - } + leaderSolver := newBalanceSolver(h, cluster, write, transferLeader) + ops = leaderSolver.solve() + if len(ops) > 0 { + return ops } schedulerCounter.WithLabelValues(h.GetName(), "skip").Inc() @@ -387,7 +382,12 @@ type balanceSolver struct { rwTy rwType opTy opType - // temporary states + cur *solution + best *solution + ops []*operator.Operator +} + +type solution struct { srcStoreID uint64 srcPeerStat *statistics.HotPeerStat region *core.RegionInfo @@ -455,28 +455,36 @@ func (bs *balanceSolver) solve() []*operator.Operator { if !bs.isValid() || !bs.allowBalance() { return nil } - bs.srcStoreID = bs.selectSrcStoreID() - if bs.srcStoreID == 0 { - return nil - } + bs.cur = &solution{} + bs.best = nil + bs.ops = nil - for _, srcPeerStat := range bs.getPeerList() { - bs.srcPeerStat = srcPeerStat - bs.region = bs.getRegion() - if bs.region == nil { - continue - } - dstCandidates := bs.getDstCandidateIDs() - if len(dstCandidates) <= 0 { - continue - } - bs.dstStoreID = bs.selectDstStoreID(dstCandidates) - ops := bs.buildOperators() - if len(ops) > 0 { - return ops + srcStores := bs.sortSrcStoreIDs(bs.filterSrcStores()) + for _, srcStoreID := range srcStores { + bs.cur.srcStoreID = srcStoreID + + for _, srcPeerStat := range bs.sortHotPeers() { + bs.cur.srcPeerStat = srcPeerStat + bs.cur.region = bs.getRegion() + if bs.cur.region == nil { + continue + } + + dstStores := bs.sortDstStoreIDs(bs.filterDstStores()) + for _, dstStoreID := range dstStores { + bs.cur.dstStoreID = dstStoreID + + if bs.betterSolution() { + if ops := bs.buildOperators(); len(ops) > 0 { + bs.ops = ops + clone := *bs.cur + bs.best = &clone + } + } + } } } - return nil + return bs.ops } func (bs *balanceSolver) allowBalance() bool { @@ -490,28 +498,35 @@ func (bs *balanceSolver) allowBalance() bool { } } -func (bs *balanceSolver) selectSrcStoreID() uint64 { - var id uint64 - switch bs.opTy { - case movePeer: - id = selectSrcStoreByByteRate(bs.stLoadDetail) - case transferLeader: - if bs.rwTy == write { - id = selectSrcStoreByCount(bs.stLoadDetail) - } else { - id = selectSrcStoreByByteRate(bs.stLoadDetail) +func (bs *balanceSolver) filterSrcStores() map[uint64]*storeLoadDetail { + ret := make(map[uint64]*storeLoadDetail) + for id, detail := range bs.stLoadDetail { + if bs.cluster.GetStore(id) == nil { + log.Error("failed to get the source store", zap.Uint64("store-id", id)) + continue } + if len(detail.HotPeers) == 0 { + continue + } + ret[id] = detail } - if id != 0 && bs.cluster.GetStore(id) == nil { - log.Error("failed to get the source store", zap.Uint64("store-id", id)) + return ret +} + +func (bs *balanceSolver) sortSrcStoreIDs(loadDetail map[uint64]*storeLoadDetail) []uint64 { + var cmp storeLoadCmp + if bs.opTy == transferLeader && bs.rwTy == write { + cmp = neg(sliceCmp(countCmp, byteRateCmp)) + } else { + cmp = neg(sliceCmp(byteRateCmp, countCmp)) } - return id + return sortSrcStores(loadDetail, cmp) } -func (bs *balanceSolver) getPeerList() []*statistics.HotPeerStat { - ret := bs.stLoadDetail[bs.srcStoreID].HotPeers - bs.sche.r.Shuffle(len(ret), func(i, j int) { - ret[i], ret[j] = ret[j], ret[i] +func (bs *balanceSolver) sortHotPeers() []*statistics.HotPeerStat { + ret := bs.stLoadDetail[bs.cur.srcStoreID].HotPeers + sort.Slice(ret, func(i, j int) bool { + return ret[i].GetBytesRate() > ret[j].GetBytesRate() }) return ret } @@ -548,21 +563,21 @@ func (bs *balanceSolver) isRegionAvailable(region *core.RegionInfo) bool { } func (bs *balanceSolver) getRegion() *core.RegionInfo { - region := bs.cluster.GetRegion(bs.srcPeerStat.ID()) + region := bs.cluster.GetRegion(bs.cur.srcPeerStat.ID()) if !bs.isRegionAvailable(region) { return nil } switch bs.opTy { case movePeer: - srcPeer := region.GetStorePeer(bs.srcStoreID) + srcPeer := region.GetStorePeer(bs.cur.srcStoreID) if srcPeer == nil { - log.Debug("region does not have a peer on source store, maybe stat out of date", zap.Uint64("region-id", bs.srcPeerStat.ID())) + log.Debug("region does not have a peer on source store, maybe stat out of date", zap.Uint64("region-id", bs.cur.srcPeerStat.ID())) return nil } case transferLeader: - if region.GetLeader().GetStoreId() != bs.srcStoreID { - log.Debug("region leader is not on source store, maybe stat out of date", zap.Uint64("region-id", bs.srcPeerStat.ID())) + if region.GetLeader().GetStoreId() != bs.cur.srcStoreID { + log.Debug("region leader is not on source store, maybe stat out of date", zap.Uint64("region-id", bs.cur.srcPeerStat.ID())) return nil } default: @@ -572,7 +587,7 @@ func (bs *balanceSolver) getRegion() *core.RegionInfo { return region } -func (bs *balanceSolver) getDstCandidateIDs() map[uint64]struct{} { +func (bs *balanceSolver) filterDstStores() map[uint64]*storeLoadDetail { var ( filters []filter.Filter candidates []*core.StoreInfo @@ -582,18 +597,18 @@ func (bs *balanceSolver) getDstCandidateIDs() map[uint64]struct{} { case movePeer: var scoreGuard filter.Filter if bs.cluster.IsPlacementRulesEnabled() { - scoreGuard = filter.NewRuleFitFilter(bs.sche.GetName(), bs.cluster, bs.region, bs.srcStoreID) + scoreGuard = filter.NewRuleFitFilter(bs.sche.GetName(), bs.cluster, bs.cur.region, bs.cur.srcStoreID) } else { - srcStore := bs.cluster.GetStore(bs.srcStoreID) + srcStore := bs.cluster.GetStore(bs.cur.srcStoreID) if srcStore == nil { return nil } - scoreGuard = filter.NewDistinctScoreFilter(bs.sche.GetName(), bs.cluster.GetLocationLabels(), bs.cluster.GetRegionStores(bs.region), srcStore) + scoreGuard = filter.NewDistinctScoreFilter(bs.sche.GetName(), bs.cluster.GetLocationLabels(), bs.cluster.GetRegionStores(bs.cur.region), srcStore) } filters = []filter.Filter{ filter.StoreStateFilter{ActionScope: bs.sche.GetName(), MoveRegion: true}, - filter.NewExcludedFilter(bs.sche.GetName(), bs.region.GetStoreIds(), bs.region.GetStoreIds()), + filter.NewExcludedFilter(bs.sche.GetName(), bs.cur.region.GetStoreIds(), bs.cur.region.GetStoreIds()), filter.NewHealthFilter(bs.sche.GetName()), scoreGuard, } @@ -606,46 +621,55 @@ func (bs *balanceSolver) getDstCandidateIDs() map[uint64]struct{} { filter.NewHealthFilter(bs.sche.GetName()), } - candidates = bs.cluster.GetFollowerStores(bs.region) + candidates = bs.cluster.GetFollowerStores(bs.cur.region) default: return nil } - ret := make(map[uint64]struct{}, len(candidates)) + srcLd := bs.stLoadDetail[bs.cur.srcStoreID].LoadPred.min() + ret := make(map[uint64]*storeLoadDetail, len(candidates)) for _, store := range candidates { if !filter.Target(bs.cluster, store, filters) { - ret[store.GetID()] = struct{}{} + detail := bs.stLoadDetail[store.GetID()] + dstLd := detail.LoadPred.max() + if bs.opTy == transferLeader && bs.rwTy == write { + if srcLd.Count-1 < dstLd.Count+1 { + continue + } + } else { + if srcLd.ByteRate*hotRegionScheduleFactor < dstLd.ByteRate+bs.cur.srcPeerStat.GetBytesRate() { + continue + } + } + ret[store.GetID()] = detail } } return ret } -func (bs *balanceSolver) selectDstStoreID(candidateIDs map[uint64]struct{}) uint64 { - candidateLoadDetail := make(map[uint64]*storeLoadDetail, len(candidateIDs)) - for id := range candidateIDs { - candidateLoadDetail[id] = bs.stLoadDetail[id] - } - switch bs.opTy { - case movePeer: - return selectDstStoreByByteRate(candidateLoadDetail, bs.srcPeerStat.GetBytesRate(), bs.stLoadDetail[bs.srcStoreID]) - case transferLeader: - if bs.rwTy == write { - return selectDstStoreByCount(candidateLoadDetail, bs.srcPeerStat.GetBytesRate(), bs.stLoadDetail[bs.srcStoreID]) - } - return selectDstStoreByByteRate(candidateLoadDetail, bs.srcPeerStat.GetBytesRate(), bs.stLoadDetail[bs.srcStoreID]) - default: - return 0 +func (bs *balanceSolver) sortDstStoreIDs(loadDetail map[uint64]*storeLoadDetail) []uint64 { + var cmp storeLoadCmp + if bs.opTy == transferLeader && bs.rwTy == write { + cmp = sliceCmp(countCmp, byteRateCmp) + } else { + cmp = sliceCmp(byteRateCmp, countCmp) } + return sortDstStores(loadDetail, cmp) +} + +// betterSolution returns true if `bs.cur` is a betterSolution solution than `bs.best`. +func (bs *balanceSolver) betterSolution() bool { + return bs.best == nil } func (bs *balanceSolver) isReadyToBuild() bool { - if bs.srcStoreID == 0 || bs.dstStoreID == 0 || - bs.srcPeerStat == nil || bs.region == nil { + if bs.cur.srcStoreID == 0 || bs.cur.dstStoreID == 0 || + bs.cur.srcPeerStat == nil || bs.cur.region == nil { return false } - if bs.srcStoreID != bs.srcPeerStat.StoreID || - bs.region.GetID() != bs.srcPeerStat.ID() { + if bs.cur.srcStoreID != bs.cur.srcPeerStat.StoreID || + bs.cur.region.GetID() != bs.cur.srcPeerStat.ID() { return false } return true @@ -662,16 +686,16 @@ func (bs *balanceSolver) buildOperators() []*operator.Operator { switch bs.opTy { case movePeer: - srcPeer := bs.region.GetStorePeer(bs.srcStoreID) // checked in getRegionAndSrcPeer - dstPeer := &metapb.Peer{StoreId: bs.dstStoreID, IsLearner: srcPeer.IsLearner} - bs.sche.peerLimit = bs.sche.adjustBalanceLimit(bs.srcStoreID, bs.stLoadDetail) - op, err = operator.CreateMovePeerOperator("move-hot-"+bs.rwTy.String()+"-region", bs.cluster, bs.region, operator.OpHotRegion, bs.srcStoreID, dstPeer) + srcPeer := bs.cur.region.GetStorePeer(bs.cur.srcStoreID) // checked in getRegionAndSrcPeer + dstPeer := &metapb.Peer{StoreId: bs.cur.dstStoreID, IsLearner: srcPeer.IsLearner} + bs.sche.peerLimit = bs.sche.adjustBalanceLimit(bs.cur.srcStoreID, bs.stLoadDetail) + op, err = operator.CreateMovePeerOperator("move-hot-"+bs.rwTy.String()+"-region", bs.cluster, bs.cur.region, operator.OpHotRegion, bs.cur.srcStoreID, dstPeer) case transferLeader: - if bs.region.GetStoreVoter(bs.dstStoreID) == nil { + if bs.cur.region.GetStoreVoter(bs.cur.dstStoreID) == nil { return nil } - bs.sche.leaderLimit = bs.sche.adjustBalanceLimit(bs.srcStoreID, bs.stLoadDetail) - op, err = operator.CreateTransferLeaderOperator("transfer-hot-"+bs.rwTy.String()+"-leader", bs.cluster, bs.region, bs.srcStoreID, bs.dstStoreID, operator.OpHotRegion) + bs.sche.leaderLimit = bs.sche.adjustBalanceLimit(bs.cur.srcStoreID, bs.stLoadDetail) + op, err = operator.CreateTransferLeaderOperator("transfer-hot-"+bs.rwTy.String()+"-leader", bs.cluster, bs.cur.region, bs.cur.srcStoreID, bs.cur.dstStoreID, operator.OpHotRegion) } if err != nil { @@ -685,11 +709,11 @@ func (bs *balanceSolver) buildOperators() []*operator.Operator { schedulerCounter.WithLabelValues(bs.sche.GetName(), "new-operator"), schedulerCounter.WithLabelValues(bs.sche.GetName(), bs.opTy.String())) - infl := Influence{ByteRate: bs.srcPeerStat.GetBytesRate()} + infl := Influence{ByteRate: bs.cur.srcPeerStat.GetBytesRate()} if bs.opTy == transferLeader && bs.rwTy == write { infl.ByteRate = 0 } - bs.sche.addPendingInfluence(op, bs.srcStoreID, bs.dstStoreID, infl, bs.rwTy, bs.opTy) + bs.sche.addPendingInfluence(op, bs.cur.srcStoreID, bs.cur.dstStoreID, infl, bs.rwTy, bs.opTy) return []*operator.Operator{op} } @@ -707,83 +731,20 @@ func sortStores(loadDetail map[uint64]*storeLoadDetail, better func(lp1, lp2 *st return ids } -// Prefer store with larger `count`. -func selectSrcStoreByCount(loadDetail map[uint64]*storeLoadDetail) uint64 { - stores := sortStores(loadDetail, func(lp1, lp2 *storeLoadPred) bool { - ld1, ld2 := lp1.min(), lp2.min() - if ld1.Count > ld2.Count || - (ld1.Count == ld2.Count && ld1.ByteRate > ld2.ByteRate) { - return true - } - return false - }) - - if len(stores) > 0 && loadDetail[stores[0]].LoadPred.Current.Count > 1 { - return stores[0] - } - return 0 -} - -// Prefer store with larger `byteRate`. -func selectSrcStoreByByteRate(loadDetail map[uint64]*storeLoadDetail) uint64 { - stores := sortStores(loadDetail, func(lp1, lp2 *storeLoadPred) bool { - ld1, ld2 := lp1.min(), lp2.min() - if ld1.ByteRate > ld2.ByteRate || - (ld1.ByteRate == ld2.ByteRate && ld1.Count > ld2.Count) { - return true - } - return false - }) - - for _, id := range stores { - if loadDetail[id].LoadPred.Current.Count > 1 { - return id - } - } - return 0 -} - -// Prefer store with smaller `count`. -func selectDstStoreByCount(candidates map[uint64]*storeLoadDetail, regionBytesRate float64, srcLoadDetail *storeLoadDetail) uint64 { - stores := sortStores(candidates, func(lp1, lp2 *storeLoadPred) bool { - ld1, ld2 := lp1.max(), lp2.max() - if ld1.Count < ld2.Count || - (ld1.Count == ld2.Count && ld1.ByteRate < ld2.ByteRate) { - return true - } - return false - }) - - srcLoad := srcLoadDetail.LoadPred.min() - for _, id := range stores { - dstLoad := candidates[id].LoadPred.max() - if srcLoad.Count-1 >= dstLoad.Count+1 && - srcLoad.ByteRate*hotRegionScheduleFactor > dstLoad.ByteRate+regionBytesRate { - return id - } - } - return 0 +func sortSrcStores(loadDetail map[uint64]*storeLoadDetail, cmp storeLoadCmp) []uint64 { + return sortStores(loadDetail, + func(lp1, lp2 *storeLoadPred) bool { + ld1, ld2 := lp1.min(), lp2.min() + return cmp(&ld1, &ld2) < 0 + }) } -// Prefer store with smaller `byteRate`. -func selectDstStoreByByteRate(candidates map[uint64]*storeLoadDetail, regionBytesRate float64, srcLoadDetail *storeLoadDetail) uint64 { - stores := sortStores(candidates, func(lp1, lp2 *storeLoadPred) bool { - ld1, ld2 := lp1.max(), lp2.max() - if ld1.ByteRate < ld2.ByteRate || - (ld1.ByteRate == ld2.ByteRate && ld1.Count < ld2.Count) { - return true - } - return false - }) - - srcLoad := srcLoadDetail.LoadPred.min() - for _, id := range stores { - dstLoad := candidates[id].LoadPred.max() - if srcLoad.ByteRate*hotRegionScheduleFactor > dstLoad.ByteRate+regionBytesRate { - return id - } - } - return 0 +func sortDstStores(loadDetail map[uint64]*storeLoadDetail, cmp storeLoadCmp) []uint64 { + return sortStores(loadDetail, + func(lp1, lp2 *storeLoadPred) bool { + ld1, ld2 := lp1.max(), lp2.max() + return cmp(&ld1, &ld2) < 0 + }) } func (h *hotScheduler) adjustBalanceLimit(storeID uint64, loadDetail map[uint64]*storeLoadDetail) uint64 { diff --git a/server/schedulers/utils.go b/server/schedulers/utils.go index 682c43b2320..9f38579c5b8 100644 --- a/server/schedulers/utils.go +++ b/server/schedulers/utils.go @@ -214,6 +214,44 @@ func (load *storeLoad) ToLoadPred(infl Influence) *storeLoadPred { } } +type storeLoadCmp func(ld1, ld2 *storeLoad) int + +func sliceCmp(cmps ...storeLoadCmp) storeLoadCmp { + return func(ld1, ld2 *storeLoad) int { + for _, cmp := range cmps { + r := cmp(ld1, ld2) + if r != 0 { + return r + } + } + return 0 + } +} + +func neg(cmp storeLoadCmp) storeLoadCmp { + return func(ld1, ld2 *storeLoad) int { + return -cmp(ld1, ld2) + } +} + +func byteRateCmp(ld1, ld2 *storeLoad) int { + if ld1.ByteRate < ld2.ByteRate { + return -1 + } else if ld1.ByteRate > ld2.ByteRate { + return 1 + } + return 0 +} + +func countCmp(ld1, ld2 *storeLoad) int { + if ld1.Count < ld2.Count { + return -1 + } else if ld1.Count > ld2.Count { + return 1 + } + return 0 +} + // store load prediction type storeLoadPred struct { Current storeLoad From 67d6f24d73053e0650b3fddb8aad61ff752b0389 Mon Sep 17 00:00:00 2001 From: Luffbee Date: Tue, 4 Feb 2020 18:23:16 +0800 Subject: [PATCH 02/29] function betterThan --- server/schedulers/hot_region.go | 171 +++++++++++++++++++------------- server/schedulers/hot_test.go | 28 ++++-- server/schedulers/utils.go | 58 +++++------ 3 files changed, 146 insertions(+), 111 deletions(-) diff --git a/server/schedulers/hot_region.go b/server/schedulers/hot_region.go index bad49460b12..ba7ea70b524 100644 --- a/server/schedulers/hot_region.go +++ b/server/schedulers/hot_region.go @@ -382,9 +382,7 @@ type balanceSolver struct { rwTy rwType opTy opType - cur *solution - best *solution - ops []*operator.Operator + cur *solution } type solution struct { @@ -456,35 +454,39 @@ func (bs *balanceSolver) solve() []*operator.Operator { return nil } bs.cur = &solution{} - bs.best = nil - bs.ops = nil + var best *solution = nil + var ops []*operator.Operator = nil + var infls []Influence = nil - srcStores := bs.sortSrcStoreIDs(bs.filterSrcStores()) - for _, srcStoreID := range srcStores { + for srcStoreID := range bs.filterSrcStores() { bs.cur.srcStoreID = srcStoreID - for _, srcPeerStat := range bs.sortHotPeers() { + for _, srcPeerStat := range bs.getHotPeers() { bs.cur.srcPeerStat = srcPeerStat bs.cur.region = bs.getRegion() if bs.cur.region == nil { continue } - dstStores := bs.sortDstStoreIDs(bs.filterDstStores()) - for _, dstStoreID := range dstStores { + for dstStoreID := range bs.filterDstStores() { bs.cur.dstStoreID = dstStoreID - if bs.betterSolution() { - if ops := bs.buildOperators(); len(ops) > 0 { - bs.ops = ops + if bs.betterThan(best) { + if newOps, newInfls := bs.buildOperators(); len(newOps) > 0 { + ops = newOps + infls = newInfls clone := *bs.cur - bs.best = &clone + best = &clone } } } } } - return bs.ops + + for i := 0; i < len(ops); i++ { + bs.sche.addPendingInfluence(ops[i], best.srcStoreID, best.dstStoreID, infls[i], bs.rwTy, bs.opTy) + } + return ops } func (bs *balanceSolver) allowBalance() bool { @@ -513,21 +515,14 @@ func (bs *balanceSolver) filterSrcStores() map[uint64]*storeLoadDetail { return ret } -func (bs *balanceSolver) sortSrcStoreIDs(loadDetail map[uint64]*storeLoadDetail) []uint64 { - var cmp storeLoadCmp - if bs.opTy == transferLeader && bs.rwTy == write { - cmp = neg(sliceCmp(countCmp, byteRateCmp)) - } else { - cmp = neg(sliceCmp(byteRateCmp, countCmp)) - } - return sortSrcStores(loadDetail, cmp) -} - -func (bs *balanceSolver) sortHotPeers() []*statistics.HotPeerStat { +func (bs *balanceSolver) getHotPeers() []*statistics.HotPeerStat { ret := bs.stLoadDetail[bs.cur.srcStoreID].HotPeers sort.Slice(ret, func(i, j int) bool { return ret[i].GetBytesRate() > ret[j].GetBytesRate() }) + if len(ret) > 1000 { + ret = ret[:1000] + } return ret } @@ -648,19 +643,83 @@ func (bs *balanceSolver) filterDstStores() map[uint64]*storeLoadDetail { return ret } -func (bs *balanceSolver) sortDstStoreIDs(loadDetail map[uint64]*storeLoadDetail) []uint64 { - var cmp storeLoadCmp - if bs.opTy == transferLeader && bs.rwTy == write { - cmp = sliceCmp(countCmp, byteRateCmp) - } else { - cmp = sliceCmp(byteRateCmp, countCmp) +// betterSolution returns true if `bs.cur` is a betterSolution solution than `bs.best`. +func (bs *balanceSolver) betterThan(old *solution) bool { + if old == nil { + return true + } + + if r := bs.compareSrcStore(bs.cur.srcStoreID, old.srcStoreID); r < 0 { + return true + } else if r > 0 { + return false + } + + if r := bs.compareDstStore(bs.cur.dstStoreID, old.dstStoreID); r < 0 { + return true + } else if r > 0 { + return false } - return sortDstStores(loadDetail, cmp) + + if bs.cur.srcPeerStat != old.srcPeerStat { + // compare region + + // prefer region with larger byte rate, to converge faster + curByteRk, oldByteRk := int64(bs.cur.srcPeerStat.GetBytesRate()/100), int64(old.srcPeerStat.GetBytesRate()/100) + if curByteRk > oldByteRk { + return true + } else if curByteRk < oldByteRk { + return false + } + } + + return false } -// betterSolution returns true if `bs.cur` is a betterSolution solution than `bs.best`. -func (bs *balanceSolver) betterSolution() bool { - return bs.best == nil +// smaller is better +func (bs *balanceSolver) compareSrcStore(st1, st2 uint64) int { + if st1 != st2 { + // compare source store + lp1, lp2 := bs.stLoadDetail[st1].LoadPred, bs.stLoadDetail[st2].LoadPred + + if bs.opTy == transferLeader && bs.rwTy == write { + if r := countCmp(lp1.min(), lp2.min()); r != 0 { + return -r + } + } + if r := byteRateRankCmp(lp1.min(), lp2.min()); r != 0 { + return -r + } + + // prefer store with smaller difference + if r := byteRateRankCmp(lp1.diff(), lp2.diff()); r != 0 { + return r + } + } + return 0 +} + +// smaller is better +func (bs *balanceSolver) compareDstStore(st1, st2 uint64) int { + if st1 != st2 { + // compare source store + lp1, lp2 := bs.stLoadDetail[st1].LoadPred, bs.stLoadDetail[st2].LoadPred + + if bs.opTy == transferLeader && bs.rwTy == write { + if r := countCmp(lp1.max(), lp2.max()); r != 0 { + return r + } + } + if r := byteRateRankCmp(lp1.max(), lp2.max()); r != 0 { + return r + } + + // prefer store with smaller difference + if r := byteRateRankCmp(lp1.diff(), lp2.diff()); r != 0 { + return r + } + } + return 0 } func (bs *balanceSolver) isReadyToBuild() bool { @@ -675,9 +734,9 @@ func (bs *balanceSolver) isReadyToBuild() bool { return true } -func (bs *balanceSolver) buildOperators() []*operator.Operator { +func (bs *balanceSolver) buildOperators() ([]*operator.Operator, []Influence) { if !bs.isReadyToBuild() { - return nil + return nil, nil } var ( op *operator.Operator @@ -692,7 +751,7 @@ func (bs *balanceSolver) buildOperators() []*operator.Operator { op, err = operator.CreateMovePeerOperator("move-hot-"+bs.rwTy.String()+"-region", bs.cluster, bs.cur.region, operator.OpHotRegion, bs.cur.srcStoreID, dstPeer) case transferLeader: if bs.cur.region.GetStoreVoter(bs.cur.dstStoreID) == nil { - return nil + return nil, nil } bs.sche.leaderLimit = bs.sche.adjustBalanceLimit(bs.cur.srcStoreID, bs.stLoadDetail) op, err = operator.CreateTransferLeaderOperator("transfer-hot-"+bs.rwTy.String()+"-leader", bs.cluster, bs.cur.region, bs.cur.srcStoreID, bs.cur.dstStoreID, operator.OpHotRegion) @@ -701,7 +760,7 @@ func (bs *balanceSolver) buildOperators() []*operator.Operator { if err != nil { log.Debug("fail to create operator", zap.Error(err), zap.Stringer("opType", bs.opTy), zap.Stringer("rwType", bs.rwTy)) schedulerCounter.WithLabelValues(bs.sche.GetName(), "create-operator-fail").Inc() - return nil + return nil, nil } op.SetPriorityLevel(core.HighPriority) @@ -713,38 +772,8 @@ func (bs *balanceSolver) buildOperators() []*operator.Operator { if bs.opTy == transferLeader && bs.rwTy == write { infl.ByteRate = 0 } - bs.sche.addPendingInfluence(op, bs.cur.srcStoreID, bs.cur.dstStoreID, infl, bs.rwTy, bs.opTy) - - return []*operator.Operator{op} -} - -// Sort stores according to their load prediction. -func sortStores(loadDetail map[uint64]*storeLoadDetail, better func(lp1, lp2 *storeLoadPred) bool) []uint64 { - ids := make([]uint64, 0, len(loadDetail)) - for id := range loadDetail { - ids = append(ids, id) - } - sort.Slice(ids, func(i, j int) bool { - id1, id2 := ids[i], ids[j] - return better(loadDetail[id1].LoadPred, loadDetail[id2].LoadPred) - }) - return ids -} - -func sortSrcStores(loadDetail map[uint64]*storeLoadDetail, cmp storeLoadCmp) []uint64 { - return sortStores(loadDetail, - func(lp1, lp2 *storeLoadPred) bool { - ld1, ld2 := lp1.min(), lp2.min() - return cmp(&ld1, &ld2) < 0 - }) -} -func sortDstStores(loadDetail map[uint64]*storeLoadDetail, cmp storeLoadCmp) []uint64 { - return sortStores(loadDetail, - func(lp1, lp2 *storeLoadPred) bool { - ld1, ld2 := lp1.max(), lp2.max() - return cmp(&ld1, &ld2) < 0 - }) + return []*operator.Operator{op}, []Influence{infl} } func (h *hotScheduler) adjustBalanceLimit(storeID uint64, loadDetail map[uint64]*storeLoadDetail) uint64 { diff --git a/server/schedulers/hot_test.go b/server/schedulers/hot_test.go index b0aeeca1c8c..82f74750db2 100644 --- a/server/schedulers/hot_test.go +++ b/server/schedulers/hot_test.go @@ -439,14 +439,14 @@ func (s *testHotReadRegionSchedulerSuite) TestSchedule(c *C) { //| store_id | read_bytes_rate | //|----------|-----------------| - //| 1 | 6MB | + //| 1 | 6MB | //| 2 | 5.5MB | - //| 3 | 6MB | + //| 3 | 5.9MB | //| 4 | 3.1MB | //| 5 | 3MB | tc.UpdateStorageReadBytes(1, 6*MB*statistics.StoreHeartBeatReportInterval) tc.UpdateStorageReadBytes(2, 5.5*MB*statistics.StoreHeartBeatReportInterval) - tc.UpdateStorageReadBytes(3, 6*MB*statistics.StoreHeartBeatReportInterval) + tc.UpdateStorageReadBytes(3, 5.9*MB*statistics.StoreHeartBeatReportInterval) tc.UpdateStorageReadBytes(4, 3.1*MB*statistics.StoreHeartBeatReportInterval) tc.UpdateStorageReadBytes(5, 3*MB*statistics.StoreHeartBeatReportInterval) @@ -462,11 +462,6 @@ func (s *testHotReadRegionSchedulerSuite) TestSchedule(c *C) { tc.AddLeaderRegionWithReadInfo(5, 4, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 2, 5) // We will move leader peer of region 1 from 1 to 5 - // Store 1 will be selected as source store (max rate, count > store 3 count). - // When trying to transfer leader: - // Store 2 and store 3 are also hot, failed. - // Trying to move leader peer: - // Store 5 is selected as destination because of less hot region count. testutil.CheckTransferPeerWithLeaderTransfer(c, hb.Schedule(tc)[0], operator.OpHotRegion, 1, 5) hb.(*hotScheduler).clearPendingInfluence() @@ -524,25 +519,42 @@ func (s *testHotReadRegionSchedulerSuite) TestWithPendingInfluence(c *C) { for i := 0; i < 20; i++ { hb.(*hotScheduler).clearPendingInfluence() + op1 := hb.Schedule(tc)[0] testutil.CheckTransferLeader(c, op1, operator.OpLeader, 1, 3) + // store byte rate (min, max): (6.6, 7.1) | 6.1 | (6, 6.5) | 5 + op2 := hb.Schedule(tc)[0] testutil.CheckTransferPeerWithLeaderTransfer(c, op2, operator.OpHotRegion, 1, 4) + // store byte rate (min, max): (6.1, 7.1) | 6.1 | (6, 6.5) | (5, 5.5) + ops := hb.Schedule(tc) c.Assert(ops, HasLen, 0) } for i := 0; i < 20; i++ { hb.(*hotScheduler).clearPendingInfluence() + op1 := hb.Schedule(tc)[0] testutil.CheckTransferLeader(c, op1, operator.OpLeader, 1, 3) + // store byte rate (min, max): (6.6, 7.1) | 6.1 | (6, 6.5) | 5 + op2 := hb.Schedule(tc)[0] testutil.CheckTransferPeerWithLeaderTransfer(c, op2, operator.OpHotRegion, 1, 4) + // store byte rate (min, max): (6.1, 7.1) | 6.1 | (6, 6.5) | (5, 5.5) c.Assert(op2.Cancel(), IsTrue) + // store byte rate (min, max): (6.6, 7.1) | 6.1 | (6, 6.5) | 5 + op2 = hb.Schedule(tc)[0] testutil.CheckTransferPeerWithLeaderTransfer(c, op2, operator.OpHotRegion, 1, 4) + // store byte rate (min, max): (6.1, 7.1) | 6.1 | (6, 6.5) | (5, 5.5) + c.Assert(op1.Cancel(), IsTrue) + // store byte rate (min, max): (6.6, 7.1) | 6.1 | 6 | (5, 5.5) + op3 := hb.Schedule(tc)[0] testutil.CheckTransferPeerWithLeaderTransfer(c, op3, operator.OpHotRegion, 1, 4) + // store byte rate (min, max): (6.1, 7.1) | 6.1 | 6 | (5, 6) + ops := hb.Schedule(tc) c.Assert(ops, HasLen, 0) } diff --git a/server/schedulers/utils.go b/server/schedulers/utils.go index 9f38579c5b8..1493346fd17 100644 --- a/server/schedulers/utils.go +++ b/server/schedulers/utils.go @@ -31,9 +31,10 @@ import ( const ( // adjustRatio is used to adjust TolerantSizeRatio according to region count. - adjustRatio float64 = 0.005 - leaderTolerantSizeRatio float64 = 5.0 - minTolerantSizeRatio float64 = 1.0 + adjustRatio float64 = 0.005 + leaderTolerantSizeRatio float64 = 5.0 + minTolerantSizeRatio float64 = 1.0 + storeLoadByteRateRankSize = 100 * 1024 ) // ErrScheduleConfigNotExist the config is not correct. @@ -205,6 +206,10 @@ type storeLoad struct { Count int } +func (load *storeLoad) ByteRateRank() int64 { + return int64(load.ByteRate / storeLoadByteRateRankSize) +} + func (load *storeLoad) ToLoadPred(infl Influence) *storeLoadPred { future := *load future.ByteRate += infl.ByteRate @@ -214,30 +219,11 @@ func (load *storeLoad) ToLoadPred(infl Influence) *storeLoadPred { } } -type storeLoadCmp func(ld1, ld2 *storeLoad) int - -func sliceCmp(cmps ...storeLoadCmp) storeLoadCmp { - return func(ld1, ld2 *storeLoad) int { - for _, cmp := range cmps { - r := cmp(ld1, ld2) - if r != 0 { - return r - } - } - return 0 - } -} - -func neg(cmp storeLoadCmp) storeLoadCmp { - return func(ld1, ld2 *storeLoad) int { - return -cmp(ld1, ld2) - } -} - -func byteRateCmp(ld1, ld2 *storeLoad) int { - if ld1.ByteRate < ld2.ByteRate { +func byteRateRankCmp(ld1, ld2 *storeLoad) int { + rk1, rk2 := ld1.ByteRateRank(), ld2.ByteRateRank() + if rk1 < rk2 { return -1 - } else if ld1.ByteRate > ld2.ByteRate { + } else if rk1 > rk2 { return 1 } return 0 @@ -258,23 +244,31 @@ type storeLoadPred struct { Future storeLoad } -func (lp *storeLoadPred) min() storeLoad { +func (lp *storeLoadPred) min() *storeLoad { return minLoad(&lp.Current, &lp.Future) } -func (lp *storeLoadPred) max() storeLoad { +func (lp *storeLoadPred) max() *storeLoad { return maxLoad(&lp.Current, &lp.Future) } -func minLoad(a, b *storeLoad) storeLoad { - return storeLoad{ +func (lp *storeLoadPred) diff() *storeLoad { + mx, mn := lp.max(), lp.min() + return &storeLoad{ + ByteRate: mx.ByteRate - mn.ByteRate, + Count: mx.Count - mn.Count, + } +} + +func minLoad(a, b *storeLoad) *storeLoad { + return &storeLoad{ ByteRate: math.Min(a.ByteRate, b.ByteRate), Count: minInt(a.Count, b.Count), } } -func maxLoad(a, b *storeLoad) storeLoad { - return storeLoad{ +func maxLoad(a, b *storeLoad) *storeLoad { + return &storeLoad{ ByteRate: math.Max(a.ByteRate, b.ByteRate), Count: maxInt(a.Count, b.Count), } From 1edb33ae781eb1db43651633e488d2c012863ee3 Mon Sep 17 00:00:00 2001 From: Luffbee Date: Thu, 6 Feb 2020 20:10:21 +0800 Subject: [PATCH 03/29] add pending influence of count --- server/schedulers/hot_region.go | 2 +- server/schedulers/utils.go | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/server/schedulers/hot_region.go b/server/schedulers/hot_region.go index ba7ea70b524..9c34e3c769a 100644 --- a/server/schedulers/hot_region.go +++ b/server/schedulers/hot_region.go @@ -768,7 +768,7 @@ func (bs *balanceSolver) buildOperators() ([]*operator.Operator, []Influence) { schedulerCounter.WithLabelValues(bs.sche.GetName(), "new-operator"), schedulerCounter.WithLabelValues(bs.sche.GetName(), bs.opTy.String())) - infl := Influence{ByteRate: bs.cur.srcPeerStat.GetBytesRate()} + infl := Influence{ByteRate: bs.cur.srcPeerStat.GetBytesRate(), Count: 1} if bs.opTy == transferLeader && bs.rwTy == write { infl.ByteRate = 0 } diff --git a/server/schedulers/utils.go b/server/schedulers/utils.go index 1493346fd17..4ef73e1b996 100644 --- a/server/schedulers/utils.go +++ b/server/schedulers/utils.go @@ -165,10 +165,12 @@ func getKeyRanges(args []string) ([]core.KeyRange, error) { // Influence records operator influence. type Influence struct { ByteRate float64 + Count float64 } func (infl Influence) add(rhs *Influence, w float64) Influence { infl.ByteRate += rhs.ByteRate * w + infl.Count += rhs.Count * w return infl } @@ -213,6 +215,7 @@ func (load *storeLoad) ByteRateRank() int64 { func (load *storeLoad) ToLoadPred(infl Influence) *storeLoadPred { future := *load future.ByteRate += infl.ByteRate + future.Count += int(math.Round(infl.Count)) return &storeLoadPred{ Current: *load, Future: future, From c28388da23040eb687e2eabcddfc69c4cdffe9ad Mon Sep 17 00:00:00 2001 From: Luffbee Date: Wed, 5 Feb 2020 23:45:20 +0800 Subject: [PATCH 04/29] introduce storeLoad[Pred]Cmp --- server/schedulers/hot_region.go | 58 +++++++++++++++++++-------------- server/schedulers/utils.go | 56 +++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 25 deletions(-) diff --git a/server/schedulers/hot_region.go b/server/schedulers/hot_region.go index 9c34e3c769a..d23393c9ba1 100644 --- a/server/schedulers/hot_region.go +++ b/server/schedulers/hot_region.go @@ -680,21 +680,25 @@ func (bs *balanceSolver) betterThan(old *solution) bool { func (bs *balanceSolver) compareSrcStore(st1, st2 uint64) int { if st1 != st2 { // compare source store - lp1, lp2 := bs.stLoadDetail[st1].LoadPred, bs.stLoadDetail[st2].LoadPred - + var lpCmp storeLPCmp if bs.opTy == transferLeader && bs.rwTy == write { - if r := countCmp(lp1.min(), lp2.min()); r != 0 { - return -r - } - } - if r := byteRateRankCmp(lp1.min(), lp2.min()); r != 0 { - return -r + lpCmp = sliceLPCmp( + minLPCmp(negLoadCmp(sliceLoadCmp( + countCmp, + byteRateRankCmp))), + diffCmp(byteRateRankCmp), + ) + } else { + lpCmp = sliceLPCmp( + minLPCmp(negLoadCmp(byteRateRankCmp)), + diffCmp(byteRateRankCmp), + minLPCmp(negLoadCmp(countCmp)), + ) } - // prefer store with smaller difference - if r := byteRateRankCmp(lp1.diff(), lp2.diff()); r != 0 { - return r - } + lp1 := bs.stLoadDetail[st1].LoadPred + lp2 := bs.stLoadDetail[st2].LoadPred + return lpCmp(lp1, lp2) } return 0 } @@ -702,22 +706,26 @@ func (bs *balanceSolver) compareSrcStore(st1, st2 uint64) int { // smaller is better func (bs *balanceSolver) compareDstStore(st1, st2 uint64) int { if st1 != st2 { - // compare source store - lp1, lp2 := bs.stLoadDetail[st1].LoadPred, bs.stLoadDetail[st2].LoadPred - + // compare destination store + var lpCmp storeLPCmp if bs.opTy == transferLeader && bs.rwTy == write { - if r := countCmp(lp1.max(), lp2.max()); r != 0 { - return r - } - } - if r := byteRateRankCmp(lp1.max(), lp2.max()); r != 0 { - return r + lpCmp = sliceLPCmp( + maxLPCmp(sliceLoadCmp( + countCmp, + byteRateRankCmp)), + diffCmp(byteRateRankCmp), + ) + } else { + lpCmp = sliceLPCmp( + maxLPCmp(byteRateRankCmp), + diffCmp(byteRateRankCmp), + maxLPCmp(countCmp), + ) } - // prefer store with smaller difference - if r := byteRateRankCmp(lp1.diff(), lp2.diff()); r != 0 { - return r - } + lp1 := bs.stLoadDetail[st1].LoadPred + lp2 := bs.stLoadDetail[st2].LoadPred + return lpCmp(lp1, lp2) } return 0 } diff --git a/server/schedulers/utils.go b/server/schedulers/utils.go index 4ef73e1b996..1e678e40b69 100644 --- a/server/schedulers/utils.go +++ b/server/schedulers/utils.go @@ -222,6 +222,25 @@ func (load *storeLoad) ToLoadPred(infl Influence) *storeLoadPred { } } +type storeLoadCmp func(ld1, ld2 *storeLoad) int + +func negLoadCmp(cmp storeLoadCmp) storeLoadCmp { + return func(ld1, ld2 *storeLoad) int { + return -cmp(ld1, ld2) + } +} + +func sliceLoadCmp(cmps ...storeLoadCmp) storeLoadCmp { + return func(ld1, ld2 *storeLoad) int { + for _, cmp := range cmps { + if r := cmp(ld1, ld2); r != 0 { + return r + } + } + return 0 + } +} + func byteRateRankCmp(ld1, ld2 *storeLoad) int { rk1, rk2 := ld1.ByteRateRank(), ld2.ByteRateRank() if rk1 < rk2 { @@ -263,6 +282,43 @@ func (lp *storeLoadPred) diff() *storeLoad { } } +type storeLPCmp func(lp1, lp2 *storeLoadPred) int + +func negLPCmp(cmp storeLPCmp) storeLPCmp { + return func(lp1, lp2 *storeLoadPred) int { + return -cmp(lp1, lp2) + } +} + +func sliceLPCmp(cmps ...storeLPCmp) storeLPCmp { + return func(lp1, lp2 *storeLoadPred) int { + for _, cmp := range cmps { + if r := cmp(lp1, lp2); r != 0 { + return r + } + } + return 0 + } +} + +func minLPCmp(ldCmp storeLoadCmp) storeLPCmp { + return func(lp1, lp2 *storeLoadPred) int { + return ldCmp(lp1.min(), lp2.min()) + } +} + +func maxLPCmp(ldCmp storeLoadCmp) storeLPCmp { + return func(lp1, lp2 *storeLoadPred) int { + return ldCmp(lp1.max(), lp2.max()) + } +} + +func diffCmp(ldCmp storeLoadCmp) storeLPCmp { + return func(lp1, lp2 *storeLoadPred) int { + return ldCmp(lp1.diff(), lp2.diff()) + } +} + func minLoad(a, b *storeLoad) *storeLoad { return &storeLoad{ ByteRate: math.Min(a.ByteRate, b.ByteRate), From e70852b76a59f79c0fac31c3be96182b3cf79abb Mon Sep 17 00:00:00 2001 From: Luffbee Date: Thu, 6 Feb 2020 20:47:05 +0800 Subject: [PATCH 05/29] remove unused function --- server/schedulers/utils.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/server/schedulers/utils.go b/server/schedulers/utils.go index 1e678e40b69..4d9630445c0 100644 --- a/server/schedulers/utils.go +++ b/server/schedulers/utils.go @@ -284,12 +284,6 @@ func (lp *storeLoadPred) diff() *storeLoad { type storeLPCmp func(lp1, lp2 *storeLoadPred) int -func negLPCmp(cmp storeLPCmp) storeLPCmp { - return func(lp1, lp2 *storeLoadPred) int { - return -cmp(lp1, lp2) - } -} - func sliceLPCmp(cmps ...storeLPCmp) storeLPCmp { return func(lp1, lp2 *storeLoadPred) int { for _, cmp := range cmps { From 2a9e0001a4998a6f8c675188d5d9167a4282b4f5 Mon Sep 17 00:00:00 2001 From: Luffbee Date: Fri, 7 Feb 2020 17:01:35 +0800 Subject: [PATCH 06/29] use resourceType --- server/schedulers/hot_region.go | 195 +++++++++++++----------- server/schedulers/hot_test.go | 4 +- server/schedulers/shuffle_hot_region.go | 17 ++- 3 files changed, 114 insertions(+), 102 deletions(-) diff --git a/server/schedulers/hot_region.go b/server/schedulers/hot_region.go index d23393c9ba1..378c9d42d99 100644 --- a/server/schedulers/hot_region.go +++ b/server/schedulers/hot_region.go @@ -68,35 +68,6 @@ const ( minRegionScheduleInterval time.Duration = statistics.StoreHeartBeatReportInterval * time.Second ) -// rwType : the perspective of balance -type rwType int - -const ( - write rwType = iota - read -) - -type opType int - -const ( - movePeer opType = iota - transferLeader -) - -type storeLoadInfos struct { - ReadLeaders map[uint64]*storeLoadDetail - WriteLeaders map[uint64]*storeLoadDetail - WritePeers map[uint64]*storeLoadDetail -} - -func newStoreLoadInfos() *storeLoadInfos { - return &storeLoadInfos{ - ReadLeaders: make(map[uint64]*storeLoadDetail), - WriteLeaders: make(map[uint64]*storeLoadDetail), - WritePeers: make(map[uint64]*storeLoadDetail), - } -} - type hotScheduler struct { name string *BaseScheduler @@ -107,31 +78,30 @@ type hotScheduler struct { r *rand.Rand // states across multiple `Schedule` calls - readPendings map[*pendingInfluence]struct{} - writePendings map[*pendingInfluence]struct{} + pendings [resourceTypeLen]map[*pendingInfluence]struct{} regionPendings map[uint64][2]*operator.Operator // temporary states but exported to API or metrics - stLoadInfos *storeLoadInfos - readPendingSum map[uint64]Influence - writePendingSum map[uint64]Influence + stLoadInfos [resourceTypeLen]map[uint64]*storeLoadDetail + pendingSums [resourceTypeLen]map[uint64]Influence } func newHotScheduler(opController *schedule.OperatorController) *hotScheduler { base := NewBaseScheduler(opController) - return &hotScheduler{ + ret := &hotScheduler{ name: HotRegionName, BaseScheduler: base, leaderLimit: 1, peerLimit: 1, types: []rwType{write, read}, r: rand.New(rand.NewSource(time.Now().UnixNano())), - readPendings: map[*pendingInfluence]struct{}{}, - writePendings: map[*pendingInfluence]struct{}{}, regionPendings: make(map[uint64][2]*operator.Operator), - - stLoadInfos: newStoreLoadInfos(), } + for ty := resourceType(0); ty < resourceTypeLen; ty++ { + ret.pendings[ty] = map[*pendingInfluence]struct{}{} + ret.stLoadInfos[ty] = map[uint64]*storeLoadDetail{} + } + return ret } func newHotReadScheduler(opController *schedule.OperatorController) *hotScheduler { @@ -199,9 +169,9 @@ func (h *hotScheduler) prepareForBalance(cluster opt.Cluster) { regionRead := cluster.RegionReadStats() storeRead := storesStat.GetStoresBytesReadStat() - h.stLoadInfos.ReadLeaders = summaryStoresLoad( + h.stLoadInfos[readLeader] = summaryStoresLoad( storeRead, - h.readPendingSum, + h.pendingSums[readLeader], regionRead, minHotDegree, read, core.LeaderKind) @@ -211,16 +181,16 @@ func (h *hotScheduler) prepareForBalance(cluster opt.Cluster) { regionWrite := cluster.RegionWriteStats() storeWrite := storesStat.GetStoresBytesWriteStat() - h.stLoadInfos.WriteLeaders = summaryStoresLoad( + h.stLoadInfos[writeLeader] = summaryStoresLoad( storeWrite, - map[uint64]Influence{}, + h.pendingSums[writeLeader], regionWrite, minHotDegree, write, core.LeaderKind) - h.stLoadInfos.WritePeers = summaryStoresLoad( + h.stLoadInfos[writePeer] = summaryStoresLoad( storeWrite, - h.writePendingSum, + h.pendingSums[writePeer], regionWrite, minHotDegree, write, core.RegionKind) @@ -228,8 +198,9 @@ func (h *hotScheduler) prepareForBalance(cluster opt.Cluster) { } func (h *hotScheduler) summaryPendingInfluence() { - h.readPendingSum = summaryPendingInfluence(h.readPendings, calcPendingWeight) - h.writePendingSum = summaryPendingInfluence(h.writePendings, calcPendingWeight) + for ty := resourceType(0); ty < resourceTypeLen; ty++ { + h.pendingSums[ty] = summaryPendingInfluence(h.pendings[ty], calcPendingWeight) + } h.gcRegionPendings() } @@ -318,21 +289,19 @@ func filterHotPeers( return ret } -func (h *hotScheduler) addPendingInfluence(op *operator.Operator, srcStore, dstStore uint64, infl Influence, balanceType rwType, ty opType) { +func (h *hotScheduler) addPendingInfluence(op *operator.Operator, srcStore, dstStore uint64, infl Influence, rwTy rwType, opTy opType) { influence := newPendingInfluence(op, srcStore, dstStore, infl) regionID := op.RegionID() - if balanceType == read { - h.readPendings[influence] = struct{}{} - } else { - h.writePendings[influence] = struct{}{} - } + + rcTy := toResourceType(rwTy, opTy) + h.pendings[rcTy][influence] = struct{}{} if _, ok := h.regionPendings[regionID]; !ok { h.regionPendings[regionID] = [2]*operator.Operator{nil, nil} } { // h.pendingOpInfos[regionID][ty] = influence tmp := h.regionPendings[regionID] - tmp[ty] = op + tmp[opTy] = op h.regionPendings[regionID] = tmp } @@ -393,16 +362,13 @@ type solution struct { } func (bs *balanceSolver) init() { - switch bs.rwTy { - case read: - bs.stLoadDetail = bs.sche.stLoadInfos.ReadLeaders - case write: - switch bs.opTy { - case movePeer: - bs.stLoadDetail = bs.sche.stLoadInfos.WritePeers - case transferLeader: - bs.stLoadDetail = bs.sche.stLoadInfos.WriteLeaders - } + switch toResourceType(bs.rwTy, bs.opTy) { + case writePeer: + bs.stLoadDetail = bs.sche.stLoadInfos[writePeer] + case writeLeader: + bs.stLoadDetail = bs.sche.stLoadInfos[writeLeader] + case readLeader: + bs.stLoadDetail = bs.sche.stLoadInfos[readLeader] } for _, id := range getUnhealthyStores(bs.cluster) { delete(bs.stLoadDetail, id) @@ -437,7 +403,7 @@ func (bs *balanceSolver) isValid() bool { return false } switch bs.rwTy { - case read, write: + case write, read: default: return false } @@ -628,7 +594,7 @@ func (bs *balanceSolver) filterDstStores() map[uint64]*storeLoadDetail { if !filter.Target(bs.cluster, store, filters) { detail := bs.stLoadDetail[store.GetID()] dstLd := detail.LoadPred.max() - if bs.opTy == transferLeader && bs.rwTy == write { + if bs.rwTy == write && bs.opTy == transferLeader { if srcLd.Count-1 < dstLd.Count+1 { continue } @@ -681,7 +647,7 @@ func (bs *balanceSolver) compareSrcStore(st1, st2 uint64) int { if st1 != st2 { // compare source store var lpCmp storeLPCmp - if bs.opTy == transferLeader && bs.rwTy == write { + if bs.rwTy == write && bs.opTy == transferLeader { lpCmp = sliceLPCmp( minLPCmp(negLoadCmp(sliceLoadCmp( countCmp, @@ -708,7 +674,7 @@ func (bs *balanceSolver) compareDstStore(st1, st2 uint64) int { if st1 != st2 { // compare destination store var lpCmp storeLPCmp - if bs.opTy == transferLeader && bs.rwTy == write { + if bs.rwTy == write && bs.opTy == transferLeader { lpCmp = sliceLPCmp( maxLPCmp(sliceLoadCmp( countCmp, @@ -756,17 +722,29 @@ func (bs *balanceSolver) buildOperators() ([]*operator.Operator, []Influence) { srcPeer := bs.cur.region.GetStorePeer(bs.cur.srcStoreID) // checked in getRegionAndSrcPeer dstPeer := &metapb.Peer{StoreId: bs.cur.dstStoreID, IsLearner: srcPeer.IsLearner} bs.sche.peerLimit = bs.sche.adjustBalanceLimit(bs.cur.srcStoreID, bs.stLoadDetail) - op, err = operator.CreateMovePeerOperator("move-hot-"+bs.rwTy.String()+"-region", bs.cluster, bs.cur.region, operator.OpHotRegion, bs.cur.srcStoreID, dstPeer) + op, err = operator.CreateMovePeerOperator( + "move-hot-"+bs.rwTy.String()+"-region", + bs.cluster, + bs.cur.region, + operator.OpHotRegion, + bs.cur.srcStoreID, + dstPeer) case transferLeader: if bs.cur.region.GetStoreVoter(bs.cur.dstStoreID) == nil { return nil, nil } bs.sche.leaderLimit = bs.sche.adjustBalanceLimit(bs.cur.srcStoreID, bs.stLoadDetail) - op, err = operator.CreateTransferLeaderOperator("transfer-hot-"+bs.rwTy.String()+"-leader", bs.cluster, bs.cur.region, bs.cur.srcStoreID, bs.cur.dstStoreID, operator.OpHotRegion) + op, err = operator.CreateTransferLeaderOperator( + "transfer-hot-"+bs.rwTy.String()+"-leader", + bs.cluster, + bs.cur.region, + bs.cur.srcStoreID, + bs.cur.dstStoreID, + operator.OpHotRegion) } if err != nil { - log.Debug("fail to create operator", zap.Error(err), zap.Stringer("opType", bs.opTy), zap.Stringer("rwType", bs.rwTy)) + log.Debug("fail to create operator", zap.Error(err), zap.Stringer("rwType", bs.rwTy), zap.Stringer("opType", bs.opTy)) schedulerCounter.WithLabelValues(bs.sche.GetName(), "create-operator-fail").Inc() return nil, nil } @@ -777,9 +755,6 @@ func (bs *balanceSolver) buildOperators() ([]*operator.Operator, []Influence) { schedulerCounter.WithLabelValues(bs.sche.GetName(), bs.opTy.String())) infl := Influence{ByteRate: bs.cur.srcPeerStat.GetBytesRate(), Count: 1} - if bs.opTy == transferLeader && bs.rwTy == write { - infl.ByteRate = 0 - } return []*operator.Operator{op}, []Influence{infl} } @@ -801,8 +776,8 @@ func (h *hotScheduler) adjustBalanceLimit(storeID uint64, loadDetail map[uint64] func (h *hotScheduler) GetHotReadStatus() *statistics.StoreHotPeersInfos { h.RLock() defer h.RUnlock() - asLeader := make(statistics.StoreHotPeersStat, len(h.stLoadInfos.ReadLeaders)) - for id, detail := range h.stLoadInfos.ReadLeaders { + asLeader := make(statistics.StoreHotPeersStat, len(h.stLoadInfos[readLeader])) + for id, detail := range h.stLoadInfos[readLeader] { asLeader[id] = detail.toHotPeersStat() } return &statistics.StoreHotPeersInfos{ @@ -813,12 +788,12 @@ func (h *hotScheduler) GetHotReadStatus() *statistics.StoreHotPeersInfos { func (h *hotScheduler) GetHotWriteStatus() *statistics.StoreHotPeersInfos { h.RLock() defer h.RUnlock() - asLeader := make(statistics.StoreHotPeersStat, len(h.stLoadInfos.WriteLeaders)) - asPeer := make(statistics.StoreHotPeersStat, len(h.stLoadInfos.WritePeers)) - for id, detail := range h.stLoadInfos.WriteLeaders { + asLeader := make(statistics.StoreHotPeersStat, len(h.stLoadInfos[writeLeader])) + asPeer := make(statistics.StoreHotPeersStat, len(h.stLoadInfos[writePeer])) + for id, detail := range h.stLoadInfos[writeLeader] { asLeader[id] = detail.toHotPeersStat() } - for id, detail := range h.stLoadInfos.WritePeers { + for id, detail := range h.stLoadInfos[writePeer] { asPeer[id] = detail.toHotPeersStat() } return &statistics.StoreHotPeersInfos{ @@ -828,22 +803,17 @@ func (h *hotScheduler) GetHotWriteStatus() *statistics.StoreHotPeersInfos { } func (h *hotScheduler) GetWritePendingInfluence() map[uint64]Influence { - return h.copyPendingInfluence(write) + return h.copyPendingInfluence(writePeer) } func (h *hotScheduler) GetReadPendingInfluence() map[uint64]Influence { - return h.copyPendingInfluence(read) + return h.copyPendingInfluence(readLeader) } -func (h *hotScheduler) copyPendingInfluence(typ rwType) map[uint64]Influence { +func (h *hotScheduler) copyPendingInfluence(ty resourceType) map[uint64]Influence { h.RLock() defer h.RUnlock() - var pendingSum map[uint64]Influence - if typ == read { - pendingSum = h.readPendingSum - } else { - pendingSum = h.writePendingSum - } + pendingSum := h.pendingSums[ty] ret := make(map[uint64]Influence, len(pendingSum)) for id, infl := range pendingSum { ret[id] = infl @@ -873,13 +843,21 @@ func calcPendingWeight(op *operator.Operator) float64 { } func (h *hotScheduler) clearPendingInfluence() { - h.readPendings = map[*pendingInfluence]struct{}{} - h.writePendings = map[*pendingInfluence]struct{}{} - h.readPendingSum = nil - h.writePendingSum = nil + for ty := resourceType(0); ty < resourceTypeLen; ty++ { + h.pendings[ty] = map[*pendingInfluence]struct{}{} + h.pendingSums[ty] = nil + } h.regionPendings = make(map[uint64][2]*operator.Operator) } +// rwType : the perspective of balance +type rwType int + +const ( + write rwType = iota + read +) + func (rw rwType) String() string { switch rw { case read: @@ -891,6 +869,13 @@ func (rw rwType) String() string { } } +type opType int + +const ( + movePeer opType = iota + transferLeader +) + func (ty opType) String() string { switch ty { case movePeer: @@ -901,3 +886,27 @@ func (ty opType) String() string { return "" } } + +type resourceType int + +const ( + writePeer resourceType = iota + writeLeader + readLeader + resourceTypeLen +) + +func toResourceType(rwTy rwType, opTy opType) resourceType { + switch rwTy { + case write: + switch opTy { + case movePeer: + return writePeer + case transferLeader: + return writeLeader + } + case read: + return readLeader + } + return resourceTypeLen +} diff --git a/server/schedulers/hot_test.go b/server/schedulers/hot_test.go index 82f74750db2..ae7475f5d27 100644 --- a/server/schedulers/hot_test.go +++ b/server/schedulers/hot_test.go @@ -439,9 +439,9 @@ func (s *testHotReadRegionSchedulerSuite) TestSchedule(c *C) { //| store_id | read_bytes_rate | //|----------|-----------------| - //| 1 | 6MB | + //| 1 | 6MB | //| 2 | 5.5MB | - //| 3 | 5.9MB | + //| 3 | 5.9MB | //| 4 | 3.1MB | //| 5 | 3MB | tc.UpdateStorageReadBytes(1, 6*MB*statistics.StoreHeartBeatReportInterval) diff --git a/server/schedulers/shuffle_hot_region.go b/server/schedulers/shuffle_hot_region.go index 3ac1480abcd..2ff3c9e6ebb 100644 --- a/server/schedulers/shuffle_hot_region.go +++ b/server/schedulers/shuffle_hot_region.go @@ -75,7 +75,7 @@ type shuffleHotRegionSchedulerConfig struct { // the hot peer. type shuffleHotRegionScheduler struct { *BaseScheduler - stLoadInfos *storeLoadInfos + stLoadInfos [resourceTypeLen]map[uint64]*storeLoadDetail r *rand.Rand conf *shuffleHotRegionSchedulerConfig types []rwType @@ -84,13 +84,16 @@ type shuffleHotRegionScheduler struct { // newShuffleHotRegionScheduler creates an admin scheduler that random balance hot regions func newShuffleHotRegionScheduler(opController *schedule.OperatorController, conf *shuffleHotRegionSchedulerConfig) schedule.Scheduler { base := NewBaseScheduler(opController) - return &shuffleHotRegionScheduler{ + ret := &shuffleHotRegionScheduler{ BaseScheduler: base, conf: conf, - stLoadInfos: newStoreLoadInfos(), types: []rwType{read, write}, r: rand.New(rand.NewSource(time.Now().UnixNano())), } + for ty := resourceType(0); ty < resourceTypeLen; ty++ { + ret.stLoadInfos[ty] = map[uint64]*storeLoadDetail{} + } + return ret } func (s *shuffleHotRegionScheduler) GetName() string { @@ -122,21 +125,21 @@ func (s *shuffleHotRegionScheduler) dispatch(typ rwType, cluster opt.Cluster) [] minHotDegree := cluster.GetHotRegionCacheHitsThreshold() switch typ { case read: - s.stLoadInfos.ReadLeaders = summaryStoresLoad( + s.stLoadInfos[readLeader] = summaryStoresLoad( storesStats.GetStoresBytesReadStat(), map[uint64]Influence{}, cluster.RegionReadStats(), minHotDegree, read, core.LeaderKind) - return s.randomSchedule(cluster, s.stLoadInfos.ReadLeaders) + return s.randomSchedule(cluster, s.stLoadInfos[readLeader]) case write: - s.stLoadInfos.WriteLeaders = summaryStoresLoad( + s.stLoadInfos[writeLeader] = summaryStoresLoad( storesStats.GetStoresBytesWriteStat(), map[uint64]Influence{}, cluster.RegionWriteStats(), minHotDegree, write, core.LeaderKind) - return s.randomSchedule(cluster, s.stLoadInfos.WriteLeaders) + return s.randomSchedule(cluster, s.stLoadInfos[writeLeader]) } return nil } From 2085ff6aa2b85051579c526c646aa4f8791ebd0c Mon Sep 17 00:00:00 2001 From: Luffbee Date: Fri, 7 Feb 2020 18:01:25 +0800 Subject: [PATCH 07/29] new byteRateRank; float count --- server/schedulers/hot_region.go | 60 ++++++++++++++++++++++++++------- server/schedulers/utils.go | 49 +++++++++------------------ 2 files changed, 63 insertions(+), 46 deletions(-) diff --git a/server/schedulers/hot_region.go b/server/schedulers/hot_region.go index 378c9d42d99..7c8a9592ab1 100644 --- a/server/schedulers/hot_region.go +++ b/server/schedulers/hot_region.go @@ -15,6 +15,7 @@ package schedulers import ( "fmt" + "math" "math/rand" "sort" "sync" @@ -261,7 +262,7 @@ func summaryStoresLoad( // Build store load prediction from current load and pending influence. stLoadPred := (&storeLoad{ ByteRate: rate, - Count: len(hotPeers), + Count: float64(len(hotPeers)), }).ToLoadPred(pendings[id]) // Construct store load info. @@ -352,6 +353,10 @@ type balanceSolver struct { opTy opType cur *solution + + maxSrcByteRate float64 + minDstByteRate float64 + byteRateRankStep float64 } type solution struct { @@ -373,6 +378,15 @@ func (bs *balanceSolver) init() { for _, id := range getUnhealthyStores(bs.cluster) { delete(bs.stLoadDetail, id) } + bs.maxSrcByteRate = 0 + bs.minDstByteRate = math.MaxFloat64 + maxCurByteRate := 0.0 + for _, detail := range bs.stLoadDetail { + bs.maxSrcByteRate = math.Max(bs.maxSrcByteRate, detail.LoadPred.min().ByteRate) + bs.minDstByteRate = math.Min(bs.minDstByteRate, detail.LoadPred.max().ByteRate) + maxCurByteRate = math.Max(maxCurByteRate, detail.LoadPred.Current.ByteRate) + } + bs.byteRateRankStep = maxCurByteRate / 20 } func getUnhealthyStores(cluster opt.Cluster) []uint64 { @@ -594,14 +608,24 @@ func (bs *balanceSolver) filterDstStores() map[uint64]*storeLoadDetail { if !filter.Target(bs.cluster, store, filters) { detail := bs.stLoadDetail[store.GetID()] dstLd := detail.LoadPred.max() + if bs.rwTy == write && bs.opTy == transferLeader { - if srcLd.Count-1 < dstLd.Count+1 { + if srcLd.Count <= dstLd.Count { continue } - } else { - if srcLd.ByteRate*hotRegionScheduleFactor < dstLd.ByteRate+bs.cur.srcPeerStat.GetBytesRate() { + if !isProgressive( + srcLd.ByteRate, + dstLd.ByteRate, + bs.cur.srcPeerStat.GetBytesRate(), + 1.0) { continue } + } else if !isProgressive( + srcLd.ByteRate, + dstLd.ByteRate, + bs.cur.srcPeerStat.GetBytesRate(), + hotRegionScheduleFactor) { + continue } ret[store.GetID()] = detail } @@ -609,7 +633,11 @@ func (bs *balanceSolver) filterDstStores() map[uint64]*storeLoadDetail { return ret } -// betterSolution returns true if `bs.cur` is a betterSolution solution than `bs.best`. +func isProgressive(srcVal, dstVal, change, factor float64) bool { + return srcVal*factor >= dstVal+change +} + +// betterThan checks if `bs.cur` is a better solution than `old`. func (bs *balanceSolver) betterThan(old *solution) bool { if old == nil { return true @@ -651,13 +679,13 @@ func (bs *balanceSolver) compareSrcStore(st1, st2 uint64) int { lpCmp = sliceLPCmp( minLPCmp(negLoadCmp(sliceLoadCmp( countCmp, - byteRateRankCmp))), - diffCmp(byteRateRankCmp), + byteRateRankCmp(stepRank(bs.maxSrcByteRate, bs.byteRateRankStep))))), + diffCmp(byteRateRankCmp(stepRank(0, bs.byteRateRankStep))), ) } else { lpCmp = sliceLPCmp( - minLPCmp(negLoadCmp(byteRateRankCmp)), - diffCmp(byteRateRankCmp), + minLPCmp(negLoadCmp(byteRateRankCmp(stepRank(bs.maxSrcByteRate, bs.byteRateRankStep)))), + diffCmp(byteRateRankCmp(stepRank(0, bs.byteRateRankStep))), minLPCmp(negLoadCmp(countCmp)), ) } @@ -678,13 +706,13 @@ func (bs *balanceSolver) compareDstStore(st1, st2 uint64) int { lpCmp = sliceLPCmp( maxLPCmp(sliceLoadCmp( countCmp, - byteRateRankCmp)), - diffCmp(byteRateRankCmp), + byteRateRankCmp(stepRank(bs.minDstByteRate, bs.byteRateRankStep)))), + diffCmp(byteRateRankCmp(stepRank(0, bs.byteRateRankStep))), ) } else { lpCmp = sliceLPCmp( - maxLPCmp(byteRateRankCmp), - diffCmp(byteRateRankCmp), + maxLPCmp(byteRateRankCmp(stepRank(bs.minDstByteRate, bs.byteRateRankStep))), + diffCmp(byteRateRankCmp(stepRank(0, bs.byteRateRankStep))), maxLPCmp(countCmp), ) } @@ -696,6 +724,12 @@ func (bs *balanceSolver) compareDstStore(st1, st2 uint64) int { return 0 } +func stepRank(rk0 float64, step float64) func(float64) int64 { + return func(rate float64) int64 { + return int64((rate - rk0) / step) + } +} + func (bs *balanceSolver) isReadyToBuild() bool { if bs.cur.srcStoreID == 0 || bs.cur.dstStoreID == 0 || bs.cur.srcPeerStat == nil || bs.cur.region == nil { diff --git a/server/schedulers/utils.go b/server/schedulers/utils.go index 4d9630445c0..04792991093 100644 --- a/server/schedulers/utils.go +++ b/server/schedulers/utils.go @@ -31,10 +31,9 @@ import ( const ( // adjustRatio is used to adjust TolerantSizeRatio according to region count. - adjustRatio float64 = 0.005 - leaderTolerantSizeRatio float64 = 5.0 - minTolerantSizeRatio float64 = 1.0 - storeLoadByteRateRankSize = 100 * 1024 + adjustRatio float64 = 0.005 + leaderTolerantSizeRatio float64 = 5.0 + minTolerantSizeRatio float64 = 1.0 ) // ErrScheduleConfigNotExist the config is not correct. @@ -205,17 +204,13 @@ func summaryPendingInfluence(pendings map[*pendingInfluence]struct{}, f func(*op type storeLoad struct { ByteRate float64 - Count int -} - -func (load *storeLoad) ByteRateRank() int64 { - return int64(load.ByteRate / storeLoadByteRateRankSize) + Count float64 } func (load *storeLoad) ToLoadPred(infl Influence) *storeLoadPred { future := *load future.ByteRate += infl.ByteRate - future.Count += int(math.Round(infl.Count)) + future.Count += infl.Count return &storeLoadPred{ Current: *load, Future: future, @@ -241,14 +236,16 @@ func sliceLoadCmp(cmps ...storeLoadCmp) storeLoadCmp { } } -func byteRateRankCmp(ld1, ld2 *storeLoad) int { - rk1, rk2 := ld1.ByteRateRank(), ld2.ByteRateRank() - if rk1 < rk2 { - return -1 - } else if rk1 > rk2 { - return 1 +func byteRateRankCmp(rank func(rate float64) int64) storeLoadCmp { + return func(ld1, ld2 *storeLoad) int { + rk1, rk2 := rank(ld1.ByteRate), rank(ld2.ByteRate) + if rk1 < rk2 { + return -1 + } else if rk1 > rk2 { + return 1 + } + return 0 } - return 0 } func countCmp(ld1, ld2 *storeLoad) int { @@ -316,29 +313,15 @@ func diffCmp(ldCmp storeLoadCmp) storeLPCmp { func minLoad(a, b *storeLoad) *storeLoad { return &storeLoad{ ByteRate: math.Min(a.ByteRate, b.ByteRate), - Count: minInt(a.Count, b.Count), + Count: math.Min(a.Count, b.Count), } } func maxLoad(a, b *storeLoad) *storeLoad { return &storeLoad{ ByteRate: math.Max(a.ByteRate, b.ByteRate), - Count: maxInt(a.Count, b.Count), - } -} - -func minInt(a, b int) int { - if a < b { - return a - } - return b -} - -func maxInt(a, b int) int { - if a < b { - return b + Count: math.Max(a.Count, b.Count), } - return a } type storeLoadDetail struct { From 291be85240c4ba0a3af67685775ba822bd5edfc9 Mon Sep 17 00:00:00 2001 From: Luffbee Date: Mon, 10 Feb 2020 21:55:19 +0800 Subject: [PATCH 08/29] add key rate for storeLoad, Influence --- server/schedulers/utils.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/server/schedulers/utils.go b/server/schedulers/utils.go index 04792991093..a8fd3ba2bde 100644 --- a/server/schedulers/utils.go +++ b/server/schedulers/utils.go @@ -164,11 +164,13 @@ func getKeyRanges(args []string) ([]core.KeyRange, error) { // Influence records operator influence. type Influence struct { ByteRate float64 + KeyRate float64 Count float64 } func (infl Influence) add(rhs *Influence, w float64) Influence { infl.ByteRate += rhs.ByteRate * w + infl.KeyRate += rhs.KeyRate * w infl.Count += rhs.Count * w return infl } @@ -204,12 +206,14 @@ func summaryPendingInfluence(pendings map[*pendingInfluence]struct{}, f func(*op type storeLoad struct { ByteRate float64 + KeyRate float64 Count float64 } func (load *storeLoad) ToLoadPred(infl Influence) *storeLoadPred { future := *load future.ByteRate += infl.ByteRate + future.KeyRate += infl.KeyRate future.Count += infl.Count return &storeLoadPred{ Current: *load, @@ -275,6 +279,7 @@ func (lp *storeLoadPred) diff() *storeLoad { mx, mn := lp.max(), lp.min() return &storeLoad{ ByteRate: mx.ByteRate - mn.ByteRate, + KeyRate: mx.KeyRate - mn.KeyRate, Count: mx.Count - mn.Count, } } @@ -313,6 +318,7 @@ func diffCmp(ldCmp storeLoadCmp) storeLPCmp { func minLoad(a, b *storeLoad) *storeLoad { return &storeLoad{ ByteRate: math.Min(a.ByteRate, b.ByteRate), + KeyRate: math.Min(a.KeyRate, b.KeyRate), Count: math.Min(a.Count, b.Count), } } @@ -320,6 +326,7 @@ func minLoad(a, b *storeLoad) *storeLoad { func maxLoad(a, b *storeLoad) *storeLoad { return &storeLoad{ ByteRate: math.Max(a.ByteRate, b.ByteRate), + KeyRate: math.Max(a.KeyRate, b.KeyRate), Count: math.Max(a.Count, b.Count), } } From 1b91e045dc8c9c8e91f47442e00f9907e766b32e Mon Sep 17 00:00:00 2001 From: Luffbee Date: Mon, 10 Feb 2020 21:56:34 +0800 Subject: [PATCH 09/29] make getHotPeers key-rate-aware --- server/schedulers/hot_region.go | 42 +++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/server/schedulers/hot_region.go b/server/schedulers/hot_region.go index 74de4e33550..2388522a6d8 100644 --- a/server/schedulers/hot_region.go +++ b/server/schedulers/hot_region.go @@ -64,6 +64,8 @@ const ( hotRegionLimitFactor = 0.75 hotRegionScheduleFactor = 0.95 + maxPeerNum = 1000 + maxZombieDur time.Duration = statistics.StoreHeartBeatReportInterval * time.Second minRegionScheduleInterval time.Duration = statistics.StoreHeartBeatReportInterval * time.Second @@ -497,11 +499,43 @@ func (bs *balanceSolver) filterSrcStores() map[uint64]*storeLoadDetail { func (bs *balanceSolver) getHotPeers() []*statistics.HotPeerStat { ret := bs.stLoadDetail[bs.cur.srcStoreID].HotPeers - sort.Slice(ret, func(i, j int) bool { - return ret[i].GetByteRate() > ret[j].GetByteRate() + if len(ret) <= maxPeerNum { + return ret + } + + byteSort := make([]*statistics.HotPeerStat, len(ret)) + copy(byteSort, ret) + sort.Slice(byteSort, func(i, j int) bool { + return byteSort[i].GetByteRate() > byteSort[j].GetByteRate() }) - if len(ret) > 1000 { - ret = ret[:1000] + keySort := make([]*statistics.HotPeerStat, len(ret)) + copy(keySort, ret) + sort.Slice(keySort, func(i, j int) bool { + return keySort[i].GetKeyRate() > keySort[j].GetKeyRate() + }) + + union := make(map[*statistics.HotPeerStat]struct{}, maxPeerNum) + for len(union) < maxPeerNum { + for len(byteSort) > 0 { + peer := byteSort[0] + byteSort = byteSort[1:] + if _, ok := union[peer]; !ok { + union[peer] = struct{}{} + break + } + } + for len(keySort) > 0 { + peer := keySort[0] + keySort = keySort[1:] + if _, ok := union[peer]; !ok { + union[peer] = struct{}{} + break + } + } + } + ret = make([]*statistics.HotPeerStat, 0, len(union)) + for peer := range union { + ret = append(ret, peer) } return ret } From 228ba6c4eb2e38b59238394e1b6538c939b9c680 Mon Sep 17 00:00:00 2001 From: Luffbee Date: Mon, 10 Feb 2020 22:20:37 +0800 Subject: [PATCH 10/29] make filterDstStores key-rate-aware --- server/schedulers/hot_region.go | 51 +++++++++++++++------------------ 1 file changed, 23 insertions(+), 28 deletions(-) diff --git a/server/schedulers/hot_region.go b/server/schedulers/hot_region.go index 2388522a6d8..f8f2fd1e9f2 100644 --- a/server/schedulers/hot_region.go +++ b/server/schedulers/hot_region.go @@ -61,8 +61,7 @@ const ( // HotWriteRegionType is hot write region scheduler type. HotWriteRegionType = "hot-write-region" - hotRegionLimitFactor = 0.75 - hotRegionScheduleFactor = 0.95 + hotRegionLimitFactor = 0.75 maxPeerNum = 1000 @@ -453,7 +452,7 @@ func (bs *balanceSolver) solve() []*operator.Operator { for dstStoreID := range bs.filterDstStores() { bs.cur.dstStoreID = dstStoreID - if bs.betterThan(best) { + if bs.isProgressive() && bs.betterThan(best) { if newOps, newInfls := bs.buildOperators(); len(newOps) > 0 { ops = newOps infls = newInfls @@ -636,39 +635,35 @@ func (bs *balanceSolver) filterDstStores() map[uint64]*storeLoadDetail { return nil } - srcLd := bs.stLoadDetail[bs.cur.srcStoreID].LoadPred.min() ret := make(map[uint64]*storeLoadDetail, len(candidates)) for _, store := range candidates { if !filter.Target(bs.cluster, store, filters) { - detail := bs.stLoadDetail[store.GetID()] - dstLd := detail.LoadPred.max() - - if bs.rwTy == write && bs.opTy == transferLeader { - if srcLd.Count <= dstLd.Count { - continue - } - if !isProgressive( - srcLd.ByteRate, - dstLd.ByteRate, - bs.cur.srcPeerStat.GetByteRate(), - 1.0) { - continue - } - } else if !isProgressive( - srcLd.ByteRate, - dstLd.ByteRate, - bs.cur.srcPeerStat.GetByteRate(), - hotRegionScheduleFactor) { - continue - } - ret[store.GetID()] = detail + ret[store.GetID()] = bs.stLoadDetail[store.GetID()] } } return ret } -func isProgressive(srcVal, dstVal, change, factor float64) bool { - return srcVal*factor >= dstVal+change +// isProgressive checks `bs.cur` is a progressive solution. +func (bs *balanceSolver) isProgressive() bool { + srcLd := bs.stLoadDetail[bs.cur.srcStoreID].LoadPred.min() + dstLd := bs.stLoadDetail[bs.cur.dstStoreID].LoadPred.max() + peer := bs.cur.srcPeerStat + if bs.rwTy == write && bs.opTy == transferLeader { + if srcLd.Count > dstLd.Count && + srcLd.ByteRate >= dstLd.ByteRate+peer.GetByteRate() && + srcLd.KeyRate >= dstLd.KeyRate+peer.GetKeyRate() { + return true + } + } else { + if srcLd.ByteRate*0.95 >= dstLd.ByteRate+peer.GetByteRate() { + return true + } else if srcLd.ByteRate >= dstLd.ByteRate+peer.GetByteRate() && + srcLd.KeyRate*0.95 >= dstLd.KeyRate+peer.GetKeyRate() { + return true + } + } + return false } // betterThan checks if `bs.cur` is a better solution than `old`. From e15c95ca2a1fbb9c2b3fa372b8e5598c07ad6b2f Mon Sep 17 00:00:00 2001 From: Luffbee Date: Tue, 11 Feb 2020 18:41:06 +0800 Subject: [PATCH 11/29] make betterThan key-rate-aware --- server/schedulers/hot_region.go | 96 ++++++++++++++++++++++----------- server/schedulers/utils.go | 25 +++++---- 2 files changed, 78 insertions(+), 43 deletions(-) diff --git a/server/schedulers/hot_region.go b/server/schedulers/hot_region.go index f8f2fd1e9f2..68502b86de3 100644 --- a/server/schedulers/hot_region.go +++ b/server/schedulers/hot_region.go @@ -355,9 +355,9 @@ type balanceSolver struct { cur *solution - maxSrcByteRate float64 - minDstByteRate float64 - byteRateRankStep float64 + maxSrc *storeLoad + minDst *storeLoad + rankStep *storeLoad } type solution struct { @@ -379,15 +379,26 @@ func (bs *balanceSolver) init() { for _, id := range getUnhealthyStores(bs.cluster) { delete(bs.stLoadDetail, id) } - bs.maxSrcByteRate = 0 - bs.minDstByteRate = math.MaxFloat64 - maxCurByteRate := 0.0 + + bs.maxSrc = &storeLoad{} + bs.minDst = &storeLoad{ + ByteRate: math.MaxFloat64, + KeyRate: math.MaxFloat64, + Count: math.MaxFloat64, + } + maxCur := &storeLoad{} + for _, detail := range bs.stLoadDetail { - bs.maxSrcByteRate = math.Max(bs.maxSrcByteRate, detail.LoadPred.min().ByteRate) - bs.minDstByteRate = math.Min(bs.minDstByteRate, detail.LoadPred.max().ByteRate) - maxCurByteRate = math.Max(maxCurByteRate, detail.LoadPred.Current.ByteRate) + bs.maxSrc = maxLoad(bs.maxSrc, detail.LoadPred.min()) + bs.minDst = minLoad(bs.minDst, detail.LoadPred.max()) + maxCur = maxLoad(maxCur, &detail.LoadPred.Current) + } + + bs.rankStep = &storeLoad{ + ByteRate: maxCur.ByteRate / 10, + KeyRate: maxCur.KeyRate / 10, + Count: maxCur.Count / 10, } - bs.byteRateRankStep = maxCurByteRate / 20 } func getUnhealthyStores(cluster opt.Cluster) []uint64 { @@ -688,11 +699,23 @@ func (bs *balanceSolver) betterThan(old *solution) bool { // compare region // prefer region with larger byte rate, to converge faster - curByteRk, oldByteRk := int64(bs.cur.srcPeerStat.GetByteRate()/100), int64(old.srcPeerStat.GetByteRate()/100) - if curByteRk > oldByteRk { - return true - } else if curByteRk < oldByteRk { - return false + curKeyRk, oldKeyRk := int64(bs.cur.srcPeerStat.GetKeyRate()/10), int64(old.srcPeerStat.GetKeyRate()/10) + if curKeyRk == oldKeyRk { + curByteRk, oldByteRk := int64(bs.cur.srcPeerStat.GetByteRate()/100), int64(old.srcPeerStat.GetByteRate()/100) + if curByteRk > oldByteRk { + return true + } else if curByteRk < oldByteRk { + return false + } + } else { + srcKeyRate := bs.stLoadDetail[bs.cur.srcStoreID].LoadPred.min().KeyRate + dstKeyRate := bs.stLoadDetail[bs.cur.dstStoreID].LoadPred.max().KeyRate + srcCmpDst := int64((srcKeyRate - dstKeyRate) / (bs.rankStep.KeyRate)) + if srcCmpDst > 0 { + return curKeyRk > oldKeyRk + } else { + return curKeyRk < oldKeyRk + } } } @@ -707,16 +730,21 @@ func (bs *balanceSolver) compareSrcStore(st1, st2 uint64) int { if bs.rwTy == write && bs.opTy == transferLeader { lpCmp = sliceLPCmp( minLPCmp(negLoadCmp(sliceLoadCmp( - countCmp, - byteRateRankCmp(stepRank(bs.maxSrcByteRate, bs.byteRateRankStep))))), - diffCmp(byteRateRankCmp(stepRank(0, bs.byteRateRankStep))), + rankCmp(stLdCount, stepRank(bs.maxSrc.Count, bs.rankStep.Count)), + rankCmp(stLdKeyRate, stepRank(bs.maxSrc.KeyRate, bs.rankStep.KeyRate)), + rankCmp(stLdByteRate, stepRank(bs.maxSrc.ByteRate, bs.rankStep.ByteRate)), + ))), + diffCmp(sliceLoadCmp( + rankCmp(stLdCount, stepRank(0, bs.rankStep.Count)), + rankCmp(stLdKeyRate, stepRank(0, bs.rankStep.KeyRate)), + rankCmp(stLdByteRate, stepRank(0, bs.rankStep.ByteRate)), + )), ) } else { - lpCmp = sliceLPCmp( - minLPCmp(negLoadCmp(byteRateRankCmp(stepRank(bs.maxSrcByteRate, bs.byteRateRankStep)))), - diffCmp(byteRateRankCmp(stepRank(0, bs.byteRateRankStep))), - minLPCmp(negLoadCmp(countCmp)), - ) + lpCmp = minLPCmp(negLoadCmp(sliceLoadCmp( + rankCmp(stLdByteRate, stepRank(bs.maxSrc.ByteRate, bs.rankStep.ByteRate)), + rankCmp(stLdKeyRate, stepRank(bs.maxSrc.KeyRate, bs.rankStep.KeyRate)), + ))) } lp1 := bs.stLoadDetail[st1].LoadPred @@ -734,16 +762,20 @@ func (bs *balanceSolver) compareDstStore(st1, st2 uint64) int { if bs.rwTy == write && bs.opTy == transferLeader { lpCmp = sliceLPCmp( maxLPCmp(sliceLoadCmp( - countCmp, - byteRateRankCmp(stepRank(bs.minDstByteRate, bs.byteRateRankStep)))), - diffCmp(byteRateRankCmp(stepRank(0, bs.byteRateRankStep))), - ) + rankCmp(stLdCount, stepRank(bs.minDst.Count, bs.rankStep.Count)), + rankCmp(stLdKeyRate, stepRank(bs.minDst.KeyRate, bs.rankStep.KeyRate)), + rankCmp(stLdByteRate, stepRank(bs.minDst.ByteRate, bs.rankStep.ByteRate)), + )), + diffCmp(sliceLoadCmp( + rankCmp(stLdCount, stepRank(0, bs.rankStep.Count)), + rankCmp(stLdKeyRate, stepRank(0, bs.rankStep.KeyRate)), + rankCmp(stLdByteRate, stepRank(0, bs.rankStep.ByteRate)), + ))) } else { - lpCmp = sliceLPCmp( - maxLPCmp(byteRateRankCmp(stepRank(bs.minDstByteRate, bs.byteRateRankStep))), - diffCmp(byteRateRankCmp(stepRank(0, bs.byteRateRankStep))), - maxLPCmp(countCmp), - ) + lpCmp = maxLPCmp(sliceLoadCmp( + rankCmp(stLdByteRate, stepRank(bs.minDst.ByteRate, bs.rankStep.ByteRate)), + rankCmp(stLdKeyRate, stepRank(bs.minDst.KeyRate, bs.rankStep.KeyRate)), + )) } lp1 := bs.stLoadDetail[st1].LoadPred diff --git a/server/schedulers/utils.go b/server/schedulers/utils.go index a8fd3ba2bde..74aecec0357 100644 --- a/server/schedulers/utils.go +++ b/server/schedulers/utils.go @@ -221,6 +221,18 @@ func (load *storeLoad) ToLoadPred(infl Influence) *storeLoadPred { } } +func stLdByteRate(ld *storeLoad) float64 { + return ld.ByteRate +} + +func stLdKeyRate(ld *storeLoad) float64 { + return ld.KeyRate +} + +func stLdCount(ld *storeLoad) float64 { + return ld.Count +} + type storeLoadCmp func(ld1, ld2 *storeLoad) int func negLoadCmp(cmp storeLoadCmp) storeLoadCmp { @@ -240,9 +252,9 @@ func sliceLoadCmp(cmps ...storeLoadCmp) storeLoadCmp { } } -func byteRateRankCmp(rank func(rate float64) int64) storeLoadCmp { +func rankCmp(dim func(ld *storeLoad) float64, rank func(value float64) int64) storeLoadCmp { return func(ld1, ld2 *storeLoad) int { - rk1, rk2 := rank(ld1.ByteRate), rank(ld2.ByteRate) + rk1, rk2 := rank(dim(ld1)), rank(dim(ld2)) if rk1 < rk2 { return -1 } else if rk1 > rk2 { @@ -252,15 +264,6 @@ func byteRateRankCmp(rank func(rate float64) int64) storeLoadCmp { } } -func countCmp(ld1, ld2 *storeLoad) int { - if ld1.Count < ld2.Count { - return -1 - } else if ld1.Count > ld2.Count { - return 1 - } - return 0 -} - // store load prediction type storeLoadPred struct { Current storeLoad From bb52fe3c5573b07f3c7c12c8157f97d37c036907 Mon Sep 17 00:00:00 2001 From: Luffbee Date: Tue, 11 Feb 2020 19:14:52 +0800 Subject: [PATCH 12/29] make prepare and influence key-rate-aware --- server/schedulers/hot_region.go | 47 ++++++++++++++++++------- server/schedulers/shuffle_hot_region.go | 2 ++ 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/server/schedulers/hot_region.go b/server/schedulers/hot_region.go index 68502b86de3..83eed551abf 100644 --- a/server/schedulers/hot_region.go +++ b/server/schedulers/hot_region.go @@ -169,10 +169,12 @@ func (h *hotScheduler) prepareForBalance(cluster opt.Cluster) { minHotDegree := cluster.GetHotRegionCacheHitsThreshold() { // update read statistics regionRead := cluster.RegionReadStats() - storeRead := storesStat.GetStoresBytesReadStat() + storeByte := storesStat.GetStoresBytesReadStat() + storeKey := storesStat.GetStoresKeysReadStat() h.stLoadInfos[readLeader] = summaryStoresLoad( - storeRead, + storeByte, + storeKey, h.pendingSums[readLeader], regionRead, minHotDegree, @@ -181,17 +183,20 @@ func (h *hotScheduler) prepareForBalance(cluster opt.Cluster) { { // update write statistics regionWrite := cluster.RegionWriteStats() - storeWrite := storesStat.GetStoresBytesWriteStat() + storeByte := storesStat.GetStoresBytesWriteStat() + storeKey := storesStat.GetStoresKeysWriteStat() h.stLoadInfos[writeLeader] = summaryStoresLoad( - storeWrite, + storeByte, + storeKey, h.pendingSums[writeLeader], regionWrite, minHotDegree, write, core.LeaderKind) h.stLoadInfos[writePeer] = summaryStoresLoad( - storeWrite, + storeByte, + storeKey, h.pendingSums[writePeer], regionWrite, minHotDegree, @@ -231,6 +236,7 @@ func (h *hotScheduler) gcRegionPendings() { // Load information of all available stores. func summaryStoresLoad( storeByteRate map[uint64]float64, + storeKeyRate map[uint64]float64, pendings map[uint64]Influence, storeHotPeers map[uint64][]*statistics.HotPeerStat, minHotDegree int, @@ -240,29 +246,40 @@ func summaryStoresLoad( loadDetail := make(map[uint64]*storeLoadDetail, len(storeByteRate)) // Stores without byte rate statistics is not available to schedule. - for id, rate := range storeByteRate { + for id, byteRate := range storeByteRate { + keyRate := storeKeyRate[id] // Find all hot peers first hotPeers := make([]*statistics.HotPeerStat, 0) { - hotSum := 0.0 + byteSum := 0.0 + keySum := 0.0 for _, peer := range filterHotPeers(kind, minHotDegree, storeHotPeers[id]) { - hotSum += peer.GetByteRate() + byteSum += peer.GetByteRate() + keySum += peer.GetKeyRate() hotPeers = append(hotPeers, peer.Clone()) } // Use sum of hot peers to estimate leader-only byte rate. if kind == core.LeaderKind && rwTy == write { - rate = hotSum + byteRate = byteSum + keyRate = keySum } // Metric for debug. - ty := "byte-rate-" + rwTy.String() + "-" + kind.String() - hotPeerSummary.WithLabelValues(ty, fmt.Sprintf("%v", id)).Set(hotSum) + { + ty := "byte-rate-" + rwTy.String() + "-" + kind.String() + hotPeerSummary.WithLabelValues(ty, fmt.Sprintf("%v", id)).Set(byteSum) + } + { + ty := "key-rate-" + rwTy.String() + "-" + kind.String() + hotPeerSummary.WithLabelValues(ty, fmt.Sprintf("%v", id)).Set(keySum) + } } // Build store load prediction from current load and pending influence. stLoadPred := (&storeLoad{ - ByteRate: rate, + ByteRate: byteRate, + KeyRate: keyRate, Count: float64(len(hotPeers)), }).ToLoadPred(pendings[id]) @@ -849,7 +866,11 @@ func (bs *balanceSolver) buildOperators() ([]*operator.Operator, []Influence) { schedulerCounter.WithLabelValues(bs.sche.GetName(), "new-operator"), schedulerCounter.WithLabelValues(bs.sche.GetName(), bs.opTy.String())) - infl := Influence{ByteRate: bs.cur.srcPeerStat.GetByteRate(), Count: 1} + infl := Influence{ + ByteRate: bs.cur.srcPeerStat.GetByteRate(), + KeyRate: bs.cur.srcPeerStat.GetKeyRate(), + Count: 1, + } return []*operator.Operator{op}, []Influence{infl} } diff --git a/server/schedulers/shuffle_hot_region.go b/server/schedulers/shuffle_hot_region.go index 2ff3c9e6ebb..39b5ece1a5b 100644 --- a/server/schedulers/shuffle_hot_region.go +++ b/server/schedulers/shuffle_hot_region.go @@ -127,6 +127,7 @@ func (s *shuffleHotRegionScheduler) dispatch(typ rwType, cluster opt.Cluster) [] case read: s.stLoadInfos[readLeader] = summaryStoresLoad( storesStats.GetStoresBytesReadStat(), + storesStats.GetStoresKeysReadStat(), map[uint64]Influence{}, cluster.RegionReadStats(), minHotDegree, @@ -135,6 +136,7 @@ func (s *shuffleHotRegionScheduler) dispatch(typ rwType, cluster opt.Cluster) [] case write: s.stLoadInfos[writeLeader] = summaryStoresLoad( storesStats.GetStoresBytesWriteStat(), + storesStats.GetStoresKeysWriteStat(), map[uint64]Influence{}, cluster.RegionWriteStats(), minHotDegree, From ba561d50306d5c1aca8964e95ea4860fd0ff30f9 Mon Sep 17 00:00:00 2001 From: Luffbee Date: Tue, 11 Feb 2020 21:09:53 +0800 Subject: [PATCH 13/29] add metric; tune parameters --- server/cluster/coordinator.go | 4 ++++ server/schedulers/hot_region.go | 32 +++++++++++++++++++++----------- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/server/cluster/coordinator.go b/server/cluster/coordinator.go index f9bd418eb37..7a20ae182ca 100644 --- a/server/cluster/coordinator.go +++ b/server/cluster/coordinator.go @@ -420,6 +420,8 @@ func (c *coordinator) collectHotSpotMetrics() { infl := pendings[storeID] // TODO: add to tidb-ansible after merging pending influence into operator influence. hotSpotStatusGauge.WithLabelValues(storeAddress, storeLabel, "write_pending_influence_byte_rate").Set(infl.ByteRate) + hotSpotStatusGauge.WithLabelValues(storeAddress, storeLabel, "write_pending_influence_key_rate").Set(infl.KeyRate) + hotSpotStatusGauge.WithLabelValues(storeAddress, storeLabel, "write_pending_influence_count").Set(infl.Count) } // Collects hot read region metrics. @@ -440,6 +442,8 @@ func (c *coordinator) collectHotSpotMetrics() { infl := pendings[storeID] hotSpotStatusGauge.WithLabelValues(storeAddress, storeLabel, "read_pending_influence_byte_rate").Set(infl.ByteRate) + hotSpotStatusGauge.WithLabelValues(storeAddress, storeLabel, "read_pending_influence_key_rate").Set(infl.KeyRate) + hotSpotStatusGauge.WithLabelValues(storeAddress, storeLabel, "read_pending_influence_count").Set(infl.Count) } } diff --git a/server/schedulers/hot_region.go b/server/schedulers/hot_region.go index 83eed551abf..ed2d05f34d0 100644 --- a/server/schedulers/hot_region.go +++ b/server/schedulers/hot_region.go @@ -412,8 +412,8 @@ func (bs *balanceSolver) init() { } bs.rankStep = &storeLoad{ - ByteRate: maxCur.ByteRate / 10, - KeyRate: maxCur.KeyRate / 10, + ByteRate: maxCur.ByteRate / 20, + KeyRate: maxCur.KeyRate / 20, Count: maxCur.Count / 10, } } @@ -686,7 +686,7 @@ func (bs *balanceSolver) isProgressive() bool { } else { if srcLd.ByteRate*0.95 >= dstLd.ByteRate+peer.GetByteRate() { return true - } else if srcLd.ByteRate >= dstLd.ByteRate+peer.GetByteRate() && + } else if srcLd.ByteRate*0.99 >= dstLd.ByteRate+peer.GetByteRate() && srcLd.KeyRate*0.95 >= dstLd.KeyRate+peer.GetKeyRate() { return true } @@ -758,10 +758,15 @@ func (bs *balanceSolver) compareSrcStore(st1, st2 uint64) int { )), ) } else { - lpCmp = minLPCmp(negLoadCmp(sliceLoadCmp( - rankCmp(stLdByteRate, stepRank(bs.maxSrc.ByteRate, bs.rankStep.ByteRate)), - rankCmp(stLdKeyRate, stepRank(bs.maxSrc.KeyRate, bs.rankStep.KeyRate)), - ))) + lpCmp = sliceLPCmp( + minLPCmp(negLoadCmp(sliceLoadCmp( + rankCmp(stLdByteRate, stepRank(bs.maxSrc.ByteRate, bs.rankStep.ByteRate)), + rankCmp(stLdKeyRate, stepRank(bs.maxSrc.KeyRate, bs.rankStep.KeyRate)), + ))), + diffCmp( + rankCmp(stLdByteRate, stepRank(0, bs.rankStep.ByteRate)), + ), + ) } lp1 := bs.stLoadDetail[st1].LoadPred @@ -789,10 +794,15 @@ func (bs *balanceSolver) compareDstStore(st1, st2 uint64) int { rankCmp(stLdByteRate, stepRank(0, bs.rankStep.ByteRate)), ))) } else { - lpCmp = maxLPCmp(sliceLoadCmp( - rankCmp(stLdByteRate, stepRank(bs.minDst.ByteRate, bs.rankStep.ByteRate)), - rankCmp(stLdKeyRate, stepRank(bs.minDst.KeyRate, bs.rankStep.KeyRate)), - )) + lpCmp = sliceLPCmp( + maxLPCmp(sliceLoadCmp( + rankCmp(stLdByteRate, stepRank(bs.minDst.ByteRate, bs.rankStep.ByteRate)), + rankCmp(stLdKeyRate, stepRank(bs.minDst.KeyRate, bs.rankStep.KeyRate)), + )), + diffCmp( + rankCmp(stLdByteRate, stepRank(0, bs.rankStep.ByteRate)), + ), + ) } lp1 := bs.stLoadDetail[st1].LoadPred From cb8f5629c6ea69b1119d1ed81ad60c21f975edac Mon Sep 17 00:00:00 2001 From: Luffbee Date: Wed, 12 Feb 2020 18:10:52 +0800 Subject: [PATCH 14/29] write leader: remove byte rate constraint --- server/schedulers/hot_region.go | 1 - 1 file changed, 1 deletion(-) diff --git a/server/schedulers/hot_region.go b/server/schedulers/hot_region.go index ed2d05f34d0..dab79bab846 100644 --- a/server/schedulers/hot_region.go +++ b/server/schedulers/hot_region.go @@ -679,7 +679,6 @@ func (bs *balanceSolver) isProgressive() bool { peer := bs.cur.srcPeerStat if bs.rwTy == write && bs.opTy == transferLeader { if srcLd.Count > dstLd.Count && - srcLd.ByteRate >= dstLd.ByteRate+peer.GetByteRate() && srcLd.KeyRate >= dstLd.KeyRate+peer.GetKeyRate() { return true } From 37ca8ac09e1b1f9823aefcd79bfa48ad92b2e5ea Mon Sep 17 00:00:00 2001 From: Luffbee Date: Sat, 15 Feb 2020 15:25:27 +0800 Subject: [PATCH 15/29] progressiveRank --- server/schedulers/hot_region.go | 112 ++++++++++++++++++++------------ server/schedulers/utils.go | 20 +++--- 2 files changed, 83 insertions(+), 49 deletions(-) diff --git a/server/schedulers/hot_region.go b/server/schedulers/hot_region.go index dab79bab846..26040d377af 100644 --- a/server/schedulers/hot_region.go +++ b/server/schedulers/hot_region.go @@ -382,6 +382,8 @@ type solution struct { srcPeerStat *statistics.HotPeerStat region *core.RegionInfo dstStoreID uint64 + + progressiveRank int64 } func (bs *balanceSolver) init() { @@ -479,8 +481,9 @@ func (bs *balanceSolver) solve() []*operator.Operator { for dstStoreID := range bs.filterDstStores() { bs.cur.dstStoreID = dstStoreID + bs.cur.progressiveRank = bs.calcProgressiveRank() - if bs.isProgressive() && bs.betterThan(best) { + if bs.cur.progressiveRank < 0 && bs.betterThan(best) { if newOps, newInfls := bs.buildOperators(); len(newOps) > 0 { ops = newOps infls = newInfls @@ -672,25 +675,32 @@ func (bs *balanceSolver) filterDstStores() map[uint64]*storeLoadDetail { return ret } -// isProgressive checks `bs.cur` is a progressive solution. -func (bs *balanceSolver) isProgressive() bool { +// calcProgressiveRank checks `bs.cur` is a progressive solution. +func (bs *balanceSolver) calcProgressiveRank() int64 { srcLd := bs.stLoadDetail[bs.cur.srcStoreID].LoadPred.min() dstLd := bs.stLoadDetail[bs.cur.dstStoreID].LoadPred.max() peer := bs.cur.srcPeerStat + rank := int64(0) if bs.rwTy == write && bs.opTy == transferLeader { if srcLd.Count > dstLd.Count && srcLd.KeyRate >= dstLd.KeyRate+peer.GetKeyRate() { - return true + rank -= 1 } } else { - if srcLd.ByteRate*0.95 >= dstLd.ByteRate+peer.GetByteRate() { - return true - } else if srcLd.ByteRate*0.99 >= dstLd.ByteRate+peer.GetByteRate() && - srcLd.KeyRate*0.95 >= dstLd.KeyRate+peer.GetKeyRate() { - return true + keyDecRatio := (dstLd.KeyRate + peer.GetKeyRate()) / (srcLd.KeyRate + 1) + keyHot := peer.GetKeyRate() >= 10 + byteDecRatio := (dstLd.ByteRate + peer.GetByteRate()) / (srcLd.ByteRate + 1) + byteHot := peer.GetByteRate() > 100 + switch { + case byteHot && byteDecRatio <= 0.95 && keyHot && keyDecRatio <= 0.95: + rank = -3 + case byteDecRatio <= 0.99 && keyHot && keyDecRatio <= 0.95: + rank = -2 + case byteHot && byteDecRatio <= 0.95: + rank = -1 } } - return false + return rank } // betterThan checks if `bs.cur` is a better solution than `old`. @@ -699,6 +709,13 @@ func (bs *balanceSolver) betterThan(old *solution) bool { return true } + switch { + case bs.cur.progressiveRank < old.progressiveRank: + return true + case bs.cur.progressiveRank > old.progressiveRank: + return false + } + if r := bs.compareSrcStore(bs.cur.srcStoreID, old.srcStoreID); r < 0 { return true } else if r > 0 { @@ -714,23 +731,36 @@ func (bs *balanceSolver) betterThan(old *solution) bool { if bs.cur.srcPeerStat != old.srcPeerStat { // compare region - // prefer region with larger byte rate, to converge faster - curKeyRk, oldKeyRk := int64(bs.cur.srcPeerStat.GetKeyRate()/10), int64(old.srcPeerStat.GetKeyRate()/10) - if curKeyRk == oldKeyRk { - curByteRk, oldByteRk := int64(bs.cur.srcPeerStat.GetByteRate()/100), int64(old.srcPeerStat.GetByteRate()/100) - if curByteRk > oldByteRk { + if bs.rwTy == write && bs.opTy == transferLeader { + switch { + case bs.cur.srcPeerStat.GetKeyRate() > old.srcPeerStat.GetKeyRate(): return true - } else if curByteRk < oldByteRk { + case bs.cur.srcPeerStat.GetKeyRate() < old.srcPeerStat.GetKeyRate(): return false } } else { - srcKeyRate := bs.stLoadDetail[bs.cur.srcStoreID].LoadPred.min().KeyRate - dstKeyRate := bs.stLoadDetail[bs.cur.dstStoreID].LoadPred.max().KeyRate - srcCmpDst := int64((srcKeyRate - dstKeyRate) / (bs.rankStep.KeyRate)) - if srcCmpDst > 0 { - return curKeyRk > oldKeyRk - } else { - return curKeyRk < oldKeyRk + byteRkCmp := rankCmp(bs.cur.srcPeerStat.GetByteRate(), old.srcPeerStat.GetByteRate(), stepRank(0, 100)) + keyRkCmp := rankCmp(bs.cur.srcPeerStat.GetKeyRate(), old.srcPeerStat.GetKeyRate(), stepRank(0, 10)) + + switch bs.cur.progressiveRank { + case -2: // 0.95 < byteDecRatio <= 0.99 && keyDecRatio <= 0.95 + if keyRkCmp != 0 { + return keyRkCmp > 0 + } + if byteRkCmp != 0 { + // prefer smaller byte rate, to reduce oscillation + return byteRkCmp < 0 + } + case -3: // byteDecRatio <= 0.95 && keyDecRatio <= 0.95 + if keyRkCmp != 0 { + return keyRkCmp > 0 + } + fallthrough + case -1: // byteDecRatio <= 0.95 + if byteRkCmp != 0 { + // prefer region with larger byte rate, to converge faster + return byteRkCmp > 0 + } } } } @@ -746,24 +776,24 @@ func (bs *balanceSolver) compareSrcStore(st1, st2 uint64) int { if bs.rwTy == write && bs.opTy == transferLeader { lpCmp = sliceLPCmp( minLPCmp(negLoadCmp(sliceLoadCmp( - rankCmp(stLdCount, stepRank(bs.maxSrc.Count, bs.rankStep.Count)), - rankCmp(stLdKeyRate, stepRank(bs.maxSrc.KeyRate, bs.rankStep.KeyRate)), - rankCmp(stLdByteRate, stepRank(bs.maxSrc.ByteRate, bs.rankStep.ByteRate)), + stLdRankCmp(stLdCount, stepRank(bs.maxSrc.Count, bs.rankStep.Count)), + stLdRankCmp(stLdKeyRate, stepRank(bs.maxSrc.KeyRate, bs.rankStep.KeyRate)), + stLdRankCmp(stLdByteRate, stepRank(bs.maxSrc.ByteRate, bs.rankStep.ByteRate)), ))), diffCmp(sliceLoadCmp( - rankCmp(stLdCount, stepRank(0, bs.rankStep.Count)), - rankCmp(stLdKeyRate, stepRank(0, bs.rankStep.KeyRate)), - rankCmp(stLdByteRate, stepRank(0, bs.rankStep.ByteRate)), + stLdRankCmp(stLdCount, stepRank(0, bs.rankStep.Count)), + stLdRankCmp(stLdKeyRate, stepRank(0, bs.rankStep.KeyRate)), + stLdRankCmp(stLdByteRate, stepRank(0, bs.rankStep.ByteRate)), )), ) } else { lpCmp = sliceLPCmp( minLPCmp(negLoadCmp(sliceLoadCmp( - rankCmp(stLdByteRate, stepRank(bs.maxSrc.ByteRate, bs.rankStep.ByteRate)), - rankCmp(stLdKeyRate, stepRank(bs.maxSrc.KeyRate, bs.rankStep.KeyRate)), + stLdRankCmp(stLdByteRate, stepRank(bs.maxSrc.ByteRate, bs.rankStep.ByteRate)), + stLdRankCmp(stLdKeyRate, stepRank(bs.maxSrc.KeyRate, bs.rankStep.KeyRate)), ))), diffCmp( - rankCmp(stLdByteRate, stepRank(0, bs.rankStep.ByteRate)), + stLdRankCmp(stLdByteRate, stepRank(0, bs.rankStep.ByteRate)), ), ) } @@ -783,23 +813,23 @@ func (bs *balanceSolver) compareDstStore(st1, st2 uint64) int { if bs.rwTy == write && bs.opTy == transferLeader { lpCmp = sliceLPCmp( maxLPCmp(sliceLoadCmp( - rankCmp(stLdCount, stepRank(bs.minDst.Count, bs.rankStep.Count)), - rankCmp(stLdKeyRate, stepRank(bs.minDst.KeyRate, bs.rankStep.KeyRate)), - rankCmp(stLdByteRate, stepRank(bs.minDst.ByteRate, bs.rankStep.ByteRate)), + stLdRankCmp(stLdCount, stepRank(bs.minDst.Count, bs.rankStep.Count)), + stLdRankCmp(stLdKeyRate, stepRank(bs.minDst.KeyRate, bs.rankStep.KeyRate)), + stLdRankCmp(stLdByteRate, stepRank(bs.minDst.ByteRate, bs.rankStep.ByteRate)), )), diffCmp(sliceLoadCmp( - rankCmp(stLdCount, stepRank(0, bs.rankStep.Count)), - rankCmp(stLdKeyRate, stepRank(0, bs.rankStep.KeyRate)), - rankCmp(stLdByteRate, stepRank(0, bs.rankStep.ByteRate)), + stLdRankCmp(stLdCount, stepRank(0, bs.rankStep.Count)), + stLdRankCmp(stLdKeyRate, stepRank(0, bs.rankStep.KeyRate)), + stLdRankCmp(stLdByteRate, stepRank(0, bs.rankStep.ByteRate)), ))) } else { lpCmp = sliceLPCmp( maxLPCmp(sliceLoadCmp( - rankCmp(stLdByteRate, stepRank(bs.minDst.ByteRate, bs.rankStep.ByteRate)), - rankCmp(stLdKeyRate, stepRank(bs.minDst.KeyRate, bs.rankStep.KeyRate)), + stLdRankCmp(stLdByteRate, stepRank(bs.minDst.ByteRate, bs.rankStep.ByteRate)), + stLdRankCmp(stLdKeyRate, stepRank(bs.minDst.KeyRate, bs.rankStep.KeyRate)), )), diffCmp( - rankCmp(stLdByteRate, stepRank(0, bs.rankStep.ByteRate)), + stLdRankCmp(stLdByteRate, stepRank(0, bs.rankStep.ByteRate)), ), ) } diff --git a/server/schedulers/utils.go b/server/schedulers/utils.go index 74aecec0357..854b110f6b7 100644 --- a/server/schedulers/utils.go +++ b/server/schedulers/utils.go @@ -252,16 +252,20 @@ func sliceLoadCmp(cmps ...storeLoadCmp) storeLoadCmp { } } -func rankCmp(dim func(ld *storeLoad) float64, rank func(value float64) int64) storeLoadCmp { +func stLdRankCmp(dim func(ld *storeLoad) float64, rank func(value float64) int64) storeLoadCmp { return func(ld1, ld2 *storeLoad) int { - rk1, rk2 := rank(dim(ld1)), rank(dim(ld2)) - if rk1 < rk2 { - return -1 - } else if rk1 > rk2 { - return 1 - } - return 0 + return rankCmp(dim(ld1), dim(ld2), rank) + } +} + +func rankCmp(a, b float64, rank func(value float64) int64) int { + aRk, bRk := rank(a), rank(b) + if aRk < bRk { + return -1 + } else if aRk > bRk { + return 1 } + return 0 } // store load prediction From f49baed4e02b36d6795b98bba58dab214247d273 Mon Sep 17 00:00:00 2001 From: Luffbee Date: Mon, 17 Feb 2020 18:07:45 +0800 Subject: [PATCH 16/29] fix old tests --- server/schedulers/hot_test.go | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/server/schedulers/hot_test.go b/server/schedulers/hot_test.go index 156ecf09731..337bc906657 100644 --- a/server/schedulers/hot_test.go +++ b/server/schedulers/hot_test.go @@ -388,12 +388,12 @@ func (s *testHotReadRegionSchedulerSuite) TestSchedule(c *C) { //| store_id | read_bytes_rate | //|----------|-----------------| //| 1 | 7.5MB | - //| 2 | 4.6MB | + //| 2 | 4.9MB | //| 3 | 4.5MB | //| 4 | 6MB | //| 5 | 0MB | tc.UpdateStorageReadBytes(1, 7.5*MB*statistics.StoreHeartBeatReportInterval) - tc.UpdateStorageReadBytes(2, 4.6*MB*statistics.StoreHeartBeatReportInterval) + tc.UpdateStorageReadBytes(2, 4.9*MB*statistics.StoreHeartBeatReportInterval) tc.UpdateStorageReadBytes(3, 4.5*MB*statistics.StoreHeartBeatReportInterval) tc.UpdateStorageReadBytes(4, 6*MB*statistics.StoreHeartBeatReportInterval) tc.UpdateStorageReadBytes(5, 0) @@ -426,10 +426,6 @@ func (s *testHotReadRegionSchedulerSuite) TestSchedule(c *C) { } } - // Will transfer a hot region leader from store 1 to store 3. - // bytes_rate[store 1] * 0.9 > bytes_rate[store 3] + region_bytes_rate - // read_bytes_rate[store 3] < read_bytes_rate[store 2] - // when select dest store for hot read, we use score. testutil.CheckTransferLeader(c, hb.Schedule(tc)[0], operator.OpHotRegion, 1, 3) hb.(*hotScheduler).clearPendingInfluence() // assume handle the operator @@ -441,13 +437,13 @@ func (s *testHotReadRegionSchedulerSuite) TestSchedule(c *C) { //|----------|-----------------| //| 1 | 6MB | //| 2 | 5.5MB | - //| 3 | 5.9MB | - //| 4 | 3.1MB | + //| 3 | 5.5MB | + //| 4 | 3.4MB | //| 5 | 3MB | tc.UpdateStorageReadBytes(1, 6*MB*statistics.StoreHeartBeatReportInterval) tc.UpdateStorageReadBytes(2, 5.5*MB*statistics.StoreHeartBeatReportInterval) - tc.UpdateStorageReadBytes(3, 5.9*MB*statistics.StoreHeartBeatReportInterval) - tc.UpdateStorageReadBytes(4, 3.1*MB*statistics.StoreHeartBeatReportInterval) + tc.UpdateStorageReadBytes(3, 5.5*MB*statistics.StoreHeartBeatReportInterval) + tc.UpdateStorageReadBytes(4, 3.4*MB*statistics.StoreHeartBeatReportInterval) tc.UpdateStorageReadBytes(5, 3*MB*statistics.StoreHeartBeatReportInterval) //| region_id | leader_store | follower_store | follower_store | read_bytes_rate | From 517247a6966056f3d4fe0c5e7ea5d7cd5bd6dd80 Mon Sep 17 00:00:00 2001 From: Luffbee Date: Mon, 17 Feb 2020 18:20:36 +0800 Subject: [PATCH 17/29] remove unused code in test --- server/schedulers/hot_test.go | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/server/schedulers/hot_test.go b/server/schedulers/hot_test.go index 337bc906657..325267ae704 100644 --- a/server/schedulers/hot_test.go +++ b/server/schedulers/hot_test.go @@ -564,20 +564,6 @@ func (s *testHotCacheSuite) TestUpdateCache(c *C) { opt := mockoption.NewScheduleOptions() tc := mockcluster.NewCluster(opt) - // Add stores 1, 2, 3, 4, 5 with region counts 3, 2, 2, 2, 0. - tc.AddRegionStore(1, 3) - tc.AddRegionStore(2, 2) - tc.AddRegionStore(3, 2) - tc.AddRegionStore(4, 2) - tc.AddRegionStore(5, 0) - - // Report store read bytes. - tc.UpdateStorageReadBytes(1, 7.5*MB*statistics.StoreHeartBeatReportInterval) - tc.UpdateStorageReadBytes(2, 4.5*MB*statistics.StoreHeartBeatReportInterval) - tc.UpdateStorageReadBytes(3, 4.5*MB*statistics.StoreHeartBeatReportInterval) - tc.UpdateStorageReadBytes(4, 6*MB*statistics.StoreHeartBeatReportInterval) - tc.UpdateStorageReadBytes(5, 0) - /// For read flow tc.AddLeaderRegionWithReadInfo(1, 1, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 2, 3) tc.AddLeaderRegionWithReadInfo(2, 2, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 1, 3) @@ -598,12 +584,6 @@ func (s *testHotCacheSuite) TestUpdateCache(c *C) { c.Assert(len(stats[2]), Equals, 2) c.Assert(len(stats[3]), Equals, 0) - // For write flow - tc.UpdateStorageWrittenBytes(1, 6*MB*statistics.StoreHeartBeatReportInterval) - tc.UpdateStorageWrittenBytes(2, 3*MB*statistics.StoreHeartBeatReportInterval) - tc.UpdateStorageWrittenBytes(3, 6*MB*statistics.StoreHeartBeatReportInterval) - tc.UpdateStorageWrittenBytes(4, 3*MB*statistics.StoreHeartBeatReportInterval) - tc.UpdateStorageWrittenBytes(5, 0) tc.AddLeaderRegionWithWriteInfo(4, 1, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 2, 3) tc.AddLeaderRegionWithWriteInfo(5, 1, 20*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 2, 3) tc.AddLeaderRegionWithWriteInfo(6, 1, 0.8*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 2, 3) From bff084c13abead4890702d307c36389209e159d3 Mon Sep 17 00:00:00 2001 From: Luffbee Date: Mon, 17 Feb 2020 21:28:42 +0800 Subject: [PATCH 18/29] add test for hot cache --- pkg/mock/mockcluster/mockcluster.go | 40 ++++- server/schedulers/hot_test.go | 245 +++++++++++++++++++++++----- server/schedulers/scheduler_test.go | 12 +- 3 files changed, 244 insertions(+), 53 deletions(-) diff --git a/pkg/mock/mockcluster/mockcluster.go b/pkg/mock/mockcluster/mockcluster.go index 8814be963de..78cccdf8dd7 100644 --- a/pkg/mock/mockcluster/mockcluster.go +++ b/pkg/mock/mockcluster/mockcluster.go @@ -270,9 +270,14 @@ func (mc *Cluster) AddLeaderRegionWithRange(regionID uint64, startKey string, en } // AddLeaderRegionWithReadInfo adds region with specified leader, followers and read info. -func (mc *Cluster) AddLeaderRegionWithReadInfo(regionID uint64, leaderID uint64, readBytes uint64, reportInterval uint64, followerIds ...uint64) { +func (mc *Cluster) AddLeaderRegionWithReadInfo( + regionID uint64, leaderID uint64, + readBytes, readKeys uint64, + reportInterval uint64, + followerIds []uint64) { r := mc.newMockRegionInfo(regionID, leaderID, followerIds...) r = r.Clone(core.SetReadBytes(readBytes)) + r = r.Clone(core.SetReadKeys(readKeys)) r = r.Clone(core.SetReportInterval(reportInterval)) items := mc.HotCache.CheckRead(r, mc.StoresStats) for _, item := range items { @@ -282,9 +287,14 @@ func (mc *Cluster) AddLeaderRegionWithReadInfo(regionID uint64, leaderID uint64, } // AddLeaderRegionWithWriteInfo adds region with specified leader, followers and write info. -func (mc *Cluster) AddLeaderRegionWithWriteInfo(regionID uint64, leaderID uint64, writtenBytes uint64, reportInterval uint64, followerIds ...uint64) { +func (mc *Cluster) AddLeaderRegionWithWriteInfo( + regionID uint64, leaderID uint64, + writtenBytes, writtenKeys uint64, + reportInterval uint64, + followerIds []uint64) { r := mc.newMockRegionInfo(regionID, leaderID, followerIds...) r = r.Clone(core.SetWrittenBytes(writtenBytes)) + r = r.Clone(core.SetWrittenKeys(writtenKeys)) r = r.Clone(core.SetReportInterval(reportInterval)) items := mc.HotCache.CheckWrite(r, mc.StoresStats) for _, item := range items { @@ -404,6 +414,32 @@ func (mc *Cluster) UpdateStorageReadBytes(storeID uint64, bytesRead uint64) { mc.PutStore(newStore) } +// UpdateStorageWrittenKeys updates store written keys. +func (mc *Cluster) UpdateStorageWrittenKeys(storeID uint64, keysWritten uint64) { + store := mc.GetStore(storeID) + newStats := proto.Clone(store.GetStoreStats()).(*pdpb.StoreStats) + newStats.KeysWritten = keysWritten + now := time.Now().Second() + interval := &pdpb.TimeInterval{StartTimestamp: uint64(now - statistics.StoreHeartBeatReportInterval), EndTimestamp: uint64(now)} + newStats.Interval = interval + newStore := store.Clone(core.SetStoreStats(newStats)) + mc.Set(storeID, newStats) + mc.PutStore(newStore) +} + +// UpdateStorageReadKeys updates store read bytes. +func (mc *Cluster) UpdateStorageReadKeys(storeID uint64, keysRead uint64) { + store := mc.GetStore(storeID) + newStats := proto.Clone(store.GetStoreStats()).(*pdpb.StoreStats) + newStats.KeysRead = keysRead + now := time.Now().Second() + interval := &pdpb.TimeInterval{StartTimestamp: uint64(now - statistics.StoreHeartBeatReportInterval), EndTimestamp: uint64(now)} + newStats.Interval = interval + newStore := store.Clone(core.SetStoreStats(newStats)) + mc.Set(storeID, newStats) + mc.PutStore(newStore) +} + // UpdateStoreStatus updates store status. func (mc *Cluster) UpdateStoreStatus(id uint64) { leaderCount := mc.Regions.GetStoreLeaderCount(id) diff --git a/server/schedulers/hot_test.go b/server/schedulers/hot_test.go index 325267ae704..4bcfc29ae81 100644 --- a/server/schedulers/hot_test.go +++ b/server/schedulers/hot_test.go @@ -163,9 +163,11 @@ func (s *testHotWriteRegionSchedulerSuite) checkSchedule(c *C, tc *mockcluster.C //| 2 | 1 | 3 | 4 | 512KB | //| 3 | 1 | 2 | 4 | 512KB | // Region 1, 2 and 3 are hot regions. - tc.AddLeaderRegionWithWriteInfo(1, 1, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 2, 3) - tc.AddLeaderRegionWithWriteInfo(2, 1, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 3, 4) - tc.AddLeaderRegionWithWriteInfo(3, 1, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 2, 4) + addRegionInfo(tc, write, []testRegionInfo{ + {1, []uint64{1, 2, 3}, 512 * KB, 0}, + {2, []uint64{1, 3, 4}, 512 * KB, 0}, + {3, []uint64{1, 2, 4}, 512 * KB, 0}, + }) // Will transfer a hot region from store 1, because the total count of peers // which is hot for store 1 is more larger than other stores. @@ -242,11 +244,13 @@ func (s *testHotWriteRegionSchedulerSuite) checkSchedule(c *C, tc *mockcluster.C //| 3 | 6 | 1 | 4 | 512KB | //| 4 | 5 | 6 | 4 | 512KB | //| 5 | 3 | 4 | 5 | 512KB | - tc.AddLeaderRegionWithWriteInfo(1, 1, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 2, 3) - tc.AddLeaderRegionWithWriteInfo(2, 1, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 2, 3) - tc.AddLeaderRegionWithWriteInfo(3, 6, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 1, 4) - tc.AddLeaderRegionWithWriteInfo(4, 5, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 6, 4) - tc.AddLeaderRegionWithWriteInfo(5, 3, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 4, 5) + addRegionInfo(tc, write, []testRegionInfo{ + {1, []uint64{1, 2, 3}, 512 * KB, 0}, + {2, []uint64{1, 2, 3}, 512 * KB, 0}, + {3, []uint64{6, 1, 4}, 512 * KB, 0}, + {4, []uint64{5, 6, 4}, 512 * KB, 0}, + {5, []uint64{3, 4, 5}, 512 * KB, 0}, + }) // 6 possible operator. // Assuming different operators have the same possibility, @@ -323,12 +327,14 @@ func (s *testHotWriteRegionSchedulerSuite) TestWithPendingInfluence(c *C) { //| 5 | 1 | 2 | 3 | 512KB | //| 6 | 1 | 2 | 3 | 512KB | // All regions are hot. - tc.AddLeaderRegionWithWriteInfo(1, 1, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 2, 3) - tc.AddLeaderRegionWithWriteInfo(2, 1, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 2, 3) - tc.AddLeaderRegionWithWriteInfo(3, 1, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 2, 3) - tc.AddLeaderRegionWithWriteInfo(4, 1, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 2, 3) - tc.AddLeaderRegionWithWriteInfo(5, 1, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 2, 3) - tc.AddLeaderRegionWithWriteInfo(6, 1, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 2, 3) + addRegionInfo(tc, write, []testRegionInfo{ + {1, []uint64{1, 2, 3}, 512 * KB, 0}, + {2, []uint64{1, 2, 3}, 512 * KB, 0}, + {3, []uint64{1, 2, 3}, 512 * KB, 0}, + {4, []uint64{1, 2, 3}, 512 * KB, 0}, + {5, []uint64{1, 2, 3}, 512 * KB, 0}, + {6, []uint64{1, 2, 3}, 512 * KB, 0}, + }) for i := 0; i < 20; i++ { hb.(*hotScheduler).clearPendingInfluence() @@ -403,13 +409,14 @@ func (s *testHotReadRegionSchedulerSuite) TestSchedule(c *C) { //| 1 | 1 | 2 | 3 | 512KB | //| 2 | 2 | 1 | 3 | 512KB | //| 3 | 1 | 2 | 3 | 512KB | - //| 11 | 1 | 2 | 3 | 24KB | + //| 11 | 1 | 2 | 3 | 7KB | // Region 1, 2 and 3 are hot regions. - tc.AddLeaderRegionWithReadInfo(1, 1, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 2, 3) - tc.AddLeaderRegionWithReadInfo(2, 2, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 1, 3) - tc.AddLeaderRegionWithReadInfo(3, 1, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 2, 3) - // lower than hot read flow rate, but higher than write flow rate - tc.AddLeaderRegionWithReadInfo(11, 1, 7*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 2, 3) + addRegionInfo(tc, read, []testRegionInfo{ + {1, []uint64{1, 2, 3}, 512 * KB, 0}, + {2, []uint64{2, 1, 3}, 512 * KB, 0}, + {3, []uint64{1, 2, 3}, 512 * KB, 0}, + {11, []uint64{1, 2, 3}, 7 * KB, 0}, + }) c.Assert(tc.IsRegionHot(tc.GetRegion(1)), IsTrue) c.Assert(tc.IsRegionHot(tc.GetRegion(11)), IsFalse) @@ -429,7 +436,7 @@ func (s *testHotReadRegionSchedulerSuite) TestSchedule(c *C) { testutil.CheckTransferLeader(c, hb.Schedule(tc)[0], operator.OpHotRegion, 1, 3) hb.(*hotScheduler).clearPendingInfluence() // assume handle the operator - tc.AddLeaderRegionWithReadInfo(3, 3, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 1, 2) + tc.AddLeaderRegionWithReadInfo(3, 3, 512*KB*statistics.RegionHeartBeatReportInterval, 0, statistics.RegionHeartBeatReportInterval, []uint64{1, 2}) // After transfer a hot region leader from store 1 to store 3 // the three region leader will be evenly distributed in three stores @@ -454,8 +461,10 @@ func (s *testHotReadRegionSchedulerSuite) TestSchedule(c *C) { //| 4 | 1 | 2 | 3 | 512KB | //| 5 | 4 | 2 | 5 | 512KB | //| 11 | 1 | 2 | 3 | 24KB | - tc.AddLeaderRegionWithReadInfo(4, 1, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 2, 3) - tc.AddLeaderRegionWithReadInfo(5, 4, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 2, 5) + addRegionInfo(tc, read, []testRegionInfo{ + {4, []uint64{1, 2, 3}, 512 * KB, 0}, + {5, []uint64{4, 2, 5}, 512 * KB, 0}, + }) // We will move leader peer of region 1 from 1 to 5 testutil.CheckTransferPeerWithLeaderTransfer(c, hb.Schedule(tc)[0], operator.OpHotRegion, 1, 5) @@ -504,14 +513,16 @@ func (s *testHotReadRegionSchedulerSuite) TestWithPendingInfluence(c *C) { //| 6 | 2 | 1 | 3 | 512KB | //| 7 | 3 | 1 | 2 | 512KB | //| 8 | 3 | 1 | 2 | 512KB | - tc.AddLeaderRegionWithReadInfo(1, 1, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 2, 3) - tc.AddLeaderRegionWithReadInfo(2, 1, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 2, 3) - tc.AddLeaderRegionWithReadInfo(3, 1, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 2, 3) - tc.AddLeaderRegionWithReadInfo(4, 1, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 2, 3) - tc.AddLeaderRegionWithReadInfo(5, 2, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 1, 3) - tc.AddLeaderRegionWithReadInfo(6, 2, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 1, 3) - tc.AddLeaderRegionWithReadInfo(7, 3, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 1, 2) - tc.AddLeaderRegionWithReadInfo(8, 3, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 1, 2) + addRegionInfo(tc, read, []testRegionInfo{ + {1, []uint64{1, 2, 3}, 512 * KB, 0}, + {2, []uint64{1, 2, 3}, 512 * KB, 0}, + {3, []uint64{1, 2, 3}, 512 * KB, 0}, + {4, []uint64{1, 2, 3}, 512 * KB, 0}, + {5, []uint64{2, 1, 3}, 512 * KB, 0}, + {6, []uint64{2, 1, 3}, 512 * KB, 0}, + {7, []uint64{3, 2, 1}, 512 * KB, 0}, + {8, []uint64{3, 2, 1}, 512 * KB, 0}, + }) for i := 0; i < 20; i++ { hb.(*hotScheduler).clearPendingInfluence() @@ -562,38 +573,44 @@ type testHotCacheSuite struct{} func (s *testHotCacheSuite) TestUpdateCache(c *C) { opt := mockoption.NewScheduleOptions() + opt.HotRegionCacheHitsThreshold = 0 tc := mockcluster.NewCluster(opt) /// For read flow - tc.AddLeaderRegionWithReadInfo(1, 1, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 2, 3) - tc.AddLeaderRegionWithReadInfo(2, 2, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 1, 3) - tc.AddLeaderRegionWithReadInfo(3, 1, 20*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 2, 3) - // lower than hot read flow rate, but higher than write flow rate - tc.AddLeaderRegionWithReadInfo(11, 1, 7*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 2, 3) - opt.HotRegionCacheHitsThreshold = 0 + addRegionInfo(tc, read, []testRegionInfo{ + {1, []uint64{1, 2, 3}, 512 * KB, 0}, + {2, []uint64{2, 1, 3}, 512 * KB, 0}, + {3, []uint64{1, 2, 3}, 20 * KB, 0}, + // lower than hot read flow rate, but higher than write flow rate + {11, []uint64{1, 2, 3}, 7 * KB, 0}, + }) stats := tc.RegionStats(statistics.ReadFlow) c.Assert(len(stats[1]), Equals, 2) c.Assert(len(stats[2]), Equals, 1) c.Assert(len(stats[3]), Equals, 0) - tc.AddLeaderRegionWithReadInfo(3, 2, 20*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 2, 3) - tc.AddLeaderRegionWithReadInfo(11, 1, 7*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 2, 3) + addRegionInfo(tc, read, []testRegionInfo{ + {3, []uint64{2, 1, 3}, 20 * KB, 0}, + {11, []uint64{1, 2, 3}, 7 * KB, 0}, + }) stats = tc.RegionStats(statistics.ReadFlow) - c.Assert(len(stats[1]), Equals, 1) c.Assert(len(stats[2]), Equals, 2) c.Assert(len(stats[3]), Equals, 0) - tc.AddLeaderRegionWithWriteInfo(4, 1, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 2, 3) - tc.AddLeaderRegionWithWriteInfo(5, 1, 20*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 2, 3) - tc.AddLeaderRegionWithWriteInfo(6, 1, 0.8*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 2, 3) - + addRegionInfo(tc, write, []testRegionInfo{ + {4, []uint64{1, 2, 3}, 512 * KB, 0}, + {5, []uint64{1, 2, 3}, 20 * KB, 0}, + {6, []uint64{1, 2, 3}, 0.8 * KB, 0}, + }) stats = tc.RegionStats(statistics.WriteFlow) c.Assert(len(stats[1]), Equals, 2) c.Assert(len(stats[2]), Equals, 2) c.Assert(len(stats[3]), Equals, 2) - tc.AddLeaderRegionWithWriteInfo(5, 1, 20*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 2, 5) + addRegionInfo(tc, write, []testRegionInfo{ + {5, []uint64{1, 2, 5}, 20 * KB, 0}, + }) stats = tc.RegionStats(statistics.WriteFlow) c.Assert(len(stats[1]), Equals, 2) @@ -601,3 +618,141 @@ func (s *testHotCacheSuite) TestUpdateCache(c *C) { c.Assert(len(stats[3]), Equals, 1) c.Assert(len(stats[5]), Equals, 1) } + +func (s *testHotCacheSuite) TestKeyThresholds(c *C) { + opt := mockoption.NewScheduleOptions() + opt.HotRegionCacheHitsThreshold = 0 + { // only a few regions + tc := mockcluster.NewCluster(opt) + addRegionInfo(tc, read, []testRegionInfo{ + {1, []uint64{1, 2, 3}, 0, 1}, + {2, []uint64{1, 2, 3}, 0, 1 * KB}, + }) + stats := tc.RegionStats(statistics.ReadFlow) + c.Assert(stats[1], HasLen, 1) + addRegionInfo(tc, write, []testRegionInfo{ + {3, []uint64{4, 5, 6}, 0, 1}, + {4, []uint64{4, 5, 6}, 0, 1 * KB}, + }) + stats = tc.RegionStats(statistics.WriteFlow) + c.Assert(stats[4], HasLen, 1) + c.Assert(stats[5], HasLen, 1) + c.Assert(stats[6], HasLen, 1) + } + { // many regions + tc := mockcluster.NewCluster(opt) + regions := []testRegionInfo{} + for i := 1; i <= 1000; i += 2 { + regions = append(regions, testRegionInfo{ + id: uint64(i), + peers: []uint64{1, 2, 3}, + keyRate: 100 * KB, + }) + regions = append(regions, testRegionInfo{ + id: uint64(i + 1), + peers: []uint64{1, 2, 3}, + keyRate: 10 * KB, + }) + } + + { // read + addRegionInfo(tc, read, regions) + stats := tc.RegionStats(statistics.ReadFlow) + c.Assert(len(stats[1]), Greater, 500) + + // for AntiCount + addRegionInfo(tc, read, regions) + addRegionInfo(tc, read, regions) + addRegionInfo(tc, read, regions) + addRegionInfo(tc, read, regions) + stats = tc.RegionStats(statistics.ReadFlow) + c.Assert(len(stats[1]), Equals, 500) + } + { // write + addRegionInfo(tc, write, regions) + stats := tc.RegionStats(statistics.WriteFlow) + c.Assert(len(stats[1]), Greater, 500) + c.Assert(len(stats[2]), Greater, 500) + c.Assert(len(stats[3]), Greater, 500) + + // for AntiCount + addRegionInfo(tc, write, regions) + addRegionInfo(tc, write, regions) + addRegionInfo(tc, write, regions) + addRegionInfo(tc, write, regions) + stats = tc.RegionStats(statistics.WriteFlow) + c.Assert(len(stats[1]), Equals, 500) + c.Assert(len(stats[2]), Equals, 500) + c.Assert(len(stats[3]), Equals, 500) + } + } +} + +func (s *testHotCacheSuite) TestByteAndKey(c *C) { + opt := mockoption.NewScheduleOptions() + opt.HotRegionCacheHitsThreshold = 0 + tc := mockcluster.NewCluster(opt) + regions := []testRegionInfo{} + for i := 1; i <= 500; i++ { + regions = append(regions, testRegionInfo{ + id: uint64(i), + peers: []uint64{1, 2, 3}, + byteRate: 100 * KB, + keyRate: 100 * KB, + }) + } + { // read + addRegionInfo(tc, read, regions) + stats := tc.RegionStats(statistics.ReadFlow) + c.Assert(len(stats[1]), Equals, 500) + + addRegionInfo(tc, read, []testRegionInfo{ + {10001, []uint64{1, 2, 3}, 10 * KB, 10 * KB}, + {10002, []uint64{1, 2, 3}, 500 * KB, 10 * KB}, + {10003, []uint64{1, 2, 3}, 10 * KB, 500 * KB}, + {10004, []uint64{1, 2, 3}, 500 * KB, 500 * KB}, + }) + stats = tc.RegionStats(statistics.ReadFlow) + c.Assert(len(stats[1]), Equals, 503) + } + { // write + addRegionInfo(tc, write, regions) + stats := tc.RegionStats(statistics.WriteFlow) + c.Assert(len(stats[1]), Equals, 500) + c.Assert(len(stats[2]), Equals, 500) + c.Assert(len(stats[3]), Equals, 500) + addRegionInfo(tc, write, []testRegionInfo{ + {10001, []uint64{1, 2, 3}, 10 * KB, 10 * KB}, + {10002, []uint64{1, 2, 3}, 500 * KB, 10 * KB}, + {10003, []uint64{1, 2, 3}, 10 * KB, 500 * KB}, + {10004, []uint64{1, 2, 3}, 500 * KB, 500 * KB}, + }) + stats = tc.RegionStats(statistics.WriteFlow) + c.Assert(len(stats[1]), Equals, 503) + c.Assert(len(stats[2]), Equals, 503) + c.Assert(len(stats[3]), Equals, 503) + } +} + +type testRegionInfo struct { + id uint64 + peers []uint64 + byteRate float64 + keyRate float64 +} + +func addRegionInfo(tc *mockcluster.Cluster, rwTy rwType, regions []testRegionInfo) { + addFunc := tc.AddLeaderRegionWithReadInfo + if rwTy == write { + addFunc = tc.AddLeaderRegionWithWriteInfo + } + for _, r := range regions { + addFunc( + r.id, r.peers[0], + uint64(r.byteRate*statistics.RegionHeartBeatReportInterval), + uint64(r.keyRate*statistics.RegionHeartBeatReportInterval), + statistics.RegionHeartBeatReportInterval, + r.peers[1:], + ) + } +} diff --git a/server/schedulers/scheduler_test.go b/server/schedulers/scheduler_test.go index b344c03bebf..eb359d3a807 100644 --- a/server/schedulers/scheduler_test.go +++ b/server/schedulers/scheduler_test.go @@ -381,9 +381,9 @@ func (s *testShuffleHotRegionSchedulerSuite) checkBalance(c *C, tc *mockcluster. //| 1 | 1 | 2 | 3 | 512KB | //| 2 | 1 | 3 | 4 | 512KB | //| 3 | 1 | 2 | 4 | 512KB | - tc.AddLeaderRegionWithWriteInfo(1, 1, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 2, 3) - tc.AddLeaderRegionWithWriteInfo(2, 1, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 3, 4) - tc.AddLeaderRegionWithWriteInfo(3, 1, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 2, 4) + tc.AddLeaderRegionWithWriteInfo(1, 1, 512*KB*statistics.RegionHeartBeatReportInterval, 0, statistics.RegionHeartBeatReportInterval, []uint64{2, 3}) + tc.AddLeaderRegionWithWriteInfo(2, 1, 512*KB*statistics.RegionHeartBeatReportInterval, 0, statistics.RegionHeartBeatReportInterval, []uint64{3, 4}) + tc.AddLeaderRegionWithWriteInfo(3, 1, 512*KB*statistics.RegionHeartBeatReportInterval, 0, statistics.RegionHeartBeatReportInterval, []uint64{2, 4}) opt.HotRegionCacheHitsThreshold = 0 // try to get an operator @@ -421,9 +421,9 @@ func (s *testHotRegionSchedulerSuite) TestAbnormalReplica(c *C) { tc.UpdateStorageReadBytes(2, 4.5*MB*statistics.StoreHeartBeatReportInterval) tc.UpdateStorageReadBytes(3, 4.5*MB*statistics.StoreHeartBeatReportInterval) - tc.AddLeaderRegionWithReadInfo(1, 1, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 2) - tc.AddLeaderRegionWithReadInfo(2, 2, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 1, 3) - tc.AddLeaderRegionWithReadInfo(3, 1, 512*KB*statistics.RegionHeartBeatReportInterval, statistics.RegionHeartBeatReportInterval, 2, 3) + tc.AddLeaderRegionWithReadInfo(1, 1, 512*KB*statistics.RegionHeartBeatReportInterval, 0, statistics.RegionHeartBeatReportInterval, []uint64{2}) + tc.AddLeaderRegionWithReadInfo(2, 2, 512*KB*statistics.RegionHeartBeatReportInterval, 0, statistics.RegionHeartBeatReportInterval, []uint64{1, 3}) + tc.AddLeaderRegionWithReadInfo(3, 1, 512*KB*statistics.RegionHeartBeatReportInterval, 0, statistics.RegionHeartBeatReportInterval, []uint64{2, 3}) opt.HotRegionCacheHitsThreshold = 0 c.Assert(tc.IsRegionHot(tc.GetRegion(1)), IsTrue) c.Assert(hb.Schedule(tc), IsNil) From acd8281a1ea9cd540c66f7d364169d7a1098a14e Mon Sep 17 00:00:00 2001 From: Luffbee Date: Thu, 20 Feb 2020 15:24:02 +0800 Subject: [PATCH 19/29] update TestWithPendingInfluence --- server/schedulers/hot_test.go | 270 ++++++++++++++++++---------------- 1 file changed, 140 insertions(+), 130 deletions(-) diff --git a/server/schedulers/hot_test.go b/server/schedulers/hot_test.go index 4bcfc29ae81..b8d703ab454 100644 --- a/server/schedulers/hot_test.go +++ b/server/schedulers/hot_test.go @@ -296,78 +296,82 @@ func (s *testHotWriteRegionSchedulerSuite) TestWithPendingInfluence(c *C) { defer cancel() statistics.Denoising = false opt := mockoption.NewScheduleOptions() - tc := mockcluster.NewCluster(opt) hb, err := schedule.CreateScheduler(HotWriteRegionType, schedule.NewOperatorController(ctx, nil, nil), core.NewStorage(kv.NewMemoryKV()), nil) c.Assert(err, IsNil) opt.HotRegionCacheHitsThreshold = 0 opt.LeaderScheduleLimit = 0 - tc.AddRegionStore(1, 20) - tc.AddRegionStore(2, 20) - tc.AddRegionStore(3, 20) - tc.AddRegionStore(4, 20) - - //| store_id | write_bytes_rate | - //|----------|------------------| - //| 1 | 8MB | - //| 2 | 6MB | - //| 3 | 6MB | - //| 4 | 4MB | - tc.UpdateStorageWrittenBytes(1, 8*MB*statistics.StoreHeartBeatReportInterval) - tc.UpdateStorageWrittenBytes(2, 6*MB*statistics.StoreHeartBeatReportInterval) - tc.UpdateStorageWrittenBytes(3, 6*MB*statistics.StoreHeartBeatReportInterval) - tc.UpdateStorageWrittenBytes(4, 4*MB*statistics.StoreHeartBeatReportInterval) - - //| region_id | leader_store | follower_store | follower_store | written_bytes | - //|-----------|--------------|----------------|----------------|---------------| - //| 1 | 1 | 2 | 3 | 512KB | - //| 2 | 1 | 2 | 3 | 512KB | - //| 3 | 1 | 2 | 3 | 512KB | - //| 4 | 1 | 2 | 3 | 512KB | - //| 5 | 1 | 2 | 3 | 512KB | - //| 6 | 1 | 2 | 3 | 512KB | - // All regions are hot. - addRegionInfo(tc, write, []testRegionInfo{ - {1, []uint64{1, 2, 3}, 512 * KB, 0}, - {2, []uint64{1, 2, 3}, 512 * KB, 0}, - {3, []uint64{1, 2, 3}, 512 * KB, 0}, - {4, []uint64{1, 2, 3}, 512 * KB, 0}, - {5, []uint64{1, 2, 3}, 512 * KB, 0}, - {6, []uint64{1, 2, 3}, 512 * KB, 0}, - }) + for i := 0; i < 2; i++ { + // 0: byte rate + // 1: key rate + tc := mockcluster.NewCluster(opt) + tc.AddRegionStore(1, 20) + tc.AddRegionStore(2, 20) + tc.AddRegionStore(3, 20) + tc.AddRegionStore(4, 20) + + updateStore := tc.UpdateStorageWrittenBytes // byte rate + if i == 1 { // key rate + updateStore = tc.UpdateStorageWrittenKeys + } + updateStore(1, 8*MB*statistics.StoreHeartBeatReportInterval) + updateStore(2, 6*MB*statistics.StoreHeartBeatReportInterval) + updateStore(3, 6*MB*statistics.StoreHeartBeatReportInterval) + updateStore(4, 4*MB*statistics.StoreHeartBeatReportInterval) + + if i == 0 { // byte rate + addRegionInfo(tc, write, []testRegionInfo{ + {1, []uint64{1, 2, 3}, 512 * KB, 0}, + {2, []uint64{1, 2, 3}, 512 * KB, 0}, + {3, []uint64{1, 2, 3}, 512 * KB, 0}, + {4, []uint64{1, 2, 3}, 512 * KB, 0}, + {5, []uint64{1, 2, 3}, 512 * KB, 0}, + {6, []uint64{1, 2, 3}, 512 * KB, 0}, + }) + } else if i == 1 { // key rate + addRegionInfo(tc, write, []testRegionInfo{ + {1, []uint64{1, 2, 3}, 0, 512 * KB}, + {2, []uint64{1, 2, 3}, 0, 512 * KB}, + {3, []uint64{1, 2, 3}, 0, 512 * KB}, + {4, []uint64{1, 2, 3}, 0, 512 * KB}, + {5, []uint64{1, 2, 3}, 0, 512 * KB}, + {6, []uint64{1, 2, 3}, 0, 512 * KB}, + }) + } - for i := 0; i < 20; i++ { - hb.(*hotScheduler).clearPendingInfluence() - cnt := 0 - testLoop: - for j := 0; j < 1000; j++ { - c.Assert(cnt, LessEqual, 5) - emptyCnt := 0 - ops := hb.Schedule(tc) - for len(ops) == 0 { - emptyCnt++ - if emptyCnt >= 10 { - break testLoop + for i := 0; i < 20; i++ { + hb.(*hotScheduler).clearPendingInfluence() + cnt := 0 + testLoop: + for j := 0; j < 1000; j++ { + c.Assert(cnt, LessEqual, 5) + emptyCnt := 0 + ops := hb.Schedule(tc) + for len(ops) == 0 { + emptyCnt++ + if emptyCnt >= 10 { + break testLoop + } + ops = hb.Schedule(tc) } - ops = hb.Schedule(tc) - } - op := ops[0] - switch op.Len() { - case 1: - // balance by leader selected - testutil.CheckTransferLeaderFrom(c, op, operator.OpHotRegion, 1) - case 4: - // balance by peer selected - testutil.CheckTransferPeerWithLeaderTransfer(c, op, operator.OpHotRegion, 1, 4) - cnt++ - if cnt == 3 { - c.Assert(op.Cancel(), IsTrue) + op := ops[0] + switch op.Len() { + case 1: + // balance by leader selected + testutil.CheckTransferLeaderFrom(c, op, operator.OpHotRegion, 1) + case 4: + // balance by peer selected + testutil.CheckTransferPeerWithLeaderTransfer(c, op, operator.OpHotRegion, 1, 4) + cnt++ + if cnt == 3 { + c.Assert(op.Cancel(), IsTrue) + } + default: + c.Fatalf("wrong op: %v", op) } - default: - c.Fatalf("wrong op: %v", op) } + c.Assert(cnt, Equals, 5) } - c.Assert(cnt, Equals, 5) } } @@ -482,88 +486,94 @@ func (s *testHotReadRegionSchedulerSuite) TestWithPendingInfluence(c *C) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() opt := mockoption.NewScheduleOptions() - tc := mockcluster.NewCluster(opt) hb, err := schedule.CreateScheduler(HotReadRegionType, schedule.NewOperatorController(ctx, nil, nil), core.NewStorage(kv.NewMemoryKV()), nil) c.Assert(err, IsNil) opt.HotRegionCacheHitsThreshold = 0 - tc.AddRegionStore(1, 20) - tc.AddRegionStore(2, 20) - tc.AddRegionStore(3, 20) - tc.AddRegionStore(4, 20) - - //| store_id | write_bytes_rate | - //|----------|------------------| - //| 1 | 7.1MB | - //| 2 | 6.1MB | - //| 3 | 6MB | - //| 4 | 5MB | - tc.UpdateStorageReadBytes(1, 7.1*MB*statistics.StoreHeartBeatReportInterval) - tc.UpdateStorageReadBytes(2, 6.1*MB*statistics.StoreHeartBeatReportInterval) - tc.UpdateStorageReadBytes(3, 6*MB*statistics.StoreHeartBeatReportInterval) - tc.UpdateStorageReadBytes(4, 5*MB*statistics.StoreHeartBeatReportInterval) - - //| region_id | leader_store | follower_store | follower_store | read_bytes_rate | - //|-----------|--------------|----------------|----------------|--------------------| - //| 1 | 1 | 2 | 3 | 512KB | - //| 2 | 1 | 2 | 3 | 512KB | - //| 3 | 1 | 2 | 3 | 512KB | - //| 4 | 1 | 2 | 3 | 512KB | - //| 5 | 2 | 1 | 3 | 512KB | - //| 6 | 2 | 1 | 3 | 512KB | - //| 7 | 3 | 1 | 2 | 512KB | - //| 8 | 3 | 1 | 2 | 512KB | - addRegionInfo(tc, read, []testRegionInfo{ - {1, []uint64{1, 2, 3}, 512 * KB, 0}, - {2, []uint64{1, 2, 3}, 512 * KB, 0}, - {3, []uint64{1, 2, 3}, 512 * KB, 0}, - {4, []uint64{1, 2, 3}, 512 * KB, 0}, - {5, []uint64{2, 1, 3}, 512 * KB, 0}, - {6, []uint64{2, 1, 3}, 512 * KB, 0}, - {7, []uint64{3, 2, 1}, 512 * KB, 0}, - {8, []uint64{3, 2, 1}, 512 * KB, 0}, - }) + for i := 0; i < 2; i++ { + // 0: byte rate + // 1: key rate + tc := mockcluster.NewCluster(opt) + tc.AddRegionStore(1, 20) + tc.AddRegionStore(2, 20) + tc.AddRegionStore(3, 20) + tc.AddRegionStore(4, 20) + + updateStore := tc.UpdateStorageReadBytes // byte rate + if i == 1 { // key rate + updateStore = tc.UpdateStorageReadKeys + } + updateStore(1, 7.1*MB*statistics.StoreHeartBeatReportInterval) + updateStore(2, 6.1*MB*statistics.StoreHeartBeatReportInterval) + updateStore(3, 6*MB*statistics.StoreHeartBeatReportInterval) + updateStore(4, 5*MB*statistics.StoreHeartBeatReportInterval) + + if i == 0 { // byte rate + addRegionInfo(tc, read, []testRegionInfo{ + {1, []uint64{1, 2, 3}, 512 * KB, 0}, + {2, []uint64{1, 2, 3}, 512 * KB, 0}, + {3, []uint64{1, 2, 3}, 512 * KB, 0}, + {4, []uint64{1, 2, 3}, 512 * KB, 0}, + {5, []uint64{2, 1, 3}, 512 * KB, 0}, + {6, []uint64{2, 1, 3}, 512 * KB, 0}, + {7, []uint64{3, 2, 1}, 512 * KB, 0}, + {8, []uint64{3, 2, 1}, 512 * KB, 0}, + }) + } else if i == 1 { // key rate + addRegionInfo(tc, read, []testRegionInfo{ + {1, []uint64{1, 2, 3}, 0, 512 * KB}, + {2, []uint64{1, 2, 3}, 0, 512 * KB}, + {3, []uint64{1, 2, 3}, 0, 512 * KB}, + {4, []uint64{1, 2, 3}, 0, 512 * KB}, + {5, []uint64{2, 1, 3}, 0, 512 * KB}, + {6, []uint64{2, 1, 3}, 0, 512 * KB}, + {7, []uint64{3, 2, 1}, 0, 512 * KB}, + {8, []uint64{3, 2, 1}, 0, 512 * KB}, + }) + } - for i := 0; i < 20; i++ { - hb.(*hotScheduler).clearPendingInfluence() + for i := 0; i < 20; i++ { + hb.(*hotScheduler).clearPendingInfluence() - op1 := hb.Schedule(tc)[0] - testutil.CheckTransferLeader(c, op1, operator.OpLeader, 1, 3) - // store byte rate (min, max): (6.6, 7.1) | 6.1 | (6, 6.5) | 5 + op1 := hb.Schedule(tc)[0] + testutil.CheckTransferLeader(c, op1, operator.OpLeader, 1, 3) + // store byte/key rate (min, max): (6.6, 7.1) | 6.1 | (6, 6.5) | 5 - op2 := hb.Schedule(tc)[0] - testutil.CheckTransferPeerWithLeaderTransfer(c, op2, operator.OpHotRegion, 1, 4) - // store byte rate (min, max): (6.1, 7.1) | 6.1 | (6, 6.5) | (5, 5.5) + op2 := hb.Schedule(tc)[0] + testutil.CheckTransferPeerWithLeaderTransfer(c, op2, operator.OpHotRegion, 1, 4) + // store byte/key rate (min, max): (6.1, 7.1) | 6.1 | (6, 6.5) | (5, 5.5) - ops := hb.Schedule(tc) - c.Assert(ops, HasLen, 0) - } - for i := 0; i < 20; i++ { - hb.(*hotScheduler).clearPendingInfluence() + ops := hb.Schedule(tc) + c.Logf("%v", ops) + c.Assert(ops, HasLen, 0) + } + for i := 0; i < 20; i++ { + hb.(*hotScheduler).clearPendingInfluence() - op1 := hb.Schedule(tc)[0] - testutil.CheckTransferLeader(c, op1, operator.OpLeader, 1, 3) - // store byte rate (min, max): (6.6, 7.1) | 6.1 | (6, 6.5) | 5 + op1 := hb.Schedule(tc)[0] + testutil.CheckTransferLeader(c, op1, operator.OpLeader, 1, 3) + // store byte/key rate (min, max): (6.6, 7.1) | 6.1 | (6, 6.5) | 5 - op2 := hb.Schedule(tc)[0] - testutil.CheckTransferPeerWithLeaderTransfer(c, op2, operator.OpHotRegion, 1, 4) - // store byte rate (min, max): (6.1, 7.1) | 6.1 | (6, 6.5) | (5, 5.5) - c.Assert(op2.Cancel(), IsTrue) - // store byte rate (min, max): (6.6, 7.1) | 6.1 | (6, 6.5) | 5 + op2 := hb.Schedule(tc)[0] + testutil.CheckTransferPeerWithLeaderTransfer(c, op2, operator.OpHotRegion, 1, 4) + // store bytekey rate (min, max): (6.1, 7.1) | 6.1 | (6, 6.5) | (5, 5.5) + c.Assert(op2.Cancel(), IsTrue) + // store byte/key rate (min, max): (6.6, 7.1) | 6.1 | (6, 6.5) | 5 - op2 = hb.Schedule(tc)[0] - testutil.CheckTransferPeerWithLeaderTransfer(c, op2, operator.OpHotRegion, 1, 4) - // store byte rate (min, max): (6.1, 7.1) | 6.1 | (6, 6.5) | (5, 5.5) + op2 = hb.Schedule(tc)[0] + testutil.CheckTransferPeerWithLeaderTransfer(c, op2, operator.OpHotRegion, 1, 4) + // store byte/key rate (min, max): (6.1, 7.1) | 6.1 | (6, 6.5) | (5, 5.5) - c.Assert(op1.Cancel(), IsTrue) - // store byte rate (min, max): (6.6, 7.1) | 6.1 | 6 | (5, 5.5) + c.Assert(op1.Cancel(), IsTrue) + // store byte/key rate (min, max): (6.6, 7.1) | 6.1 | 6 | (5, 5.5) - op3 := hb.Schedule(tc)[0] - testutil.CheckTransferPeerWithLeaderTransfer(c, op3, operator.OpHotRegion, 1, 4) - // store byte rate (min, max): (6.1, 7.1) | 6.1 | 6 | (5, 6) + op3 := hb.Schedule(tc)[0] + testutil.CheckTransferPeerWithLeaderTransfer(c, op3, operator.OpHotRegion, 1, 4) + // store byte/key rate (min, max): (6.1, 7.1) | 6.1 | 6 | (5, 6) - ops := hb.Schedule(tc) - c.Assert(ops, HasLen, 0) + ops := hb.Schedule(tc) + c.Assert(ops, HasLen, 0) + } } } From 5b6abe407eb74962755f36e23ca51222b5fd574a Mon Sep 17 00:00:00 2001 From: Luffbee Date: Thu, 20 Feb 2020 21:17:32 +0800 Subject: [PATCH 20/29] add test for key rate balance --- server/schedulers/hot_test.go | 163 ++++++++++++++++++++++++++++++++-- 1 file changed, 158 insertions(+), 5 deletions(-) diff --git a/server/schedulers/hot_test.go b/server/schedulers/hot_test.go index b8d703ab454..0b92e94a96c 100644 --- a/server/schedulers/hot_test.go +++ b/server/schedulers/hot_test.go @@ -114,7 +114,7 @@ func newTestRegion(id uint64) *core.RegionInfo { type testHotWriteRegionSchedulerSuite struct{} -func (s *testHotWriteRegionSchedulerSuite) TestSchedule(c *C) { +func (s *testHotWriteRegionSchedulerSuite) TestByteRateOnly(c *C) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() statistics.Denoising = false @@ -125,12 +125,12 @@ func (s *testHotWriteRegionSchedulerSuite) TestSchedule(c *C) { c.Assert(err, IsNil) opt.HotRegionCacheHitsThreshold = 0 - s.checkSchedule(c, tc, opt, hb) + s.checkByteRateOnly(c, tc, opt, hb) opt.EnablePlacementRules = true - s.checkSchedule(c, tc, opt, hb) + s.checkByteRateOnly(c, tc, opt, hb) } -func (s *testHotWriteRegionSchedulerSuite) checkSchedule(c *C, tc *mockcluster.Cluster, opt *mockoption.ScheduleOptions, hb schedule.Scheduler) { +func (s *testHotWriteRegionSchedulerSuite) checkByteRateOnly(c *C, tc *mockcluster.Cluster, opt *mockoption.ScheduleOptions, hb schedule.Scheduler) { // Add stores 1, 2, 3, 4, 5, 6 with region counts 3, 2, 2, 2, 0, 0. tc.AddLabelsStore(1, 3, map[string]string{"zone": "z1", "host": "h1"}) @@ -291,6 +291,103 @@ func (s *testHotWriteRegionSchedulerSuite) checkSchedule(c *C, tc *mockcluster.C hb.(*hotScheduler).clearPendingInfluence() } +func (s *testHotWriteRegionSchedulerSuite) TestWithKeyRate(c *C) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + statistics.Denoising = false + opt := mockoption.NewScheduleOptions() + hb, err := schedule.CreateScheduler(HotWriteRegionType, schedule.NewOperatorController(ctx, nil, nil), core.NewStorage(kv.NewMemoryKV()), nil) + c.Assert(err, IsNil) + opt.HotRegionCacheHitsThreshold = 0 + + tc := mockcluster.NewCluster(opt) + tc.AddRegionStore(1, 20) + tc.AddRegionStore(2, 20) + tc.AddRegionStore(3, 20) + tc.AddRegionStore(4, 20) + tc.AddRegionStore(5, 20) + + tc.UpdateStorageWrittenBytes(1, 10.5*MB*statistics.StoreHeartBeatReportInterval) + tc.UpdateStorageWrittenBytes(2, 9.5*MB*statistics.StoreHeartBeatReportInterval) + tc.UpdateStorageWrittenBytes(3, 9.5*MB*statistics.StoreHeartBeatReportInterval) + tc.UpdateStorageWrittenBytes(4, 9*MB*statistics.StoreHeartBeatReportInterval) + tc.UpdateStorageWrittenBytes(5, 8.9*MB*statistics.StoreHeartBeatReportInterval) + + tc.UpdateStorageWrittenKeys(1, 10*MB*statistics.StoreHeartBeatReportInterval) + tc.UpdateStorageWrittenKeys(2, 9.5*MB*statistics.StoreHeartBeatReportInterval) + tc.UpdateStorageWrittenKeys(3, 9.8*MB*statistics.StoreHeartBeatReportInterval) + tc.UpdateStorageWrittenKeys(4, 9*MB*statistics.StoreHeartBeatReportInterval) + tc.UpdateStorageWrittenKeys(5, 9.2*MB*statistics.StoreHeartBeatReportInterval) + + addRegionInfo(tc, write, []testRegionInfo{ + {1, []uint64{2, 1, 3}, 0.5 * MB, 0.5 * MB}, + {2, []uint64{2, 1, 3}, 0.5 * MB, 0.5 * MB}, + {3, []uint64{2, 4, 3}, 0.05 * MB, 0.1 * MB}, + }) + + for i := 0; i < 100; i++ { + hb.(*hotScheduler).clearPendingInfluence() + op := hb.Schedule(tc)[0] + // byteDecRatio <= 0.95 && keyDecRatio <= 0.95 + testutil.CheckTransferPeer(c, op, operator.OpHotRegion, 1, 4) + // store byte rate (min, max): (10, 10.5) | 9.5 | 9.5 | (9, 9.5) | 8.9 + // store key rate (min, max): (9.5, 10) | 9.5 | 9.8 | (9, 9.5) | 9.2 + + op = hb.Schedule(tc)[0] + // byteDecRatio <= 0.99 && keyDecRatio <= 0.95 + testutil.CheckTransferPeer(c, op, operator.OpHotRegion, 3, 5) + // store byte rate (min, max): (10, 10.5) | 9.5 | (9.45, 9.5) | (9, 9.5) | (8.9, 8.95) + // store key rate (min, max): (9.5, 10) | 9.5 | (9.7, 9.8) | (9, 9.5) | (9.2, 9.3) + + op = hb.Schedule(tc)[0] + // byteDecRatio <= 0.95 + testutil.CheckTransferPeer(c, op, operator.OpHotRegion, 1, 5) + // store byte rate (min, max): (9.5, 10.5) | 9.5 | (9.45, 9.5) | (9, 9.5) | (8.9, 9.45) + // store key rate (min, max): (9, 10) | 9.5 | (9.7, 9.8) | (9, 9.5) | (9.2, 9.8) + } +} + +func (s *testHotWriteRegionSchedulerSuite) TestLeader(c *C) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + statistics.Denoising = false + opt := mockoption.NewScheduleOptions() + hb, err := schedule.CreateScheduler(HotWriteRegionType, schedule.NewOperatorController(ctx, nil, nil), core.NewStorage(kv.NewMemoryKV()), nil) + c.Assert(err, IsNil) + opt.HotRegionCacheHitsThreshold = 0 + + tc := mockcluster.NewCluster(opt) + tc.AddRegionStore(1, 20) + tc.AddRegionStore(2, 20) + tc.AddRegionStore(3, 20) + + tc.UpdateStorageWrittenBytes(1, 10*MB*statistics.StoreHeartBeatReportInterval) + tc.UpdateStorageWrittenBytes(2, 10*MB*statistics.StoreHeartBeatReportInterval) + tc.UpdateStorageWrittenBytes(3, 10*MB*statistics.StoreHeartBeatReportInterval) + + tc.UpdateStorageWrittenKeys(1, 10*MB*statistics.StoreHeartBeatReportInterval) + tc.UpdateStorageWrittenKeys(2, 10*MB*statistics.StoreHeartBeatReportInterval) + tc.UpdateStorageWrittenKeys(3, 10*MB*statistics.StoreHeartBeatReportInterval) + + addRegionInfo(tc, write, []testRegionInfo{ + {1, []uint64{1, 2, 3}, 0.5 * MB, 1 * MB}, + {2, []uint64{1, 2, 3}, 0.5 * MB, 1 * MB}, + {3, []uint64{2, 1, 3}, 0.5 * MB, 1 * MB}, + {4, []uint64{2, 1, 3}, 0.5 * MB, 1 * MB}, + {5, []uint64{2, 1, 3}, 0.5 * MB, 1 * MB}, + {6, []uint64{3, 1, 2}, 0.5 * MB, 1 * MB}, + {7, []uint64{3, 1, 2}, 0.5 * MB, 1 * MB}, + }) + + for i := 0; i < 100; i++ { + hb.(*hotScheduler).clearPendingInfluence() + op := hb.Schedule(tc)[0] + testutil.CheckTransferLeaderFrom(c, op, operator.OpHotRegion, 2) + + c.Assert(hb.Schedule(tc), HasLen, 0) + } +} + func (s *testHotWriteRegionSchedulerSuite) TestWithPendingInfluence(c *C) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -379,7 +476,7 @@ var _ = Suite(&testHotReadRegionSchedulerSuite{}) type testHotReadRegionSchedulerSuite struct{} -func (s *testHotReadRegionSchedulerSuite) TestSchedule(c *C) { +func (s *testHotReadRegionSchedulerSuite) TestByteRateOnly(c *C) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() opt := mockoption.NewScheduleOptions() @@ -482,6 +579,62 @@ func (s *testHotReadRegionSchedulerSuite) TestSchedule(c *C) { hb.(*hotScheduler).clearPendingInfluence() } +func (s *testHotReadRegionSchedulerSuite) TestWithKeyRate(c *C) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + statistics.Denoising = false + opt := mockoption.NewScheduleOptions() + hb, err := schedule.CreateScheduler(HotReadRegionType, schedule.NewOperatorController(ctx, nil, nil), core.NewStorage(kv.NewMemoryKV()), nil) + c.Assert(err, IsNil) + opt.HotRegionCacheHitsThreshold = 0 + + tc := mockcluster.NewCluster(opt) + tc.AddRegionStore(1, 20) + tc.AddRegionStore(2, 20) + tc.AddRegionStore(3, 20) + tc.AddRegionStore(4, 20) + tc.AddRegionStore(5, 20) + + tc.UpdateStorageReadBytes(1, 10.5*MB*statistics.StoreHeartBeatReportInterval) + tc.UpdateStorageReadBytes(2, 9.5*MB*statistics.StoreHeartBeatReportInterval) + tc.UpdateStorageReadBytes(3, 9.5*MB*statistics.StoreHeartBeatReportInterval) + tc.UpdateStorageReadBytes(4, 9*MB*statistics.StoreHeartBeatReportInterval) + tc.UpdateStorageReadBytes(5, 8.9*MB*statistics.StoreHeartBeatReportInterval) + + tc.UpdateStorageReadKeys(1, 10*MB*statistics.StoreHeartBeatReportInterval) + tc.UpdateStorageReadKeys(2, 9.5*MB*statistics.StoreHeartBeatReportInterval) + tc.UpdateStorageReadKeys(3, 9.8*MB*statistics.StoreHeartBeatReportInterval) + tc.UpdateStorageReadKeys(4, 9*MB*statistics.StoreHeartBeatReportInterval) + tc.UpdateStorageReadKeys(5, 9.2*MB*statistics.StoreHeartBeatReportInterval) + + addRegionInfo(tc, read, []testRegionInfo{ + {1, []uint64{1, 2, 4}, 0.5 * MB, 0.5 * MB}, + {2, []uint64{1, 2, 4}, 0.5 * MB, 0.5 * MB}, + {3, []uint64{3, 4, 5}, 0.05 * MB, 0.1 * MB}, + }) + + for i := 0; i < 100; i++ { + hb.(*hotScheduler).clearPendingInfluence() + op := hb.Schedule(tc)[0] + // byteDecRatio <= 0.95 && keyDecRatio <= 0.95 + testutil.CheckTransferLeader(c, op, operator.OpHotRegion, 1, 4) + // store byte rate (min, max): (10, 10.5) | 9.5 | 9.5 | (9, 9.5) | 8.9 + // store key rate (min, max): (9.5, 10) | 9.5 | 9.8 | (9, 9.5) | 9.2 + + op = hb.Schedule(tc)[0] + // byteDecRatio <= 0.99 && keyDecRatio <= 0.95 + testutil.CheckTransferLeader(c, op, operator.OpHotRegion, 3, 5) + // store byte rate (min, max): (10, 10.5) | 9.5 | (9.45, 9.5) | (9, 9.5) | (8.9, 8.95) + // store key rate (min, max): (9.5, 10) | 9.5 | (9.7, 9.8) | (9, 9.5) | (9.2, 9.3) + + op = hb.Schedule(tc)[0] + // byteDecRatio <= 0.95 + testutil.CheckTransferPeerWithLeaderTransfer(c, op, operator.OpHotRegion, 1, 5) + // store byte rate (min, max): (9.5, 10.5) | 9.5 | (9.45, 9.5) | (9, 9.5) | (8.9, 9.45) + // store key rate (min, max): (9, 10) | 9.5 | (9.7, 9.8) | (9, 9.5) | (9.2, 9.8) + } +} + func (s *testHotReadRegionSchedulerSuite) TestWithPendingInfluence(c *C) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() From ef25256f2c61bf74a0c6f48874aaba6b46534cb1 Mon Sep 17 00:00:00 2001 From: Luffbee Date: Sun, 23 Feb 2020 00:12:15 +0800 Subject: [PATCH 21/29] fix typo --- server/schedulers/hot_region.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/schedulers/hot_region.go b/server/schedulers/hot_region.go index 26040d377af..8d9980f6c93 100644 --- a/server/schedulers/hot_region.go +++ b/server/schedulers/hot_region.go @@ -684,7 +684,7 @@ func (bs *balanceSolver) calcProgressiveRank() int64 { if bs.rwTy == write && bs.opTy == transferLeader { if srcLd.Count > dstLd.Count && srcLd.KeyRate >= dstLd.KeyRate+peer.GetKeyRate() { - rank -= 1 + rank = -1 } } else { keyDecRatio := (dstLd.KeyRate + peer.GetKeyRate()) / (srcLd.KeyRate + 1) From 35ec188eab75a828285b49f97203ba615fd00d88 Mon Sep 17 00:00:00 2001 From: Luffbee Date: Mon, 24 Feb 2020 17:06:39 +0800 Subject: [PATCH 22/29] make toResourceType panic with invalid arguments --- server/schedulers/hot_region.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/schedulers/hot_region.go b/server/schedulers/hot_region.go index 8d9980f6c93..b41b30dfa34 100644 --- a/server/schedulers/hot_region.go +++ b/server/schedulers/hot_region.go @@ -1063,5 +1063,5 @@ func toResourceType(rwTy rwType, opTy opType) resourceType { case read: return readLeader } - return resourceTypeLen + panic(fmt.Sprintf("invalid arguments for toResourceType: rwTy = %v, opTy = %v", rwTy, opTy)) } From a7240ad138c98709aa85ef317e545d763a1f5c2c Mon Sep 17 00:00:00 2001 From: Luffbee Date: Mon, 24 Feb 2020 17:08:20 +0800 Subject: [PATCH 23/29] remove unnecessary = nil --- server/schedulers/hot_region.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/server/schedulers/hot_region.go b/server/schedulers/hot_region.go index b41b30dfa34..1e2d1b739a0 100644 --- a/server/schedulers/hot_region.go +++ b/server/schedulers/hot_region.go @@ -465,9 +465,11 @@ func (bs *balanceSolver) solve() []*operator.Operator { return nil } bs.cur = &solution{} - var best *solution = nil - var ops []*operator.Operator = nil - var infls []Influence = nil + var ( + best *solution + ops []*operator.Operator + infls []Influence + ) for srcStoreID := range bs.filterSrcStores() { bs.cur.srcStoreID = srcStoreID From 73feca7eb252a43f94b267df202c4bf59b1636a0 Mon Sep 17 00:00:00 2001 From: Luffbee Date: Mon, 24 Feb 2020 17:10:41 +0800 Subject: [PATCH 24/29] add constants minHot{Byte,Key}Rate --- server/schedulers/hot_region.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/server/schedulers/hot_region.go b/server/schedulers/hot_region.go index 1e2d1b739a0..814b44c320f 100644 --- a/server/schedulers/hot_region.go +++ b/server/schedulers/hot_region.go @@ -68,6 +68,9 @@ const ( maxZombieDur time.Duration = statistics.StoreHeartBeatReportInterval * time.Second minRegionScheduleInterval time.Duration = statistics.StoreHeartBeatReportInterval * time.Second + + minHotByteRate = 100 + minHotKeyRate = 10 ) type hotScheduler struct { @@ -690,9 +693,9 @@ func (bs *balanceSolver) calcProgressiveRank() int64 { } } else { keyDecRatio := (dstLd.KeyRate + peer.GetKeyRate()) / (srcLd.KeyRate + 1) - keyHot := peer.GetKeyRate() >= 10 + keyHot := peer.GetKeyRate() >= minHotKeyRate byteDecRatio := (dstLd.ByteRate + peer.GetByteRate()) / (srcLd.ByteRate + 1) - byteHot := peer.GetByteRate() > 100 + byteHot := peer.GetByteRate() > minHotByteRate switch { case byteHot && byteDecRatio <= 0.95 && keyHot && keyDecRatio <= 0.95: rank = -3 From f7b7be2be62c9d00b28499945a8b37645eb287f0 Mon Sep 17 00:00:00 2001 From: Luffbee Date: Mon, 24 Feb 2020 17:33:50 +0800 Subject: [PATCH 25/29] simplify and update comments for calcProgressiveRank --- server/schedulers/hot_region.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/server/schedulers/hot_region.go b/server/schedulers/hot_region.go index 814b44c320f..3b1cf3f8013 100644 --- a/server/schedulers/hot_region.go +++ b/server/schedulers/hot_region.go @@ -386,6 +386,9 @@ type solution struct { region *core.RegionInfo dstStoreID uint64 + // progressiveRank measures the contribution for balance. + // The smaller the rank, the better this solution is. + // If rank < 0, this solution makes thing better. progressiveRank int64 } @@ -486,7 +489,7 @@ func (bs *balanceSolver) solve() []*operator.Operator { for dstStoreID := range bs.filterDstStores() { bs.cur.dstStoreID = dstStoreID - bs.cur.progressiveRank = bs.calcProgressiveRank() + bs.calcProgressiveRank() if bs.cur.progressiveRank < 0 && bs.betterThan(best) { if newOps, newInfls := bs.buildOperators(); len(newOps) > 0 { @@ -680,13 +683,16 @@ func (bs *balanceSolver) filterDstStores() map[uint64]*storeLoadDetail { return ret } -// calcProgressiveRank checks `bs.cur` is a progressive solution. -func (bs *balanceSolver) calcProgressiveRank() int64 { +// calcProgressiveRank calculates `bs.cur.progressiveRank`. +// See the comments of `solution.progressiveRank` for more about progressive rank. +func (bs *balanceSolver) calcProgressiveRank() { srcLd := bs.stLoadDetail[bs.cur.srcStoreID].LoadPred.min() dstLd := bs.stLoadDetail[bs.cur.dstStoreID].LoadPred.max() peer := bs.cur.srcPeerStat rank := int64(0) if bs.rwTy == write && bs.opTy == transferLeader { + // In this condition, CPU usage is the matter. + // Only consider about count and key rate. if srcLd.Count > dstLd.Count && srcLd.KeyRate >= dstLd.KeyRate+peer.GetKeyRate() { rank = -1 @@ -698,14 +704,17 @@ func (bs *balanceSolver) calcProgressiveRank() int64 { byteHot := peer.GetByteRate() > minHotByteRate switch { case byteHot && byteDecRatio <= 0.95 && keyHot && keyDecRatio <= 0.95: + // Both byte rate and key rate are balanced, the best choice. rank = -3 case byteDecRatio <= 0.99 && keyHot && keyDecRatio <= 0.95: + // Byte rate is not worsened, key rate is balanced. rank = -2 case byteHot && byteDecRatio <= 0.95: + // Byte rate is balanced, ignore the key rate. rank = -1 } } - return rank + bs.cur.progressiveRank = rank } // betterThan checks if `bs.cur` is a better solution than `old`. From bae99b841bcb022f72c9fb70328ea371a04ec0ef Mon Sep 17 00:00:00 2001 From: Luffbee Date: Tue, 25 Feb 2020 09:55:15 +0800 Subject: [PATCH 26/29] add constants *RankStepRatio --- server/schedulers/hot_region.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/server/schedulers/hot_region.go b/server/schedulers/hot_region.go index 3b1cf3f8013..f73246277ec 100644 --- a/server/schedulers/hot_region.go +++ b/server/schedulers/hot_region.go @@ -71,6 +71,10 @@ const ( minHotByteRate = 100 minHotKeyRate = 10 + + byteRateRankStepRatio = 0.05 + keyRateRankStepRatio = 0.05 + countRankStepRatio = 0.1 ) type hotScheduler struct { @@ -420,9 +424,9 @@ func (bs *balanceSolver) init() { } bs.rankStep = &storeLoad{ - ByteRate: maxCur.ByteRate / 20, - KeyRate: maxCur.KeyRate / 20, - Count: maxCur.Count / 10, + ByteRate: maxCur.ByteRate * byteRateRankStepRatio, + KeyRate: maxCur.KeyRate * keyRateRankStepRatio, + Count: maxCur.Count * countRankStepRatio, } } From 48e689e505e4d01ac0a382872765143bb8b92561 Mon Sep 17 00:00:00 2001 From: Luffbee Date: Tue, 25 Feb 2020 16:54:39 +0800 Subject: [PATCH 27/29] add constants: *DecRatio --- server/schedulers/hot_region.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/server/schedulers/hot_region.go b/server/schedulers/hot_region.go index f73246277ec..645a04e3f06 100644 --- a/server/schedulers/hot_region.go +++ b/server/schedulers/hot_region.go @@ -75,6 +75,9 @@ const ( byteRateRankStepRatio = 0.05 keyRateRankStepRatio = 0.05 countRankStepRatio = 0.1 + + greatDecRatio = 0.95 + minorDecRatio = 0.99 ) type hotScheduler struct { @@ -707,13 +710,13 @@ func (bs *balanceSolver) calcProgressiveRank() { byteDecRatio := (dstLd.ByteRate + peer.GetByteRate()) / (srcLd.ByteRate + 1) byteHot := peer.GetByteRate() > minHotByteRate switch { - case byteHot && byteDecRatio <= 0.95 && keyHot && keyDecRatio <= 0.95: + case byteHot && byteDecRatio <= greatDecRatio && keyHot && keyDecRatio <= greatDecRatio: // Both byte rate and key rate are balanced, the best choice. rank = -3 - case byteDecRatio <= 0.99 && keyHot && keyDecRatio <= 0.95: + case byteDecRatio <= minorDecRatio && keyHot && keyDecRatio <= greatDecRatio: // Byte rate is not worsened, key rate is balanced. rank = -2 - case byteHot && byteDecRatio <= 0.95: + case byteHot && byteDecRatio <= greatDecRatio: // Byte rate is balanced, ignore the key rate. rank = -1 } @@ -761,7 +764,7 @@ func (bs *balanceSolver) betterThan(old *solution) bool { keyRkCmp := rankCmp(bs.cur.srcPeerStat.GetKeyRate(), old.srcPeerStat.GetKeyRate(), stepRank(0, 10)) switch bs.cur.progressiveRank { - case -2: // 0.95 < byteDecRatio <= 0.99 && keyDecRatio <= 0.95 + case -2: // greatDecRatio < byteDecRatio <= minorDecRatio && keyDecRatio <= greatDecRatio if keyRkCmp != 0 { return keyRkCmp > 0 } @@ -769,12 +772,12 @@ func (bs *balanceSolver) betterThan(old *solution) bool { // prefer smaller byte rate, to reduce oscillation return byteRkCmp < 0 } - case -3: // byteDecRatio <= 0.95 && keyDecRatio <= 0.95 + case -3: // byteDecRatio <= greatDecRatio && keyDecRatio <= greatDecRatio if keyRkCmp != 0 { return keyRkCmp > 0 } fallthrough - case -1: // byteDecRatio <= 0.95 + case -1: // byteDecRatio <= greatDecRatio if byteRkCmp != 0 { // prefer region with larger byte rate, to converge faster return byteRkCmp > 0 From b0ca96076e220da4d61e3da957fbe56058a10de2 Mon Sep 17 00:00:00 2001 From: Luffbee Date: Tue, 25 Feb 2020 16:58:43 +0800 Subject: [PATCH 28/29] rename getHotPeers and add comments --- server/schedulers/hot_region.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/schedulers/hot_region.go b/server/schedulers/hot_region.go index 645a04e3f06..f159e5481e6 100644 --- a/server/schedulers/hot_region.go +++ b/server/schedulers/hot_region.go @@ -487,7 +487,7 @@ func (bs *balanceSolver) solve() []*operator.Operator { for srcStoreID := range bs.filterSrcStores() { bs.cur.srcStoreID = srcStoreID - for _, srcPeerStat := range bs.getHotPeers() { + for _, srcPeerStat := range bs.filterHotPeers() { bs.cur.srcPeerStat = srcPeerStat bs.cur.region = bs.getRegion() if bs.cur.region == nil { @@ -542,8 +542,9 @@ func (bs *balanceSolver) filterSrcStores() map[uint64]*storeLoadDetail { return ret } -func (bs *balanceSolver) getHotPeers() []*statistics.HotPeerStat { +func (bs *balanceSolver) filterHotPeers() []*statistics.HotPeerStat { ret := bs.stLoadDetail[bs.cur.srcStoreID].HotPeers + // Return at most maxPeerNum peers, to prevent balanceSolver.solve() too slow. if len(ret) <= maxPeerNum { return ret } From 460083344da44821b2e4dc1b8859ab49d4ef35e6 Mon Sep 17 00:00:00 2001 From: Luffbee Date: Wed, 26 Feb 2020 15:10:14 +0800 Subject: [PATCH 29/29] add comments for *RankStepRatio --- server/schedulers/hot_region.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/schedulers/hot_region.go b/server/schedulers/hot_region.go index f159e5481e6..de4a14fde9e 100644 --- a/server/schedulers/hot_region.go +++ b/server/schedulers/hot_region.go @@ -72,6 +72,8 @@ const ( minHotByteRate = 100 minHotKeyRate = 10 + // rank step ratio decide the step when calculate rank + // step = max current * rank step ratio byteRateRankStepRatio = 0.05 keyRateRankStepRatio = 0.05 countRankStepRatio = 0.1