From 3a746336dff66b48dfa9501163efd4c30fc4d578 Mon Sep 17 00:00:00 2001 From: Songqing Zhang Date: Tue, 5 Sep 2023 13:48:14 +0800 Subject: [PATCH] Improve host's leader number bounds when doing leader balance (#5670) Improve host bounds when doing leader balance Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> --- .gitignore | 1 + .../job/LeaderBalanceJobExecutor.cpp | 25 ++++++++++++++++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 8890b286dbc..7d9538a6057 100644 --- a/.gitignore +++ b/.gitignore @@ -54,6 +54,7 @@ core.* workspace.* .metals/ .cproject +.ycm_extra_conf.py #py *.egg-info diff --git a/src/meta/processors/job/LeaderBalanceJobExecutor.cpp b/src/meta/processors/job/LeaderBalanceJobExecutor.cpp index 80140fca653..3b9a0de31e0 100644 --- a/src/meta/processors/job/LeaderBalanceJobExecutor.cpp +++ b/src/meta/processors/job/LeaderBalanceJobExecutor.cpp @@ -349,9 +349,28 @@ ErrorOr LeaderBalanceJobExecutor::buildLeaderBala if (dependentOnZone) { for (auto it = allHostParts.begin(); it != allHostParts.end(); it++) { - auto min = it->second.size() / replicaFactor; - LOG(INFO) << "Host: " << it->first << " Bounds: " << min << " : " << min + 1; - hostBounds_[it->first] = std::make_pair(min, min + 1); + size_t localParts = it->second.size(); + size_t localAvg = localParts / replicaFactor; + size_t localMin = localAvg; + size_t localMax = localAvg; + if (localParts % replicaFactor != 0) { + localMax += 1; + } + + if (useDeviation) { + size_t localMinTmp = std::ceil(static_cast(localParts) / replicaFactor * + (1 - FLAGS_leader_balance_deviation)); + size_t localMaxTmp = std::floor(static_cast(localParts) / replicaFactor * + (1 + FLAGS_leader_balance_deviation)); + if (localMinTmp <= localMaxTmp) { + localMin = localMinTmp; + localMax = localMaxTmp; + } + } + LOG(INFO) << "Host:" << it->first << "'s leader balance plan, expected min load: " << localMin + << ", max load: " << localMax << " avg: " << localAvg; + + hostBounds_[it->first] = std::make_pair(localMin, localMax); } } else { size_t activeSize = activeHosts.size();