Skip to content

Commit

Permalink
HBASE-22686 ZkSplitLogWorkerCoordination doesn't allow a regionserver…
Browse files Browse the repository at this point in the history
… to pick up all of the split work it is capable of
  • Loading branch information
apurtell committed Jul 12, 2019
1 parent 438bf32 commit 4106107
Showing 1 changed file with 3 additions and 28 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -324,34 +324,10 @@ public boolean progress() {
}

/**
* This function calculates how many splitters this RS should create based on expected average
* tasks per RS and the hard limit upper bound(maxConcurrentTasks) set by configuration. <br>
* At any given time, a RS allows spawn MIN(Expected Tasks/RS, Hard Upper Bound)
* @param numTasks total number of split tasks available
* @return number of tasks this RS can grab
*/
private int getNumExpectedTasksPerRS(int numTasks) {
// at lease one RS(itself) available
int availableRSs = 1;
try {
List<String> regionServers =
ZKUtil.listChildrenNoWatch(watcher, watcher.getZNodePaths().rsZNode);
availableRSs = Math.max(availableRSs, (regionServers == null) ? 0 : regionServers.size());
} catch (KeeperException e) {
// do nothing
LOG.debug("getAvailableRegionServers got ZooKeeper exception", e);
}
int expectedTasksPerRS = (numTasks / availableRSs) + ((numTasks % availableRSs == 0) ? 0 : 1);
return Math.max(1, expectedTasksPerRS); // at least be one
}

/**
* @param expectedTasksPerRS Average number of tasks to be handled by each RS
* @return true if more splitters are available, otherwise false.
*/
private boolean areSplittersAvailable(int expectedTasksPerRS) {
return (Math.min(expectedTasksPerRS, maxConcurrentTasks)
- this.tasksInProgress.get()) > 0;
private boolean areSplittersAvailable() {
return maxConcurrentTasks - tasksInProgress.get() > 0;
}

/**
Expand Down Expand Up @@ -432,11 +408,10 @@ public void taskLoop() throws InterruptedException {
}
}
int numTasks = paths.size();
int expectedTasksPerRS = getNumExpectedTasksPerRS(numTasks);
boolean taskGrabbed = false;
for (int i = 0; i < numTasks; i++) {
while (!shouldStop) {
if (this.areSplittersAvailable(expectedTasksPerRS)) {
if (this.areSplittersAvailable()) {
LOG.debug("Current region server " + server.getServerName()
+ " is ready to take more tasks, will get task list and try grab tasks again.");
int idx = (i + offset) % paths.size();
Expand Down

0 comments on commit 4106107

Please sign in to comment.