From c81748781da4b7173eeb95bc2cb361c00805439d Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Thu, 22 Apr 2021 11:36:25 +0800 Subject: [PATCH] HBASE-25757 Move BaseLoadBalancer to hbase-balancer module --- .../hadoop/hbase/master/LoadBalancer.java | 5 +- .../master/balancer/BaseLoadBalancer.java | 59 ++++++------------- .../master/balancer/CandidateGenerator.java | 30 ++++------ .../master/balancer/ClusterInfoProvider.java | 26 ++++++++ .../balancer/LoadCandidateGenerator.java | 0 .../LocalityBasedCandidateGenerator.java | 14 ++--- .../balancer/RandomCandidateGenerator.java | 35 +++++++++++ .../RegionReplicaCandidateGenerator.java | 28 ++++----- .../RegionReplicaRackCandidateGenerator.java | 53 +++++++++++++++++ .../master/balancer/SimpleLoadBalancer.java | 0 .../TestRegionHDFSBlockLocationFinder.java | 25 ++++++++ .../favored/FavoredNodeLoadBalancer.java | 19 +++--- .../balancer/FavoredStochasticBalancer.java | 28 +++++---- .../balancer/MaintenanceLoadBalancer.java | 4 +- .../balancer/MasterClusterInfoProvider.java | 44 +++++++++++++- .../balancer/StochasticLoadBalancer.java | 56 ++---------------- .../rsgroup/RSGroupBasedLoadBalancer.java | 27 +++++++-- .../hbase/master/TestRegionPlacement2.java | 6 +- .../master/balancer/TestBaseLoadBalancer.java | 21 +++---- ...ochasticLoadBalancerHeterogeneousCost.java | 2 +- 20 files changed, 302 insertions(+), 180 deletions(-) rename {hbase-server => hbase-balancer}/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java (96%) rename {hbase-server => hbase-balancer}/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java (91%) rename {hbase-server => hbase-balancer}/src/main/java/org/apache/hadoop/hbase/master/balancer/CandidateGenerator.java (84%) rename {hbase-server => hbase-balancer}/src/main/java/org/apache/hadoop/hbase/master/balancer/LoadCandidateGenerator.java (100%) rename {hbase-server => hbase-balancer}/src/main/java/org/apache/hadoop/hbase/master/balancer/LocalityBasedCandidateGenerator.java (91%) create mode 100644 hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/RandomCandidateGenerator.java rename {hbase-server => hbase-balancer}/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionReplicaCandidateGenerator.java (84%) create mode 100644 hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionReplicaRackCandidateGenerator.java rename {hbase-server => hbase-balancer}/src/main/java/org/apache/hadoop/hbase/master/balancer/SimpleLoadBalancer.java (100%) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java similarity index 96% rename from hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java rename to hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java index b49ca824236a..8b37da1dea8c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java +++ b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.conf.ConfigurationObserver; +import org.apache.hadoop.hbase.master.balancer.ClusterInfoProvider; import org.apache.yetus.audience.InterfaceAudience; /** @@ -69,9 +70,9 @@ public interface LoadBalancer extends Configurable, Stoppable, ConfigurationObse /** - * Set the master service. + * Set the cluster info provider. Usually it is just a wrapper of master. */ - void setMasterServices(MasterServices masterServices); + void setClusterInfoProvider(ClusterInfoProvider provider); /** * Perform the major balance operation for cluster, will invoke {@link #balanceTable} to do actual diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java similarity index 91% rename from hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java rename to hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java index 4374de1ed549..35053e1d8fb7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java +++ b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java @@ -40,9 +40,7 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.RegionInfo; -import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.master.LoadBalancer; -import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.RackManager; import org.apache.hadoop.hbase.master.RegionPlan; import org.apache.yetus.audience.InterfaceAudience; @@ -63,6 +61,8 @@ */ @InterfaceAudience.Private public abstract class BaseLoadBalancer implements LoadBalancer { + + private static final Logger LOG = LoggerFactory.getLogger(BaseLoadBalancer.class); public static final String BALANCER_DECISION_BUFFER_ENABLED = "hbase.master.balancer.decision.buffer.enabled"; @@ -71,9 +71,7 @@ public abstract class BaseLoadBalancer implements LoadBalancer { protected static final int MIN_SERVER_BALANCE = 2; private volatile boolean stopped = false; - static final List EMPTY_REGION_LIST = Collections.emptyList(); - - static final Predicate IDLE_SERVER_PREDICATOR + private static final Predicate IDLE_SERVER_PREDICATOR = load -> load.getRegionMetrics().isEmpty(); protected RegionHDFSBlockLocationFinder regionFinder; @@ -110,11 +108,9 @@ private void createRegionFinder() { protected float overallSlop; protected Configuration config = HBaseConfiguration.create(); protected RackManager rackManager; - static final Logger LOG = LoggerFactory.getLogger(BaseLoadBalancer.class); protected MetricsBalancer metricsBalancer = null; protected ClusterMetrics clusterStatus = null; - protected ServerName masterServerName; - protected MasterServices services; + protected ClusterInfoProvider provider; @Override public void setConf(Configuration conf) { @@ -161,24 +157,17 @@ public synchronized void setClusterMetrics(ClusterMetrics st) { @Override - public void setMasterServices(MasterServices masterServices) { - masterServerName = masterServices.getServerName(); - this.services = masterServices; + public void setClusterInfoProvider(ClusterInfoProvider provider) { + this.provider = provider; if (useRegionFinder) { - this.regionFinder.setClusterInfoProvider(new MasterClusterInfoProvider(services)); + this.regionFinder.setClusterInfoProvider(provider); } } @Override public void postMasterStartupInitialize() { - if (services != null && regionFinder != null) { - try { - Set regions = - services.getAssignmentManager().getRegionStates().getRegionAssignments().keySet(); - regionFinder.refreshAndWait(regions); - } catch (Exception e) { - LOG.warn("Refreshing region HDFS Block dist failed with exception, ignoring", e); - } + if (provider != null && regionFinder != null) { + regionFinder.refreshAndWait(provider.getAssignedRegions()); } } @@ -288,19 +277,12 @@ public Map> roundRobinAssignment(List r return assignments; } - protected BalancerClusterState createCluster(List servers, + private BalancerClusterState createCluster(List servers, Collection regions) throws HBaseIOException { - boolean hasRegionReplica = false; + boolean hasRegionReplica= false; try { - if (services != null && services.getTableDescriptors() != null) { - Map tds = services.getTableDescriptors().getAll(); - for (RegionInfo regionInfo : regions) { - TableDescriptor td = tds.get(regionInfo.getTable().getNameWithNamespaceInclAsString()); - if (td != null && td.getRegionReplication() > 1) { - hasRegionReplica = true; - break; - } - } + if (provider != null) { + hasRegionReplica = provider.hasRegionReplica(regions); } } catch (IOException ioe) { throw new HBaseIOException(ioe); @@ -320,7 +302,7 @@ protected BalancerClusterState createCluster(List servers, for (ServerName server : servers) { if (!clusterState.containsKey(server)) { - clusterState.put(server, EMPTY_REGION_LIST); + clusterState.put(server, Collections.emptyList()); } } return new BalancerClusterState(regions, clusterState, null, this.regionFinder, @@ -328,8 +310,7 @@ protected BalancerClusterState createCluster(List servers, } private List findIdleServers(List servers) { - return this.services.getServerManager() - .getOnlineServersListWithPredicator(servers, IDLE_SERVER_PREDICATOR); + return provider.getOnlineServersListWithPredicator(servers, IDLE_SERVER_PREDICATOR); } /** @@ -615,13 +596,9 @@ private void roundRobinAssignment(BalancerClusterState cluster, List } } - protected Map> getRegionAssignmentsByServer( - Collection regions) { - if (this.services != null && this.services.getAssignmentManager() != null) { - return this.services.getAssignmentManager().getSnapShotOfAssignment(regions); - } else { - return new HashMap<>(); - } + private Map> + getRegionAssignmentsByServer(Collection regions) { + return provider != null ? provider.getSnapShotOfAssignment(regions) : Collections.emptyMap(); } private Map> toEnsumbleTableLoad( diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/CandidateGenerator.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/CandidateGenerator.java similarity index 84% rename from hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/CandidateGenerator.java rename to hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/CandidateGenerator.java index 06b5623d1b5a..63c747efe530 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/CandidateGenerator.java +++ b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/CandidateGenerator.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.master.balancer; import java.util.Map; +import java.util.concurrent.ThreadLocalRandom; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.yetus.audience.InterfaceAudience; @@ -34,23 +35,19 @@ abstract class CandidateGenerator { * From a list of regions pick a random one. Null can be returned which * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move * rather than swap. - * * @param cluster The state of the cluster * @param server index of the server - * @param chanceOfNoSwap Chance that this will decide to try a move rather - * than a swap. - * @return a random {@link RegionInfo} or null if an asymmetrical move is - * suggested. + * @param chanceOfNoSwap Chance that this will decide to try a move rather than a swap. + * @return a random {@link RegionInfo} or null if an asymmetrical move is suggested. */ - int pickRandomRegion(BalancerClusterState cluster, int server, - double chanceOfNoSwap) { + int pickRandomRegion(BalancerClusterState cluster, int server, double chanceOfNoSwap) { // Check to see if this is just a move. - if (cluster.regionsPerServer[server].length == 0 - || StochasticLoadBalancer.RANDOM.nextFloat() < chanceOfNoSwap) { + if (cluster.regionsPerServer[server].length == 0 || + ThreadLocalRandom.current().nextFloat() < chanceOfNoSwap) { // signal a move only. return -1; } - int rand = StochasticLoadBalancer.RANDOM.nextInt(cluster.regionsPerServer[server].length); + int rand = ThreadLocalRandom.current().nextInt(cluster.regionsPerServer[server].length); return cluster.regionsPerServer[server][rand]; } @@ -59,7 +56,7 @@ int pickRandomServer(BalancerClusterState cluster) { return -1; } - return StochasticLoadBalancer.RANDOM.nextInt(cluster.numServers); + return ThreadLocalRandom.current().nextInt(cluster.numServers); } int pickRandomRack(BalancerClusterState cluster) { @@ -67,7 +64,7 @@ int pickRandomRack(BalancerClusterState cluster) { return -1; } - return StochasticLoadBalancer.RANDOM.nextInt(cluster.numRacks); + return ThreadLocalRandom.current().nextInt(cluster.numRacks); } int pickOtherRandomServer(BalancerClusterState cluster, int serverIndex) { @@ -94,8 +91,7 @@ int pickOtherRandomRack(BalancerClusterState cluster, int rackIndex) { } } - BalanceAction pickRandomRegions(BalancerClusterState cluster, - int thisServer, int otherServer) { + BalanceAction pickRandomRegions(BalancerClusterState cluster, int thisServer, int otherServer) { if (thisServer < 0 || otherServer < 0) { return BalanceAction.NULL_ACTION; } @@ -114,14 +110,12 @@ BalanceAction pickRandomRegions(BalancerClusterState cluster, return getAction(thisServer, thisRegion, otherServer, otherRegion); } - protected BalanceAction getAction(int fromServer, int fromRegion, - int toServer, int toRegion) { + protected BalanceAction getAction(int fromServer, int fromRegion, int toServer, int toRegion) { if (fromServer < 0 || toServer < 0) { return BalanceAction.NULL_ACTION; } if (fromRegion >= 0 && toRegion >= 0) { - return new SwapRegionsAction(fromServer, fromRegion, - toServer, toRegion); + return new SwapRegionsAction(fromServer, fromRegion, toServer, toRegion); } else if (fromRegion >= 0) { return new MoveRegionAction(fromRegion, fromServer, toServer); } else if (toRegion >= 0) { diff --git a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/ClusterInfoProvider.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/ClusterInfoProvider.java index 47247b144e0a..0686cf8a0df7 100644 --- a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/ClusterInfoProvider.java +++ b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/ClusterInfoProvider.java @@ -18,9 +18,14 @@ package org.apache.hadoop.hbase.master.balancer; import java.io.IOException; +import java.util.Collection; import java.util.List; +import java.util.Map; +import java.util.function.Predicate; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HDFSBlocksDistribution; +import org.apache.hadoop.hbase.ServerMetrics; +import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.TableDescriptor; @@ -45,6 +50,11 @@ public interface ClusterInfoProvider { */ TableDescriptor getTableDescriptor(TableName tableName) throws IOException; + /** + * Returns the number of tables on this cluster. + */ + int getNumberOfTables() throws IOException; + /** * Compute the block distribution for the given region. *

@@ -52,4 +62,20 @@ public interface ClusterInfoProvider { */ HDFSBlocksDistribution computeHDFSBlocksDistribution(Configuration conf, TableDescriptor tableDescriptor, RegionInfo regionInfo) throws IOException; + + /** + * Check whether we have region replicas enabled for the tables of the given regions. + */ + boolean hasRegionReplica(Collection regions) throws IOException; + + /** + * Returns a copy of the internal list of online servers matched by the given {@code filter}. + */ + List getOnlineServersListWithPredicator(List servers, + Predicate filter); + + /** + * Get a snapshot of the current assignment status. + */ + Map> getSnapShotOfAssignment(Collection regions); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/LoadCandidateGenerator.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/LoadCandidateGenerator.java similarity index 100% rename from hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/LoadCandidateGenerator.java rename to hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/LoadCandidateGenerator.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/LocalityBasedCandidateGenerator.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/LocalityBasedCandidateGenerator.java similarity index 91% rename from hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/LocalityBasedCandidateGenerator.java rename to hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/LocalityBasedCandidateGenerator.java index 9da884f40d0d..c8e56f193bf0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/LocalityBasedCandidateGenerator.java +++ b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/LocalityBasedCandidateGenerator.java @@ -36,8 +36,8 @@ BalanceAction generate(BalancerClusterState cluster) { int currentServer = cluster.regionIndexToServerIndex[region]; if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities( BalancerClusterState.LocalityType.SERVER)[region]) { - Optional potential = tryMoveOrSwap(cluster, - currentServer, region, cluster.getOrComputeRegionsToMostLocalEntities( + Optional potential = tryMoveOrSwap(cluster, currentServer, region, + cluster.getOrComputeRegionsToMostLocalEntities( BalancerClusterState.LocalityType.SERVER)[region]); if (potential.isPresent()) { return potential.get(); @@ -48,16 +48,16 @@ BalanceAction generate(BalancerClusterState cluster) { return BalanceAction.NULL_ACTION; } - private Optional tryMoveOrSwap(BalancerClusterState cluster, - int fromServer, int fromRegion, int toServer) { + private Optional tryMoveOrSwap(BalancerClusterState cluster, int fromServer, + int fromRegion, int toServer) { // Try move first. We know apriori fromRegion has the highest locality on toServer if (cluster.serverHasTooFewRegions(toServer)) { return Optional.of(getAction(fromServer, fromRegion, toServer, -1)); } // Compare locality gain/loss from swapping fromRegion with regions on toServer - double fromRegionLocalityDelta = getWeightedLocality(cluster, fromRegion, toServer) - - getWeightedLocality(cluster, fromRegion, fromServer); - int toServertotalRegions = cluster.regionsPerServer[toServer].length; + double fromRegionLocalityDelta = getWeightedLocality(cluster, fromRegion, toServer) - + getWeightedLocality(cluster, fromRegion, fromServer); + int toServertotalRegions = cluster.regionsPerServer[toServer].length; if (toServertotalRegions > 0) { int startIndex = ThreadLocalRandom.current().nextInt(toServertotalRegions); for (int i = 0; i < toServertotalRegions; i++) { diff --git a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/RandomCandidateGenerator.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/RandomCandidateGenerator.java new file mode 100644 index 000000000000..dad201152300 --- /dev/null +++ b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/RandomCandidateGenerator.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +class RandomCandidateGenerator extends CandidateGenerator { + + @Override + BalanceAction generate(BalancerClusterState cluster) { + + int thisServer = pickRandomServer(cluster); + + // Pick the other server + int otherServer = pickOtherRandomServer(cluster, thisServer); + + return pickRandomRegions(cluster, thisServer, otherServer); + } +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionReplicaCandidateGenerator.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionReplicaCandidateGenerator.java similarity index 84% rename from hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionReplicaCandidateGenerator.java rename to hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionReplicaCandidateGenerator.java index 4badd4d2902d..c5b87e83c508 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionReplicaCandidateGenerator.java +++ b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionReplicaCandidateGenerator.java @@ -18,30 +18,29 @@ package org.apache.hadoop.hbase.master.balancer; +import java.util.concurrent.ThreadLocalRandom; import org.apache.yetus.audience.InterfaceAudience; /** - * Generates candidates which moves the replicas out of the region server for - * co-hosted region replicas + * Generates candidates which moves the replicas out of the region server for co-hosted region + * replicas */ @InterfaceAudience.Private class RegionReplicaCandidateGenerator extends CandidateGenerator { - StochasticLoadBalancer.RandomCandidateGenerator randomGenerator = - new StochasticLoadBalancer.RandomCandidateGenerator(); + RandomCandidateGenerator randomGenerator = new RandomCandidateGenerator(); /** - * Randomly select one regionIndex out of all region replicas co-hosted in the same group - * (a group is a server, host or rack) - * + * Randomly select one regionIndex out of all region replicas co-hosted in the same group (a group + * is a server, host or rack) * @param primariesOfRegionsPerGroup either Cluster.primariesOfRegionsPerServer, - * primariesOfRegionsPerHost or primariesOfRegionsPerRack + * primariesOfRegionsPerHost or primariesOfRegionsPerRack * @param regionsPerGroup either Cluster.regionsPerServer, regionsPerHost or regionsPerRack * @param regionIndexToPrimaryIndex Cluster.regionsIndexToPrimaryIndex * @return a regionIndex for the selected primary or -1 if there is no co-locating */ int selectCoHostedRegionPerGroup(int[] primariesOfRegionsPerGroup, int[] regionsPerGroup, - int[] regionIndexToPrimaryIndex) { + int[] regionIndexToPrimaryIndex) { int currentPrimary = -1; int currentPrimaryIndex = -1; int selectedPrimaryIndex = -1; @@ -50,13 +49,12 @@ int selectCoHostedRegionPerGroup(int[] primariesOfRegionsPerGroup, int[] regions // ids for the regions hosted in server, a consecutive repetition means that replicas // are co-hosted for (int j = 0; j <= primariesOfRegionsPerGroup.length; j++) { - int primary = j < primariesOfRegionsPerGroup.length - ? primariesOfRegionsPerGroup[j] : -1; + int primary = j < primariesOfRegionsPerGroup.length ? primariesOfRegionsPerGroup[j] : -1; if (primary != currentPrimary) { // check for whether we see a new primary int numReplicas = j - currentPrimaryIndex; if (numReplicas > 1) { // means consecutive primaries, indicating co-location // decide to select this primary region id or not - double currentRandom = StochasticLoadBalancer.RANDOM.nextDouble(); + double currentRandom = ThreadLocalRandom.current().nextDouble(); // we don't know how many region replicas are co-hosted, we will randomly select one // using reservoir sampling (http://gregable.com/2007/10/reservoir-sampling.html) if (currentRandom > currentLargestRandom) { @@ -89,10 +87,8 @@ BalanceAction generate(BalancerClusterState cluster) { return BalanceAction.NULL_ACTION; } - int regionIndex = selectCoHostedRegionPerGroup( - cluster.primariesOfRegionsPerServer[serverIndex], - cluster.regionsPerServer[serverIndex], - cluster.regionIndexToPrimaryIndex); + int regionIndex = selectCoHostedRegionPerGroup(cluster.primariesOfRegionsPerServer[serverIndex], + cluster.regionsPerServer[serverIndex], cluster.regionIndexToPrimaryIndex); // if there are no pairs of region replicas co-hosted, default to random generator if (regionIndex == -1) { diff --git a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionReplicaRackCandidateGenerator.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionReplicaRackCandidateGenerator.java new file mode 100644 index 000000000000..cb00f8e08aac --- /dev/null +++ b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionReplicaRackCandidateGenerator.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import java.util.concurrent.ThreadLocalRandom; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * Generates candidates which moves the replicas out of the rack for co-hosted region replicas in + * the same rack + */ +@InterfaceAudience.Private +class RegionReplicaRackCandidateGenerator extends RegionReplicaCandidateGenerator { + @Override + BalanceAction generate(BalancerClusterState cluster) { + int rackIndex = pickRandomRack(cluster); + if (cluster.numRacks <= 1 || rackIndex == -1) { + return super.generate(cluster); + } + + int regionIndex = selectCoHostedRegionPerGroup(cluster.primariesOfRegionsPerRack[rackIndex], + cluster.regionsPerRack[rackIndex], cluster.regionIndexToPrimaryIndex); + + // if there are no pairs of region replicas co-hosted, default to random generator + if (regionIndex == -1) { + // default to randompicker + return randomGenerator.generate(cluster); + } + + int serverIndex = cluster.regionIndexToServerIndex[regionIndex]; + int toRackIndex = pickOtherRandomRack(cluster, rackIndex); + + int rand = ThreadLocalRandom.current().nextInt(cluster.serversPerRack[toRackIndex].length); + int toServerIndex = cluster.serversPerRack[toRackIndex][rand]; + int toRegionIndex = pickRandomRegion(cluster, toServerIndex, 0.9f); + return getAction(serverIndex, regionIndex, toServerIndex, toRegionIndex); + } +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SimpleLoadBalancer.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/SimpleLoadBalancer.java similarity index 100% rename from hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SimpleLoadBalancer.java rename to hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/SimpleLoadBalancer.java diff --git a/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRegionHDFSBlockLocationFinder.java b/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRegionHDFSBlockLocationFinder.java index 41d420b5db17..f51764fd39f0 100644 --- a/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRegionHDFSBlockLocationFinder.java +++ b/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRegionHDFSBlockLocationFinder.java @@ -27,10 +27,13 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; +import java.util.function.Predicate; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.ClusterMetrics; import org.apache.hadoop.hbase.HBaseClassTestRule; @@ -111,6 +114,28 @@ public HDFSBlocksDistribution computeHDFSBlocksDistribution(Configuration conf, TableDescriptor tableDescriptor, RegionInfo regionInfo) throws IOException { return generate(regionInfo); } + + @Override + public boolean hasRegionReplica(Collection regions) throws IOException { + return false; + } + + @Override + public List getOnlineServersListWithPredicator(List servers, + Predicate filter) { + return Collections.emptyList(); + } + + @Override + public Map> + getSnapShotOfAssignment(Collection regions) { + return Collections.emptyMap(); + } + + @Override + public int getNumberOfTables() { + return 0; + } }); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/favored/FavoredNodeLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/favored/FavoredNodeLoadBalancer.java index 60a2c6cae13f..77fa60ecef14 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/favored/FavoredNodeLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/favored/FavoredNodeLoadBalancer.java @@ -29,7 +29,6 @@ import java.util.List; import java.util.Map; import java.util.Set; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.HBaseInterfaceAudience; @@ -38,6 +37,7 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.favored.FavoredNodesPlan.Position; +import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.RackManager; import org.apache.hadoop.hbase.master.RegionPlan; import org.apache.hadoop.hbase.master.ServerManager; @@ -70,6 +70,7 @@ public class FavoredNodeLoadBalancer extends BaseLoadBalancer implements FavoredNodesPromoter { private static final Logger LOG = LoggerFactory.getLogger(FavoredNodeLoadBalancer.class); + private MasterServices services; private RackManager rackManager; private Configuration conf; private FavoredNodesManager fnm; @@ -79,6 +80,10 @@ public void setConf(Configuration conf) { this.conf = conf; } + public void setMasterServices(MasterServices services) { + this.services = services; + } + @Override public synchronized void initialize() throws HBaseIOException { super.initialize(); @@ -95,7 +100,7 @@ public List balanceTable(TableName tableName, List plans = new ArrayList<>(); // perform a scan of the meta to get the latest updates (if any) SnapshotOfRegionAssignmentFromMeta snaphotOfRegionAssignment = - new SnapshotOfRegionAssignmentFromMeta(super.services.getConnection()); + new SnapshotOfRegionAssignmentFromMeta(services.getConnection()); try { snaphotOfRegionAssignment.initialize(); } catch (IOException ie) { @@ -105,7 +110,7 @@ public List balanceTable(TableName tableName, // This is not used? Findbugs says so: Map // serverNameToServerNameWithoutCode = new HashMap<>(); Map serverNameWithoutCodeToServerName = new HashMap<>(); - ServerManager serverMgr = super.services.getServerManager(); + ServerManager serverMgr = services.getServerManager(); for (ServerName sn : serverMgr.getOnlineServersList()) { ServerName s = ServerName.valueOf(sn.getHostname(), sn.getPort(), ServerName.NON_STARTCODE); // FindBugs complains about useless store! serverNameToServerNameWithoutCode.put(sn, s); @@ -136,9 +141,9 @@ public List balanceTable(TableName tableName, } // the region is currently on none of the favored nodes // get it on one of them if possible - ServerMetrics l1 = super.services.getServerManager() + ServerMetrics l1 = services.getServerManager() .getLoad(serverNameWithoutCodeToServerName.get(favoredNodes.get(1))); - ServerMetrics l2 = super.services.getServerManager() + ServerMetrics l2 = services.getServerManager() .getLoad(serverNameWithoutCodeToServerName.get(favoredNodes.get(2))); if (l1 != null && l2 != null) { if (l1.getRegionMetrics().size() > l2.getRegionMetrics().size()) { @@ -301,8 +306,8 @@ private void assignRegionToAvailableFavoredNode(Map serve List onlineServers = getOnlineFavoredNodes(servers, favoredNodes); if (onlineServers.size() > 0) { - destination = onlineServers.get(RANDOM.nextInt(onlineServers.size())); + destination = onlineServers.get(ThreadLocalRandom.current().nextInt(onlineServers.size())); } boolean alwaysAssign = getConf().getBoolean(FAVORED_ALWAYS_ASSIGN_REGIONS, true); @@ -398,9 +399,10 @@ public Map> retainAssignment(Map regions) throws IOException { + TableDescriptors tds = services.getTableDescriptors(); + if (tds == null) { + return false; + } + for (RegionInfo region : regions) { + TableDescriptor td = tds.get(region.getTable()); + if (td != null && td.getRegionReplication() > 1) { + return true; + } + } + return false; + } + + @Override + public List getOnlineServersListWithPredicator(List servers, + Predicate filter) { + ServerManager sm = services.getServerManager(); + return sm != null ? sm.getOnlineServersListWithPredicator(servers, filter) : + Collections.emptyList(); + } + + @Override + public Map> getSnapShotOfAssignment(Collection regions) { + AssignmentManager am = services.getAssignmentManager(); + return am != null ? am.getSnapShotOfAssignment(regions) : Collections.emptyMap(); + } + + @Override + public int getNumberOfTables() throws IOException { + return services.getTableDescriptors().getAll().size(); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java index c0ab4d9e9a9f..e21eeb522130 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java @@ -27,7 +27,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Random; +import java.util.concurrent.ThreadLocalRandom; import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.ClusterMetrics; @@ -122,7 +122,6 @@ public class StochasticLoadBalancer extends BaseLoadBalancer { protected static final String COST_FUNCTIONS_COST_FUNCTIONS_KEY = "hbase.master.balancer.stochastic.additionalCostFunctions"; - protected static final Random RANDOM = new Random(System.currentTimeMillis()); private static final Logger LOG = LoggerFactory.getLogger(StochasticLoadBalancer.class); Map> loads = new HashMap<>(); @@ -278,14 +277,14 @@ protected void setSlop(Configuration conf) { public synchronized void setClusterMetrics(ClusterMetrics st) { super.setClusterMetrics(st); updateRegionLoad(); - for(CostFromRegionLoadFunction cost : regionLoadFunctions) { + for (CostFromRegionLoadFunction cost : regionLoadFunctions) { cost.setClusterMetrics(st); } // update metrics size try { // by-table or ensemble mode - int tablesCount = isByTable ? services.getTableDescriptors().getAll().size() : 1; + int tablesCount = isByTable ? provider.getNumberOfTables() : 1; int functionsCount = getCostFunctionNames().length; updateMetricsSize(tablesCount * (functionsCount + 1)); // +1 for overall @@ -365,7 +364,7 @@ protected boolean needsBalance(TableName tableName, BalancerClusterState cluster } BalanceAction nextAction(BalancerClusterState cluster) { - return candidateGenerators.get(RANDOM.nextInt(candidateGenerators.size())) + return candidateGenerators.get(ThreadLocalRandom.current().nextInt(candidateGenerators.size())) .generate(cluster); } @@ -683,53 +682,6 @@ protected double computeCost(BalancerClusterState cluster, double previousCost) return total; } - static class RandomCandidateGenerator extends CandidateGenerator { - - @Override - BalanceAction generate(BalancerClusterState cluster) { - - int thisServer = pickRandomServer(cluster); - - // Pick the other server - int otherServer = pickOtherRandomServer(cluster, thisServer); - - return pickRandomRegions(cluster, thisServer, otherServer); - } - } - - /** - * Generates candidates which moves the replicas out of the rack for - * co-hosted region replicas in the same rack - */ - static class RegionReplicaRackCandidateGenerator extends RegionReplicaCandidateGenerator { - @Override - BalanceAction generate(BalancerClusterState cluster) { - int rackIndex = pickRandomRack(cluster); - if (cluster.numRacks <= 1 || rackIndex == -1) { - return super.generate(cluster); - } - - int regionIndex = selectCoHostedRegionPerGroup( - cluster.primariesOfRegionsPerRack[rackIndex], - cluster.regionsPerRack[rackIndex], - cluster.regionIndexToPrimaryIndex); - - // if there are no pairs of region replicas co-hosted, default to random generator - if (regionIndex == -1) { - // default to randompicker - return randomGenerator.generate(cluster); - } - - int serverIndex = cluster.regionIndexToServerIndex[regionIndex]; - int toRackIndex = pickOtherRandomRack(cluster, rackIndex); - - int rand = RANDOM.nextInt(cluster.serversPerRack[toRackIndex].length); - int toServerIndex = cluster.serversPerRack[toRackIndex][rand]; - int toRegionIndex = pickRandomRegion(cluster, toServerIndex, 0.9f); - return getAction(serverIndex, regionIndex, toServerIndex, toRegionIndex); - } - } - /** * Base class of StochasticLoadBalancer's Cost Functions. */ diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java index 89db7695fe21..4c7c5ddfce17 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java @@ -34,12 +34,16 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.constraint.ConstraintException; +import org.apache.hadoop.hbase.favored.FavoredNodeLoadBalancer; import org.apache.hadoop.hbase.favored.FavoredNodesManager; import org.apache.hadoop.hbase.favored.FavoredNodesPromoter; import org.apache.hadoop.hbase.master.LoadBalancer; import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.RegionPlan; +import org.apache.hadoop.hbase.master.balancer.ClusterInfoProvider; +import org.apache.hadoop.hbase.master.balancer.FavoredStochasticBalancer; import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory; +import org.apache.hadoop.hbase.master.balancer.MasterClusterInfoProvider; import org.apache.hadoop.hbase.net.Address; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.ReflectionUtils; @@ -114,7 +118,6 @@ public void setClusterMetrics(ClusterMetrics sm) { } } - @Override public void setMasterServices(MasterServices masterServices) { this.masterServices = masterServices; } @@ -362,14 +365,23 @@ public void initialize() throws IOException { balancerClass = LoadBalancerFactory.getDefaultLoadBalancerClass(); } internalBalancer = ReflectionUtils.newInstance(balancerClass); - if (internalBalancer instanceof FavoredNodesPromoter) { - favoredNodesManager = new FavoredNodesManager(masterServices); - } internalBalancer.setConf(config); - internalBalancer.setMasterServices(masterServices); + internalBalancer.setClusterInfoProvider(new MasterClusterInfoProvider(masterServices)); if(clusterStatus != null) { internalBalancer.setClusterMetrics(clusterStatus); } + // special handling for favor node balancers + if (internalBalancer instanceof FavoredNodesPromoter) { + favoredNodesManager = new FavoredNodesManager(masterServices); + if (internalBalancer instanceof FavoredNodeLoadBalancer) { + ((FavoredNodeLoadBalancer) internalBalancer).setMasterServices(masterServices); + } + if (internalBalancer instanceof FavoredStochasticBalancer) { + ((FavoredStochasticBalancer) internalBalancer).setMasterServices(masterServices); + } + } + + internalBalancer.initialize(); // init fallback groups this.fallbackEnabled = config.getBoolean(FALLBACK_GROUP_ENABLE_KEY, false); @@ -479,4 +491,9 @@ private List getFallBackCandidates(List servers) { } return serverNames == null || serverNames.isEmpty() ? servers : serverNames; } + + @Override + public void setClusterInfoProvider(ClusterInfoProvider provider) { + throw new UnsupportedOperationException("Just call set master service instead"); + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement2.java index 47337f9f7c18..6f850e1017cc 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement2.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement2.java @@ -83,7 +83,8 @@ public static void tearDownAfterClass() throws Exception { @Test public void testFavoredNodesPresentForRoundRobinAssignment() throws IOException { - LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(TEST_UTIL.getConfiguration()); + FavoredNodeLoadBalancer balancer = + (FavoredNodeLoadBalancer) LoadBalancerFactory.getLoadBalancer(TEST_UTIL.getConfiguration()); balancer.setMasterServices(TEST_UTIL.getMiniHBaseCluster().getMaster()); balancer.initialize(); List servers = new ArrayList<>(); @@ -144,7 +145,8 @@ public void testFavoredNodesPresentForRoundRobinAssignment() throws IOException @Test public void testFavoredNodesPresentForRandomAssignment() throws IOException { - LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(TEST_UTIL.getConfiguration()); + FavoredNodeLoadBalancer balancer = + (FavoredNodeLoadBalancer) LoadBalancerFactory.getLoadBalancer(TEST_UTIL.getConfiguration()); balancer.setMasterServices(TEST_UTIL.getMiniHBaseCluster().getMaster()); balancer.initialize(); List servers = new ArrayList<>(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java index 98c0bddb1ce7..ded564439e02 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java @@ -32,6 +32,7 @@ import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; +import java.util.function.Predicate; import java.util.stream.Collectors; import org.apache.commons.lang3.ArrayUtils; import org.apache.hadoop.conf.Configuration; @@ -43,10 +44,8 @@ import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.master.LoadBalancer; -import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.RackManager; import org.apache.hadoop.hbase.master.RegionPlan; -import org.apache.hadoop.hbase.master.ServerManager; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; @@ -93,9 +92,8 @@ public static void beforeAllTests() throws Exception { conf.setClass("hbase.util.ip.to.rack.determiner", MockMapping.class, DNSToSwitchMapping.class); loadBalancer = new MockBalancer(); loadBalancer.setConf(conf); - MasterServices st = Mockito.mock(MasterServices.class); - Mockito.when(st.getServerName()).thenReturn(master); - loadBalancer.setMasterServices(st); + ClusterInfoProvider provider = Mockito.mock(ClusterInfoProvider.class); + loadBalancer.setClusterInfoProvider(provider); // Set up the rack topologies (5 machines per rack) rackManager = Mockito.mock(RackManager.class); @@ -242,19 +240,18 @@ private void testRandomAssignment(int numberOfIdleServers) throws Exception { Configuration conf = HBaseConfiguration.create(); conf.setClass("hbase.util.ip.to.rack.determiner", MockMapping.class, DNSToSwitchMapping.class); balancer.setConf(conf); - ServerManager sm = Mockito.mock(ServerManager.class); - Mockito.when(sm.getOnlineServersListWithPredicator(allServers, BaseLoadBalancer.IDLE_SERVER_PREDICATOR)) - .thenReturn(idleServers); - MasterServices services = Mockito.mock(MasterServices.class); - Mockito.when(services.getServerManager()).thenReturn(sm); - balancer.setMasterServices(services); + ClusterInfoProvider provider = Mockito.mock(ClusterInfoProvider.class); + Mockito.when( + provider.getOnlineServersListWithPredicator(Mockito.anyList(), Mockito.any(Predicate.class))) + .thenReturn(idleServers); + balancer.setClusterInfoProvider(provider); RegionInfo hri1 = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName())) .setStartKey(Bytes.toBytes("key1")) .setEndKey(Bytes.toBytes("key2")) .setSplit(false) .setRegionId(100) .build(); - assertNull(balancer.randomAssignment(hri1, Collections.EMPTY_LIST)); + assertNull(balancer.randomAssignment(hri1, Collections.emptyList())); assertNull(balancer.randomAssignment(hri1, null)); for (int i = 0; i != 3; ++i) { ServerName sn = balancer.randomAssignment(hri1, allServers); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerHeterogeneousCost.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerHeterogeneousCost.java index 8a2ec848a751..8ea71183b86a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerHeterogeneousCost.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerHeterogeneousCost.java @@ -282,7 +282,7 @@ private ServerAndLoad createServer(final String host) { } static class FairRandomCandidateGenerator extends - StochasticLoadBalancer.RandomCandidateGenerator { + RandomCandidateGenerator { @Override public BalanceAction pickRandomRegions(BalancerClusterState cluster,