diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfo.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfo.java index 65314cb10d04..a55aa62a054d 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfo.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfo.java @@ -96,7 +96,7 @@ public boolean containsServer(Address hostPort) { /** * Get list of servers. */ - public Set
getServers() { + public SortedSet
getServers() { return servers; } diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdmin.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdmin.java index 15daf1450de6..e698f9ac6f25 100644 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdmin.java +++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdmin.java @@ -114,4 +114,12 @@ void moveServersAndTables(Set
servers, Set tables, * @throws IOException if a remote or network exception occurs */ void updateRSGroupConfig(String groupName, Map configuration) throws IOException; + + /** + * Update the configuration and trigger an online config change + * on all the regionservers in the RSGroup. + * @param groupName the group name + * @throws IOException if a remote or network exception occurs + */ + void updateConfiguration(String groupName) throws IOException; } diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminClient.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminClient.java index b4e5e80d272c..08357f64bc9b 100644 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminClient.java +++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminClient.java @@ -21,10 +21,14 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.EnumSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; +import org.apache.hadoop.hbase.ClusterMetrics; +import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.client.Admin; @@ -269,4 +273,19 @@ public void updateRSGroupConfig(String groupName, Map configurat throw ProtobufUtil.handleRemoteException(e); } } + + @Override + public void updateConfiguration(String groupName) throws IOException { + RSGroupInfo rsGroupInfo = getRSGroupInfo(groupName); + if (rsGroupInfo == null) { + throw new IllegalArgumentException("RSGroup does not exist: " + groupName); + } + ClusterMetrics status = + admin.getClusterMetrics(EnumSet.of(ClusterMetrics.Option.SERVERS_NAME)); + List groupServers = status.getServersName().stream().filter( + s -> rsGroupInfo.containsServer(s.getAddress())).collect(Collectors.toList()); + for (ServerName server : groupServers) { + admin.updateConfiguration(server); + } + } } diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java index 0c65dbf0f53f..744749f41c0a 100644 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java +++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java @@ -631,6 +631,15 @@ public void updateRSGroupConfig(String groupName, Map configurat } } + /** + * Because the {@link RSGroupAdminClient#updateConfiguration(String)} calls + * {@link org.apache.hadoop.hbase.client.Admin#updateConfiguration(ServerName)} method, the + * implementation of this method on the Server side is empty. + */ + @Override + public void updateConfiguration(String groupName) throws IOException { + } + private Map rsGroupGetRegionsInTransition(String groupName) throws IOException { Map rit = Maps.newTreeMap(); diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java index 0b875f59eb75..c4ae7d448aba 100644 --- a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java +++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java @@ -41,6 +41,7 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Waiter; +import org.apache.hadoop.hbase.client.AbstractTestUpdateConfiguration; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.TableDescriptor; @@ -62,7 +63,7 @@ import org.apache.hbase.thirdparty.com.google.common.collect.Maps; import org.apache.hbase.thirdparty.com.google.common.collect.Sets; -public abstract class TestRSGroupsBase { +public abstract class TestRSGroupsBase extends AbstractTestUpdateConfiguration { protected static final Logger LOG = LoggerFactory.getLogger(TestRSGroupsBase.class); //shared diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestUpdateRSGroupConfiguration.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestUpdateRSGroupConfiguration.java new file mode 100644 index 000000000000..18d9980f3a60 --- /dev/null +++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestUpdateRSGroupConfiguration.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.rsgroup; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.util.stream.Collectors; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.JVMClusterUtil; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category(MediumTests.class) +public class TestUpdateRSGroupConfiguration extends TestRSGroupsBase { + protected static final Logger LOG = LoggerFactory.getLogger(TestUpdateRSGroupConfiguration.class); + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestUpdateRSGroupConfiguration.class); + private static final String TEST_GROUP = "test"; + private static final String TEST2_GROUP = "test2"; + + @BeforeClass + public static void setUp() throws Exception { + setUpConfigurationFiles(TEST_UTIL); + setUpTestBeforeClass(); + addResourceToRegionServerConfiguration(TEST_UTIL); + } + + @AfterClass + public static void tearDown() throws Exception { + tearDownAfterClass(); + } + + @Before + public void beforeMethod() throws Exception { + setUpBeforeMethod(); + } + + @After + public void afterMethod() throws Exception { + tearDownAfterMethod(); + } + + @Test + public void testOnlineConfigChangeInRSGroup() throws Exception { + addGroup(TEST_GROUP, 1); + rsGroupAdmin.updateConfiguration(TEST_GROUP); + } + + @Test + public void testNonexistentRSGroup() throws Exception { + try { + rsGroupAdmin.updateConfiguration(TEST2_GROUP); + fail("Group does not exist: test2"); + } catch (IllegalArgumentException iae) { + // expected + } + } + + @Test + public void testCustomOnlineConfigChangeInRSGroup() throws Exception { + // Check the default configuration of the RegionServers + TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads().forEach(thread -> { + Configuration conf = thread.getRegionServer().getConfiguration(); + assertEquals(0, conf.getInt("hbase.custom.config", 0)); + }); + + replaceHBaseSiteXML(); + RSGroupInfo testRSGroup = addGroup(TEST_GROUP, 1); + RSGroupInfo test2RSGroup = addGroup(TEST2_GROUP, 1); + rsGroupAdmin.updateConfiguration(TEST_GROUP); + + // Check the configuration of the RegionServer in test rsgroup, should be update + Configuration regionServerConfiguration = + TEST_UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().stream() + .map(JVMClusterUtil.RegionServerThread::getRegionServer) + .filter(regionServer -> + (regionServer.getServerName().getAddress().equals(testRSGroup.getServers().first()))) + .collect(Collectors.toList()).get(0).getConfiguration(); + int custom = regionServerConfiguration.getInt("hbase.custom.config", 0); + assertEquals(1000, custom); + + // Check the configuration of the RegionServer in test2 rsgroup, should not be update + regionServerConfiguration = + TEST_UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().stream() + .map(JVMClusterUtil.RegionServerThread::getRegionServer) + .filter(regionServer -> + (regionServer.getServerName().getAddress().equals(test2RSGroup.getServers().first()))) + .collect(Collectors.toList()).get(0).getConfiguration(); + custom = regionServerConfiguration.getInt("hbase.custom.config", 0); + assertEquals(0, custom); + + restoreHBaseSiteXML(); + } +} diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java index faa0bc0005fd..99d36ee7fe34 100644 --- a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java +++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java @@ -132,6 +132,11 @@ public void updateRSGroupConfig(String groupName, Map configurat verify(); } + @Override + public void updateConfiguration(String groupName) throws IOException { + wrapped.updateConfiguration(groupName); + } + public void verify() throws IOException { Map groupMap = Maps.newHashMap(); Set zList = Sets.newHashSet(); diff --git a/hbase-rsgroup/src/test/resources/override-hbase-site.xml b/hbase-rsgroup/src/test/resources/override-hbase-site.xml new file mode 100644 index 000000000000..8bef31a4d772 --- /dev/null +++ b/hbase-rsgroup/src/test/resources/override-hbase-site.xml @@ -0,0 +1,146 @@ + + + + + + hbase.custom.config + 1000 + + + hbase.regionserver.msginterval + 1000 + Interval between messages from the RegionServer to HMaster + in milliseconds. Default is 15. Set this value low if you want unit + tests to be responsive. + + + + hbase.defaults.for.version.skip + true + + + hbase.server.thread.wakefrequency + 1000 + Time to sleep in between searches for work (in milliseconds). + Used as sleep interval by service threads such as hbase:meta scanner and log roller. + + + + hbase.master.event.waiting.time + 50 + Time to sleep between checks to see if a table event took place. + + + + hbase.regionserver.handler.count + 5 + + + hbase.master.info.port + -1 + The port for the hbase master web UI + Set to -1 if you do not want the info server to run. + + + + hbase.master.port + 0 + Always have masters and regionservers come up on port '0' so we don't clash over + default ports. + + + + hbase.regionserver.port + 0 + Always have masters and regionservers come up on port '0' so we don't clash over + default ports. + + + + hbase.ipc.client.fallback-to-simple-auth-allowed + true + + + + hbase.regionserver.info.port + -1 + The port for the hbase regionserver web UI + Set to -1 if you do not want the info server to run. + + + + hbase.regionserver.info.port.auto + true + Info server auto port bind. Enables automatic port + search if hbase.regionserver.info.port is already in use. + Enabled for testing to run multiple tests on one machine. + + + + hbase.regionserver.safemode + false + + Turn on/off safe mode in region server. Always on for production, always off + for tests. + + + + hbase.hregion.max.filesize + 67108864 + + Maximum desired file size for an HRegion. If filesize exceeds + value + (value / 2), the HRegion is split in two. Default: 256M. + + Keep the maximum filesize small so we split more often in tests. + + + + hadoop.log.dir + ${user.dir}/../logs + + + hbase.zookeeper.property.clientPort + 21818 + Property from ZooKeeper's config zoo.cfg. + The port at which the clients will connect. + + + + hbase.defaults.for.version.skip + true + + Set to true to skip the 'hbase.defaults.for.version'. + Setting this to true can be useful in contexts other than + the other side of a maven generation; i.e. running in an + ide. You'll want to set this boolean to true to avoid + seeing the RuntimeException complaint: "hbase-default.xml file + seems to be for and old version of HBase (@@@VERSION@@@), this + version is X.X.X-SNAPSHOT" + + + + hbase.table.sanity.checks + false + Skip sanity checks in tests + + + diff --git a/hbase-shell/src/main/ruby/hbase/rsgroup_admin.rb b/hbase-shell/src/main/ruby/hbase/rsgroup_admin.rb index af268e9a5d33..b25219ad399c 100644 --- a/hbase-shell/src/main/ruby/hbase/rsgroup_admin.rb +++ b/hbase-shell/src/main/ruby/hbase/rsgroup_admin.rb @@ -220,5 +220,11 @@ def alter_rsgroup_config(rsgroup_name, *args) end @admin.updateRSGroupConfig(rsgroup_name, configuration) end + + #---------------------------------------------------------------------------------------------- + # Updates the configuration of all the regionservers in the rsgroup. + def update_rsgroup_config(groupName) + @admin.updateConfiguration(groupName) + end end end diff --git a/hbase-shell/src/main/ruby/shell.rb b/hbase-shell/src/main/ruby/shell.rb index 6daf0de88b2d..eb62c5a3d745 100644 --- a/hbase-shell/src/main/ruby/shell.rb +++ b/hbase-shell/src/main/ruby/shell.rb @@ -552,6 +552,7 @@ def self.exception_handler(hide_traceback) commands: %w[ update_config update_all_config + update_rsgroup_config ] ) diff --git a/hbase-shell/src/main/ruby/shell/commands/update_rsgroup_config.rb b/hbase-shell/src/main/ruby/shell/commands/update_rsgroup_config.rb new file mode 100644 index 000000000000..4587a71ae89e --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/update_rsgroup_config.rb @@ -0,0 +1,37 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class UpdateRsgroupConfig < Command + def help + <<-EOF +Reload a subset of configuration on all servers in the rsgroup. See +http://hbase.apache.org/book.html#dyn_config for more details. Here is how +you would run the command in the hbase shell: + hbase> update_rsgroup_config 'groupName' +EOF + end + + def command(groupName) + rsgroup_admin.update_rsgroup_config(groupName) + end + end + end +end