From 349c790f1806ed4d95d381b0c5a3b191e6c65f42 Mon Sep 17 00:00:00 2001 From: LiangJun He <2005hithlj@163.com> Date: Thu, 18 Aug 2022 21:16:14 +0800 Subject: [PATCH 1/5] HBASE-27304 Support using IP to expose master/rs servers for some special scenarios --- .../org/apache/hadoop/hbase/HConstants.java | 10 ++ .../apache/hadoop/hbase/HBaseServerBase.java | 10 +- .../hadoop/hbase/master/ServerManager.java | 5 +- .../apache/hadoop/hbase/util/RegionMover.java | 13 ++ .../hadoop/hbase/master/TestMasterUseIp.java | 76 ++++++++++ .../regionserver/TestRegionServerUseIp.java | 76 ++++++++++ .../hbase/util/TestRegionMoverUseIp.java | 139 ++++++++++++++++++ 7 files changed, 325 insertions(+), 4 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterUseIp.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerUseIp.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMoverUseIp.java diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java index 421fd9b408cb..063c7dd15b79 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java @@ -1571,6 +1571,16 @@ public enum OperationStatusCode { */ public static final int BATCH_ROWS_THRESHOLD_DEFAULT = 5000; + /** + * Support whether to use IP for master/rs servers for some special scenarios, default is false. + */ + public final static String HBASE_SERVER_USEIP_ENABLED_KEY = "hbase.server.useip.enabled"; + + /** + * Default value of {@link #HBASE_SERVER_USEIP_ENABLED_KEY} + */ + public final static boolean HBASE_SERVER_USEIP_ENABLED_DEFAULT = false; + private HConstants() { // Can't be instantiated with this ctor. } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java index 9c21b3f0f559..9053b0a10b83 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java @@ -252,9 +252,13 @@ public HBaseServerBase(Configuration conf, String name) throws IOException { this.rpcServices = createRpcServices(); useThisHostnameInstead = getUseThisHostnameInstead(conf); InetSocketAddress addr = rpcServices.getSocketAddress(); - String hostName = StringUtils.isBlank(useThisHostnameInstead) - ? addr.getHostName() - : this.useThisHostnameInstead; + + boolean useIp = conf.getBoolean(HConstants.HBASE_SERVER_USEIP_ENABLED_KEY, + HConstants.HBASE_SERVER_USEIP_ENABLED_DEFAULT); + String isaHostName = + useIp ? addr.getAddress().getHostAddress() : addr.getAddress().getHostName(); + String hostName = + StringUtils.isBlank(useThisHostnameInstead) ? isaHostName : useThisHostnameInstead; serverName = ServerName.valueOf(hostName, addr.getPort(), this.startcode); // login the zookeeper client principal (if using security) ZKAuthentication.loginClient(this.conf, HConstants.ZK_CLIENT_KEYTAB_FILE, diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java index d39dcc9ddad8..a53a9fb52f41 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java @@ -219,8 +219,11 @@ ServerName regionServerStartup(RegionServerStartupRequest request, int versionNu // in, it should have been removed from serverAddressToServerInfo and queued // for processing by ProcessServerShutdown. + boolean useIp = master.getConfiguration().getBoolean(HConstants.HBASE_SERVER_USEIP_ENABLED_KEY, + HConstants.HBASE_SERVER_USEIP_ENABLED_DEFAULT); + String isaHostName = useIp ? ia.getHostAddress() : ia.getHostName(); final String hostname = - request.hasUseThisHostnameInstead() ? request.getUseThisHostnameInstead() : ia.getHostName(); + request.hasUseThisHostnameInstead() ? request.getUseThisHostnameInstead() : isaHostName; ServerName sn = ServerName.valueOf(hostname, request.getPort(), request.getServerStartCode()); checkClockSkew(sn, request.getServerCurrentTime()); checkIsDead(sn, "STARTUP"); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java index 7ff8f34c64b0..1651d914ad5d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java @@ -26,6 +26,8 @@ import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; @@ -48,6 +50,7 @@ import java.util.concurrent.TimeoutException; import java.util.function.Predicate; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.ClusterMetrics.Option; import org.apache.hadoop.hbase.HBaseConfiguration; @@ -181,6 +184,16 @@ public RegionMoverBuilder(String hostname, Configuration conf) { } else { this.port = conf.getInt(HConstants.REGIONSERVER_PORT, HConstants.DEFAULT_REGIONSERVER_PORT); } + boolean useIp = conf.getBoolean(HConstants.HBASE_SERVER_USEIP_ENABLED_KEY, + HConstants.HBASE_SERVER_USEIP_ENABLED_DEFAULT); + if (useIp && StringUtils.isNotBlank(this.hostname)) { + try { + this.hostname = InetAddress.getByName(this.hostname).getHostAddress(); + } catch (UnknownHostException e) { + LOG.error(e.getMessage()); + throw new RuntimeException(e); + } + } this.filename = defaultDir + File.separator + System.getProperty("user.name") + this.hostname + ":" + Integer.toString(this.port); this.conf = conf; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterUseIp.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterUseIp.java new file mode 100644 index 000000000000..4cb85150851f --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterUseIp.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import static org.junit.Assert.assertEquals; + +import java.net.InetAddress; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.SingleProcessHBaseCluster; +import org.apache.hadoop.hbase.StartTestingClusterOption; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.junit.After; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category({ MasterTests.class, MediumTests.class }) +public class TestMasterUseIp { + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestMasterUseIp.class); + + private static final Logger LOG = LoggerFactory.getLogger(TestMasterUseIp.class); + + private HBaseTestingUtil TEST_UTIL; + private SingleProcessHBaseCluster CLUSTER; + + private static final int NUM_MASTERS = 1; + private static final int NUM_RS = 1; + + @Before + public void setup() throws Exception { + Configuration conf = HBaseConfiguration.create(); + conf.setBoolean(HConstants.HBASE_SERVER_USEIP_ENABLED_KEY, true); + TEST_UTIL = new HBaseTestingUtil(conf); + StartTestingClusterOption option = StartTestingClusterOption.builder().numMasters(NUM_MASTERS) + .numRegionServers(NUM_RS).numDataNodes(NUM_RS).build(); + CLUSTER = TEST_UTIL.startMiniCluster(option); + } + + @After + public void teardown() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testMasterUseIp() throws Exception { + String hostname = CLUSTER.getMaster(0).getServerName().getHostname(); + String ip = InetAddress.getByName(hostname).getHostAddress(); + LOG.info("hostname= " + hostname + " ,ip=" + ip); + assertEquals(ip, hostname); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerUseIp.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerUseIp.java new file mode 100644 index 000000000000..a2439a878882 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerUseIp.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import static org.junit.Assert.assertEquals; + +import java.net.InetAddress; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.SingleProcessHBaseCluster; +import org.apache.hadoop.hbase.StartTestingClusterOption; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.junit.After; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category({ RegionServerTests.class, MediumTests.class }) +public class TestRegionServerUseIp { + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestRegionServerUseIp.class); + + private static final Logger LOG = LoggerFactory.getLogger(TestRegionServerUseIp.class); + + private HBaseTestingUtil TEST_UTIL; + private SingleProcessHBaseCluster CLUSTER; + + private static final int NUM_MASTERS = 1; + private static final int NUM_RS = 1; + + @Before + public void setup() throws Exception { + Configuration conf = HBaseConfiguration.create(); + conf.setBoolean(HConstants.HBASE_SERVER_USEIP_ENABLED_KEY, true); + TEST_UTIL = new HBaseTestingUtil(conf); + StartTestingClusterOption option = StartTestingClusterOption.builder().numMasters(NUM_MASTERS) + .numRegionServers(NUM_RS).numDataNodes(NUM_RS).build(); + CLUSTER = TEST_UTIL.startMiniCluster(option); + } + + @After + public void teardown() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testRegionServerUseIp() throws Exception { + String hostname = CLUSTER.getRegionServer(0).getServerName().getHostname(); + String ip = InetAddress.getByName(hostname).getHostAddress(); + LOG.info("hostname= " + hostname + " ,ip=" + ip); + assertEquals(ip, hostname); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMoverUseIp.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMoverUseIp.java new file mode 100644 index 000000000000..6051097f6803 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMoverUseIp.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.SingleProcessHBaseCluster; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category({ MiscTests.class, LargeTests.class }) +public class TestRegionMoverUseIp { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestRegionMoverUseIp.class); + private static final Logger LOG = LoggerFactory.getLogger(TestRegionMoverUseIp.class); + + @Rule + public TestName name = new TestName(); + + private static HBaseTestingUtil TEST_UTIL; + private static ServerName rs0; + private static ServerName rs1; + private static ServerName rs2; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + Configuration conf = HBaseConfiguration.create(); + conf.setBoolean(HConstants.HBASE_SERVER_USEIP_ENABLED_KEY, true); + TEST_UTIL = new HBaseTestingUtil(conf); + TEST_UTIL.startMiniCluster(3); + + SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + rs0 = cluster.getRegionServer(0).getServerName(); + rs1 = cluster.getRegionServer(1).getServerName(); + rs2 = cluster.getRegionServer(2).getServerName(); + LOG.info("rs0 hostname=" + rs0.getHostname()); + LOG.info("rs1 hostname=" + rs1.getHostname()); + LOG.info("rs2 hostname=" + rs2.getHostname()); + TEST_UTIL.getAdmin().balancerSwitch(false, true); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Before + public void setUp() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(tableName) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("fam1")).build(); + int startKey = 0; + int endKey = 80000; + TEST_UTIL.getAdmin().createTable(tableDesc, Bytes.toBytes(startKey), Bytes.toBytes(endKey), 9); + } + + @Test + public void testRegionUnloadUesIp() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + Admin admin = TEST_UTIL.getAdmin(); + Table table = TEST_UTIL.getConnection().getTable(tableName); + List puts = IntStream.range(10, 50000).mapToObj(i -> new Put(Bytes.toBytes(i)) + .addColumn(Bytes.toBytes("fam1"), Bytes.toBytes("q1"), Bytes.toBytes("val_" + i))) + .collect(Collectors.toList()); + table.put(puts); + admin.flush(tableName); + admin.compact(tableName); + Thread.sleep(3000); + HRegionServer hRegionServer0 = cluster.getRegionServer(0); + HRegionServer hRegionServer1 = cluster.getRegionServer(1); + HRegionServer hRegionServer2 = cluster.getRegionServer(2); + int numRegions0 = hRegionServer0.getNumberOfOnlineRegions(); + int numRegions1 = hRegionServer1.getNumberOfOnlineRegions(); + int numRegions2 = hRegionServer2.getNumberOfOnlineRegions(); + + Assert.assertTrue(numRegions0 >= 3); + Assert.assertTrue(numRegions1 >= 3); + Assert.assertTrue(numRegions2 >= 3); + int totalRegions = numRegions0 + numRegions1 + numRegions2; + + // source RS: rs0 + String sourceRSName = rs0.getAddress().toString(); + RegionMover.RegionMoverBuilder rmBuilder = + new RegionMover.RegionMoverBuilder(sourceRSName, TEST_UTIL.getConfiguration()).ack(true) + .maxthreads(8); + try (RegionMover regionMover = rmBuilder.build()) { + regionMover.unload(); + int newNumRegions0 = hRegionServer0.getNumberOfOnlineRegions(); + int newNumRegions1 = hRegionServer1.getNumberOfOnlineRegions(); + int newNumRegions2 = hRegionServer2.getNumberOfOnlineRegions(); + Assert.assertEquals(0, newNumRegions0); + Assert.assertEquals(totalRegions, newNumRegions1 + newNumRegions2); + } + } +} From 8bfdee7756633696b0a65a987d6c33f37fe850dd Mon Sep 17 00:00:00 2001 From: LiangJun He <2005hithlj@163.com> Date: Mon, 22 Aug 2022 22:12:49 +0800 Subject: [PATCH 2/5] HBASE-27304 Support using IP to expose master/rs servers for some special scenarios --- .../src/main/java/org/apache/hadoop/hbase/HConstants.java | 8 +++++++- .../java/org/apache/hadoop/hbase/HBaseServerBase.java | 2 ++ .../org/apache/hadoop/hbase/master/ServerManager.java | 2 ++ .../java/org/apache/hadoop/hbase/util/RegionMover.java | 3 +++ 4 files changed, 14 insertions(+), 1 deletion(-) diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java index 063c7dd15b79..dac7de6eb5b4 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java @@ -1572,7 +1572,13 @@ public enum OperationStatusCode { public static final int BATCH_ROWS_THRESHOLD_DEFAULT = 5000; /** - * Support whether to use IP for master/rs servers for some special scenarios, default is false. + * In some scenarios, such as the elastic scaling scenario on the cloud, the HBase client may not + * be able to resolve the hostname of the newly added node. If the network is interconnected, the + * client can actually access the HBase cluster nodes through ip. However, since the HBase client + * obtains the Master/RS address info from or the ZK or the meta table, so the Master/RS of the + * HBase cluster needs to expose the service with ip instead of the hostname. Therefore, We can + * use hostname by default, but at the same time, we can also provide a config to support whether + * to use ip for Master/RS service. See HBASE-27304 for details. */ public final static String HBASE_SERVER_USEIP_ENABLED_KEY = "hbase.server.useip.enabled"; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java index 9053b0a10b83..2ef583514393 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java @@ -253,6 +253,8 @@ public HBaseServerBase(Configuration conf, String name) throws IOException { useThisHostnameInstead = getUseThisHostnameInstead(conf); InetSocketAddress addr = rpcServices.getSocketAddress(); + // if use-ip is enabled, we will use ip to expose Master/RS service for client, + // see HBASE-27304 for details. boolean useIp = conf.getBoolean(HConstants.HBASE_SERVER_USEIP_ENABLED_KEY, HConstants.HBASE_SERVER_USEIP_ENABLED_DEFAULT); String isaHostName = diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java index a53a9fb52f41..1bf3c73d59a4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java @@ -219,6 +219,8 @@ ServerName regionServerStartup(RegionServerStartupRequest request, int versionNu // in, it should have been removed from serverAddressToServerInfo and queued // for processing by ProcessServerShutdown. + // if use-ip is enabled, we will use ip to expose Master/RS service for client, + // see HBASE-27304 for details. boolean useIp = master.getConfiguration().getBoolean(HConstants.HBASE_SERVER_USEIP_ENABLED_KEY, HConstants.HBASE_SERVER_USEIP_ENABLED_DEFAULT); String isaHostName = useIp ? ia.getHostAddress() : ia.getHostName(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java index 1651d914ad5d..18394b9bed85 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java @@ -184,6 +184,9 @@ public RegionMoverBuilder(String hostname, Configuration conf) { } else { this.port = conf.getInt(HConstants.REGIONSERVER_PORT, HConstants.DEFAULT_REGIONSERVER_PORT); } + + // if use-ip is enabled, we will resolve the hostname to ip to ensure that the RegionMover + // logic can be executed normally, see HBASE-27304 for details. boolean useIp = conf.getBoolean(HConstants.HBASE_SERVER_USEIP_ENABLED_KEY, HConstants.HBASE_SERVER_USEIP_ENABLED_DEFAULT); if (useIp && StringUtils.isNotBlank(this.hostname)) { From a4cec48afbdace85b4489667ca53f32dd7faa5d0 Mon Sep 17 00:00:00 2001 From: LiangJun He <2005hithlj@163.com> Date: Wed, 24 Aug 2022 00:32:55 +0800 Subject: [PATCH 3/5] HBASE-27304 Support using IP to expose master/rs servers for some special scenarios --- .../apache/hadoop/hbase/util/RegionMover.java | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java index 18394b9bed85..903cd28c26e3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java @@ -27,7 +27,6 @@ import java.io.FileOutputStream; import java.io.IOException; import java.net.InetAddress; -import java.net.UnknownHostException; import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; @@ -50,7 +49,6 @@ import java.util.concurrent.TimeoutException; import java.util.function.Predicate; import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.ClusterMetrics.Option; import org.apache.hadoop.hbase.HBaseConfiguration; @@ -70,6 +68,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hbase.thirdparty.com.google.common.net.InetAddresses; import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine; import org.apache.hbase.thirdparty.org.apache.commons.collections4.CollectionUtils; @@ -119,6 +118,17 @@ private RegionMover(RegionMoverBuilder builder) throws IOException { setConf(builder.conf); this.conn = ConnectionFactory.createConnection(conf); this.admin = conn.getAdmin(); + + // if master's hostname is ip, it indicates that the master/RS has enabled use-ip, we need to + // resolve the current hostname to ip to ensure that the RegionMover logic can be executed + // normally, see HBASE-27304 for details. + ServerName master = admin.getClusterMetrics(EnumSet.of(Option.MASTER)).getMasterName(); + if (InetAddresses.isInetAddress(master.getHostname())) { + if (!InetAddresses.isInetAddress(hostname)) { + this.hostname = InetAddress.getByName(this.hostname).getHostAddress(); + } + } + // Only while running unit tests, builder.rackManager will not be null for the convenience of // providing custom rackManager. Otherwise for regular workflow/user triggered action, // builder.rackManager is supposed to be null. Hence, setter of builder.rackManager is @@ -184,19 +194,6 @@ public RegionMoverBuilder(String hostname, Configuration conf) { } else { this.port = conf.getInt(HConstants.REGIONSERVER_PORT, HConstants.DEFAULT_REGIONSERVER_PORT); } - - // if use-ip is enabled, we will resolve the hostname to ip to ensure that the RegionMover - // logic can be executed normally, see HBASE-27304 for details. - boolean useIp = conf.getBoolean(HConstants.HBASE_SERVER_USEIP_ENABLED_KEY, - HConstants.HBASE_SERVER_USEIP_ENABLED_DEFAULT); - if (useIp && StringUtils.isNotBlank(this.hostname)) { - try { - this.hostname = InetAddress.getByName(this.hostname).getHostAddress(); - } catch (UnknownHostException e) { - LOG.error(e.getMessage()); - throw new RuntimeException(e); - } - } this.filename = defaultDir + File.separator + System.getProperty("user.name") + this.hostname + ":" + Integer.toString(this.port); this.conf = conf; From 0e15a1e077d11d18787efea21d7b6fd65347e816 Mon Sep 17 00:00:00 2001 From: LiangJun He <2005hithlj@163.com> Date: Wed, 24 Aug 2022 00:39:10 +0800 Subject: [PATCH 4/5] HBASE-27304 Support using IP to expose master/rs servers for some special scenarios --- .../src/main/java/org/apache/hadoop/hbase/util/RegionMover.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java index 903cd28c26e3..509dd742160a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java @@ -124,7 +124,7 @@ private RegionMover(RegionMoverBuilder builder) throws IOException { // normally, see HBASE-27304 for details. ServerName master = admin.getClusterMetrics(EnumSet.of(Option.MASTER)).getMasterName(); if (InetAddresses.isInetAddress(master.getHostname())) { - if (!InetAddresses.isInetAddress(hostname)) { + if (!InetAddresses.isInetAddress(this.hostname)) { this.hostname = InetAddress.getByName(this.hostname).getHostAddress(); } } From 6d02ab722ca08b45bdbce77b28351201ed2c9d45 Mon Sep 17 00:00:00 2001 From: LiangJun He <2005hithlj@163.com> Date: Wed, 24 Aug 2022 11:45:07 +0800 Subject: [PATCH 5/5] HBASE-27304 Support using IP to expose master/rs servers for some special scenarios --- .../main/java/org/apache/hadoop/hbase/util/RegionMover.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java index 509dd742160a..b7285fb9f945 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java @@ -119,8 +119,8 @@ private RegionMover(RegionMoverBuilder builder) throws IOException { this.conn = ConnectionFactory.createConnection(conf); this.admin = conn.getAdmin(); - // if master's hostname is ip, it indicates that the master/RS has enabled use-ip, we need to - // resolve the current hostname to ip to ensure that the RegionMover logic can be executed + // if the hostname of master is ip, it indicates that the master/RS has enabled use-ip, we need + // to resolve the current hostname to ip to ensure that the RegionMover logic can be executed // normally, see HBASE-27304 for details. ServerName master = admin.getClusterMetrics(EnumSet.of(Option.MASTER)).getMasterName(); if (InetAddresses.isInetAddress(master.getHostname())) {