From 56fd0353e3c1c7bf3cb0c9c5b938358fcd1aa46c Mon Sep 17 00:00:00 2001 From: binlijin Date: Fri, 18 Oct 2019 19:26:20 +0800 Subject: [PATCH 1/4] HBASE-23187 Update parent region state to SPLIT in meta --- .../hadoop/hbase/MetaTableAccessor.java | 1 + .../master/assignment/TestRegionSplit.java | 186 ++++++++++++++++++ 2 files changed, 187 insertions(+) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionSplit.java diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java index ad54324c7b3d..b2b510289e19 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java @@ -1580,6 +1580,7 @@ public static void splitRegion(Connection connection, RegionInfo parent, long pa Put putParent = makePutFromRegionInfo(RegionInfoBuilder.newBuilder(parent) .setOffline(true) .setSplit(true).build(), time); + addRegionStateToPut(putParent, State.SPLIT); addDaughtersToPut(putParent, splitA, splitB); // Puts for daughters diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionSplit.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionSplit.java new file mode 100644 index 000000000000..8c7f3d518c13 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionSplit.java @@ -0,0 +1,186 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.assignment; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.StartMiniClusterOption; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.Waiter; +import org.apache.hadoop.hbase.client.CompactionState; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility; +import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; +import org.apache.hadoop.hbase.procedure2.ProcedureMetrics; +import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.JVMClusterUtil; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +@Category({MasterTests.class, LargeTests.class}) +public class TestRegionSplit { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestRegionSplit.class); + + private static final Logger LOG = LoggerFactory.getLogger(TestRegionSplit.class); + + protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + private static String ColumnFamilyName = "cf"; + + private static final int startRowNum = 11; + private static final int rowCount = 60; + + @Rule + public TestName name = new TestName(); + + private static void setupConf(Configuration conf) { + } + + @BeforeClass + public static void setupCluster() throws Exception { + setupConf(UTIL.getConfiguration()); + StartMiniClusterOption option = + StartMiniClusterOption.builder().numMasters(1).numRegionServers(3).numDataNodes(3).build(); + UTIL.startMiniCluster(option); + } + + @AfterClass + public static void cleanupTest() throws Exception { + try { + UTIL.shutdownMiniCluster(); + } catch (Exception e) { + LOG.warn("failure shutting down cluster", e); + } + } + + @Before + public void setup() throws Exception { + // Turn off the meta scanner so it don't remove parent on us. + UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(false); + // Disable compaction. + for (int i = 0; i < UTIL.getHBaseCluster().getLiveRegionServerThreads().size(); i++) { + UTIL.getHBaseCluster().getRegionServer(i).getCompactSplitThread().switchCompaction(false); + } + } + + @After + public void tearDown() throws Exception { + for (TableDescriptor htd : UTIL.getAdmin().listTableDescriptors()) { + UTIL.deleteTable(htd.getTableName()); + } + } + + @Test + public void testSplitTableRegion() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + final ProcedureExecutor procExec = getMasterProcedureExecutor(); + + RegionInfo[] regions = + MasterProcedureTestingUtility.createTable(procExec, tableName, null, ColumnFamilyName); + insertData(tableName); + int splitRowNum = startRowNum + rowCount / 2; + byte[] splitKey = Bytes.toBytes("" + splitRowNum); + + assertTrue("not able to find a splittable region", regions != null); + assertTrue("not able to find a splittable region", regions.length == 1); + + // Split region of the table + long procId = procExec.submitProcedure( + new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey)); + // Wait the completion + ProcedureTestingUtility.waitProcedure(procExec, procId); + ProcedureTestingUtility.assertProcNotFailed(procExec, procId); + + assertTrue("not able to split table", UTIL.getHBaseCluster().getRegions(tableName).size() == 2); + + //disable table + UTIL.getAdmin().disableTable(tableName); + Thread.sleep(500); + + //stop master + UTIL.getHBaseCluster().stopMaster(0); + UTIL.getHBaseCluster().waitOnMaster(0); + Thread.sleep(500); + + //restart master + JVMClusterUtil.MasterThread t = UTIL.getHBaseCluster().startMaster(); + Thread.sleep(500); + + // enable table + UTIL.getAdmin().enableTable(tableName); + Thread.sleep(500); + + assertEquals("Table region not correct.", 2, + UTIL.getHBaseCluster().getRegions(tableName).size()); + } + + private void insertData(final TableName tableName) throws IOException { + Table t = UTIL.getConnection().getTable(tableName); + Put p; + for (int i = 0; i < rowCount / 2; i++) { + p = new Put(Bytes.toBytes("" + (startRowNum + i))); + p.addColumn(Bytes.toBytes(ColumnFamilyName), Bytes.toBytes("q1"), Bytes.toBytes(i)); + t.put(p); + p = new Put(Bytes.toBytes("" + (startRowNum + rowCount - i - 1))); + p.addColumn(Bytes.toBytes(ColumnFamilyName), Bytes.toBytes("q1"), Bytes.toBytes(i)); + t.put(p); + } + UTIL.getAdmin().flush(tableName); + } + + private ProcedureExecutor getMasterProcedureExecutor() { + return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor(); + } +} From 37842d6a402b8f6bb22b61c78d62f592086def77 Mon Sep 17 00:00:00 2001 From: binlijin Date: Mon, 21 Oct 2019 15:23:01 +0800 Subject: [PATCH 2/4] fix comment --- .../java/org/apache/hadoop/hbase/MetaTableAccessor.java | 1 - .../hadoop/hbase/master/assignment/RegionStates.java | 8 ++++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java index b2b510289e19..ad54324c7b3d 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java @@ -1580,7 +1580,6 @@ public static void splitRegion(Connection connection, RegionInfo parent, long pa Put putParent = makePutFromRegionInfo(RegionInfoBuilder.newBuilder(parent) .setOffline(true) .setSplit(true).build(), time); - addRegionStateToPut(putParent, State.SPLIT); addDaughtersToPut(putParent, splitA, splitB); // Puts for daughters diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStates.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStates.java index e6017c0f837c..6654620fee78 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStates.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStates.java @@ -350,9 +350,13 @@ boolean include(final RegionStateNode node, final boolean offline) { if (LOG.isTraceEnabled()) { LOG.trace("WORKING ON " + node + " " + node.getRegionInfo()); } - if (node.isInState(State.SPLIT)) return false; - if (node.isInState(State.OFFLINE) && !offline) return false; final RegionInfo hri = node.getRegionInfo(); + if (node.isInState(State.SPLIT) || hri.isSplit()) { + return false; + } + if ((node.isInState(State.OFFLINE) || hri.isOffline()) && !offline) { + return false; + } return (!hri.isOffline() && !hri.isSplit()) || ((hri.isOffline() || hri.isSplit()) && offline); } From 9b570c15b1c32af410a329370774a5e5c96d1a01 Mon Sep 17 00:00:00 2001 From: binlijin Date: Mon, 28 Oct 2019 19:44:09 +0800 Subject: [PATCH 3/4] fix checkstyle --- .../master/assignment/TestRegionSplit.java | 27 +++++-------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionSplit.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionSplit.java index 8c7f3d518c13..570c2df2be08 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionSplit.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionSplit.java @@ -17,36 +17,26 @@ */ package org.apache.hadoop.hbase.master.assignment; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; + import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.StartMiniClusterOption; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.Waiter; -import org.apache.hadoop.hbase.client.CompactionState; -import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.RegionInfo; -import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.TableDescriptor; -import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants; -import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; -import org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility; -import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; -import org.apache.hadoop.hbase.procedure2.ProcedureMetrics; import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; -import org.apache.hadoop.hbase.regionserver.HRegion; -import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.JVMClusterUtil; import org.junit.After; @@ -61,11 +51,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - @Category({MasterTests.class, LargeTests.class}) public class TestRegionSplit { From 2108f0176fda6459b5c7daa5e04105a58f0cabdd Mon Sep 17 00:00:00 2001 From: binlijin Date: Mon, 28 Oct 2019 20:49:56 +0800 Subject: [PATCH 4/4] fix checkstyle --- .../apache/hadoop/hbase/master/assignment/TestRegionSplit.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionSplit.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionSplit.java index 570c2df2be08..f723af820886 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionSplit.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionSplit.java @@ -19,9 +19,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; - import java.io.IOException; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility;