diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java index 376ff820c3fd..c20807d29434 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java @@ -50,6 +50,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CoordinatedStateException; import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.HConstants; @@ -68,6 +69,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Admin.MasterSwitchType; +import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager; @@ -3293,7 +3295,8 @@ Set rebuildUserRegions() throws HRegionInfo regionInfo = hrl.getRegionInfo(); if (regionInfo == null) continue; int replicaId = regionInfo.getReplicaId(); - State state = RegionStateStore.getRegionState(result, replicaId); + State state = RegionStateStore.getRegionState(result, replicaId, + ConfigUtil.isZKAssignmentInUse(server.getConfiguration())); // keep a track of replicas to close. These were the replicas of the split parents // from the previous life of the master. The master should have closed them before // but it couldn't maybe because it crashed @@ -3303,7 +3306,8 @@ Set rebuildUserRegions() throws } } ServerName lastHost = hrl.getServerName(); - ServerName regionLocation = RegionStateStore.getRegionServer(result, replicaId); + ServerName regionLocation = RegionStateStore.getRegionServer(result, replicaId, + ConfigUtil.isZKAssignmentInUse(server.getConfiguration())); if (tableStateManager.isTableState(regionInfo.getTable(), ZooKeeperProtos.Table.State.DISABLED)) { // force region to forget it hosts for disabled/disabling tables. @@ -3343,6 +3347,61 @@ Set rebuildUserRegions() throws return offlineServers; } + void deleteNonZkBasedQualifiersForZkBasedAssignment() throws IOException { + boolean isZKAssignmentInUse = ConfigUtil.isZKAssignmentInUse(server.getConfiguration()); + if (isZKAssignmentInUse) { + List results = MetaTableAccessor.fullScanOfMeta(server.getConnection()); + List redundantCQDeletes = new ArrayList<>(); + for (Result result : results) { + RegionLocations rl = MetaTableAccessor.getRegionLocations(result); + if (rl == null) { + LOG.error("No location found for " + result); + continue; + } + HRegionLocation[] locations = rl.getRegionLocations(); + if (locations == null) { + LOG.error("No location found for " + rl); + continue; + } + for (HRegionLocation hrl : locations) { + if (hrl == null) { + continue; + } + HRegionInfo regionInfo = hrl.getRegionInfo(); + if (regionInfo == null) { + LOG.error("No region info found " + hrl); + continue; + } + int replicaId = regionInfo.getReplicaId(); + Cell cell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY, + RegionStateStore.getServerNameColumn(replicaId)); + if (cell != null && cell.getValueLength() > 0) { + Delete delete = + new Delete(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()); + delete.addColumns(HConstants.CATALOG_FAMILY, + RegionStateStore.getServerNameColumn(replicaId)); + redundantCQDeletes.add(delete); + } + cell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY, + RegionStateStore.getStateColumn(replicaId)); + if (cell != null && cell.getValueLength() > 0) { + Delete delete = + new Delete(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()); + delete + .addColumns(HConstants.CATALOG_FAMILY, RegionStateStore.getStateColumn(replicaId)); + redundantCQDeletes.add(delete); + } + } + } + if (!redundantCQDeletes.isEmpty()) { + LOG.info("Meta contains multiple info:sn and/or info:state values that are not required " + + "for ZK based region assignment workflows. Preparing to delete these CQs. Number of" + + " Deletes: " + redundantCQDeletes.size()); + MetaTableAccessor.deleteFromMetaTable(server.getConnection(), redundantCQDeletes); + } + } + } + /** * Recover the tables that were not fully moved to DISABLED state. These * tables are in DISABLING state when the master restarted/switched. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 92fb6d1e43d1..df8402bc8c50 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -938,6 +938,8 @@ private void finishActiveMasterInitialization(MonitoredTask status) // Set master as 'initialized'. setInitialized(true); + this.assignmentManager.deleteNonZkBasedQualifiersForZkBasedAssignment(); + assignmentManager.checkIfShouldMoveSystemRegionAsync(); status.setStatus("Starting quota manager"); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStateStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStateStore.java index 476b4d51381a..282173097d16 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStateStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStateStore.java @@ -73,9 +73,9 @@ public class RegionStateStore { * @return A ServerName instance or {@link HRegionInfo#getServerName(Result)} * if necessary fields not found or empty. */ - static ServerName getRegionServer(final Result r, int replicaId) { + static ServerName getRegionServer(final Result r, int replicaId, boolean isZKAssignmentInUse) { Cell cell = r.getColumnLatestCell(HConstants.CATALOG_FAMILY, getServerNameColumn(replicaId)); - if (cell == null || cell.getValueLength() == 0) { + if (cell == null || cell.getValueLength() == 0 || isZKAssignmentInUse) { RegionLocations locations = MetaTableAccessor.getRegionLocations(r); if (locations != null) { HRegionLocation location = locations.getRegionLocation(replicaId); @@ -89,7 +89,7 @@ static ServerName getRegionServer(final Result r, int replicaId) { cell.getValueOffset(), cell.getValueLength())); } - private static byte[] getServerNameColumn(int replicaId) { + static byte[] getServerNameColumn(int replicaId) { return replicaId == 0 ? HConstants.SERVERNAME_QUALIFIER : Bytes.toBytes(HConstants.SERVERNAME_QUALIFIER_STR + META_REPLICA_ID_DELIMITER @@ -101,14 +101,16 @@ private static byte[] getServerNameColumn(int replicaId) { * @param r Result to pull the region state from * @return the region state, or OPEN if there's no value written. */ - static State getRegionState(final Result r, int replicaId) { + static State getRegionState(final Result r, int replicaId, boolean isZKAssignmentInUse) { Cell cell = r.getColumnLatestCell(HConstants.CATALOG_FAMILY, getStateColumn(replicaId)); - if (cell == null || cell.getValueLength() == 0) return State.OPEN; - return State.valueOf(Bytes.toString(cell.getValueArray(), - cell.getValueOffset(), cell.getValueLength())); + if (cell == null || cell.getValueLength() == 0 || isZKAssignmentInUse) { + return State.OPEN; + } + return State + .valueOf(Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength())); } - private static byte[] getStateColumn(int replicaId) { + static byte[] getStateColumn(int replicaId) { return replicaId == 0 ? HConstants.STATE_QUALIFIER : Bytes.toBytes(HConstants.STATE_QUALIFIER_STR + META_REPLICA_ID_DELIMITER diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ConfigUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ConfigUtil.java index 882d199c8c78..87e4b21a4f90 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ConfigUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ConfigUtil.java @@ -30,4 +30,12 @@ public static boolean useZKForAssignment(Configuration conf) { // To change the default, please also update ZooKeeperWatcher.java return conf.getBoolean("hbase.assignment.usezk", true); } + + public static boolean isZKAssignmentInUse(Configuration conf) { + // ZK based region assignment is in use only if "hbase.assignment.usezk" is true + // and "hbase.assignment.usezk.migrating" is false. + return ConfigUtil.useZKForAssignment(conf) && !conf + .getBoolean("hbase.assignment.usezk.migrating", false); + } + } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestZKLessAssignment.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestZKLessAssignment.java new file mode 100644 index 000000000000..4c64ae15b1f4 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestZKLessAssignment.java @@ -0,0 +1,92 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.client; + +import java.util.List; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.MetaTableAccessor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.util.Bytes; + +@Category({ LargeTests.class}) +public class TestZKLessAssignment { + + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + @Test + public void testZkLessAssignmentRollbackAndRollForward() throws Exception { + Configuration config = TEST_UTIL.getConfiguration(); + config.setBoolean("hbase.assignment.usezk.migrating", true); + TEST_UTIL.startMiniCluster(2); + TableName tableName = TableName.valueOf("testZkLessAssignmentRollbackAndRollForward"); + TEST_UTIL.createTable(tableName.getName(), new byte[][] { Bytes.toBytes("cf1") }, config) + .close(); + + try (Connection connection = ConnectionFactory.createConnection(config)) { + List results = MetaTableAccessor.fullScanOfMeta(connection); + boolean isSNQualifierExist = false; + boolean isStateQualifierExist = false; + for (Result result : results) { + Cell cell = + result.getColumnLatestCell(HConstants.CATALOG_FAMILY, HConstants.SERVERNAME_QUALIFIER); + if (cell != null && cell.getValueLength() > 0) { + isSNQualifierExist = true; + } + cell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY, HConstants.STATE_QUALIFIER); + if (cell != null && cell.getValueLength() > 0) { + isStateQualifierExist = true; + } + } + Assert.assertTrue(isSNQualifierExist); + Assert.assertTrue(isStateQualifierExist); + } + + TEST_UTIL.shutdownMiniHBaseCluster(); + config.unset("hbase.assignment.usezk.migrating"); + TEST_UTIL.restartHBaseCluster(2); + + try (Connection connection = ConnectionFactory.createConnection(config)) { + List results = MetaTableAccessor.fullScanOfMeta(connection); + boolean isSNQualifierExist = false; + boolean isStateQualifierExist = false; + for (Result result : results) { + Cell cell = + result.getColumnLatestCell(HConstants.CATALOG_FAMILY, HConstants.SERVERNAME_QUALIFIER); + if (cell != null && cell.getValueLength() > 0) { + isSNQualifierExist = true; + } + cell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY, HConstants.STATE_QUALIFIER); + if (cell != null && cell.getValueLength() > 0) { + isStateQualifierExist = true; + } + } + Assert.assertFalse(isSNQualifierExist); + Assert.assertFalse(isStateQualifierExist); + } + } + +}