Skip to content

Commit

Permalink
HBASE-26433 Rollback from ZK-less to ZK-based assignment could produc…
Browse files Browse the repository at this point in the history
…e inconsistent state - doubly assigned regions (#3826)

Signed-off-by: Andrew Purtell <[email protected]>
Signed-off-by: Geoffrey Jacoby <[email protected]>
Signed-off-by: David Manning <[email protected]>
  • Loading branch information
virajjasani authored Nov 9, 2021
1 parent 40b4cb1 commit 0d214ff
Show file tree
Hide file tree
Showing 5 changed files with 173 additions and 10 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CoordinatedStateException;
import org.apache.hadoop.hbase.HBaseIOException;
import org.apache.hadoop.hbase.HConstants;
Expand All @@ -68,6 +69,7 @@
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Admin.MasterSwitchType;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.RegionReplicaUtil;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
Expand Down Expand Up @@ -3293,7 +3295,8 @@ Set<ServerName> rebuildUserRegions() throws
HRegionInfo regionInfo = hrl.getRegionInfo();
if (regionInfo == null) continue;
int replicaId = regionInfo.getReplicaId();
State state = RegionStateStore.getRegionState(result, replicaId);
State state = RegionStateStore.getRegionState(result, replicaId,
ConfigUtil.isZKAssignmentInUse(server.getConfiguration()));
// keep a track of replicas to close. These were the replicas of the split parents
// from the previous life of the master. The master should have closed them before
// but it couldn't maybe because it crashed
Expand All @@ -3303,7 +3306,8 @@ Set<ServerName> rebuildUserRegions() throws
}
}
ServerName lastHost = hrl.getServerName();
ServerName regionLocation = RegionStateStore.getRegionServer(result, replicaId);
ServerName regionLocation = RegionStateStore.getRegionServer(result, replicaId,
ConfigUtil.isZKAssignmentInUse(server.getConfiguration()));
if (tableStateManager.isTableState(regionInfo.getTable(),
ZooKeeperProtos.Table.State.DISABLED)) {
// force region to forget it hosts for disabled/disabling tables.
Expand Down Expand Up @@ -3343,6 +3347,61 @@ Set<ServerName> rebuildUserRegions() throws
return offlineServers;
}

void deleteNonZkBasedQualifiersForZkBasedAssignment() throws IOException {
boolean isZKAssignmentInUse = ConfigUtil.isZKAssignmentInUse(server.getConfiguration());
if (isZKAssignmentInUse) {
List<Result> results = MetaTableAccessor.fullScanOfMeta(server.getConnection());
List<Delete> redundantCQDeletes = new ArrayList<>();
for (Result result : results) {
RegionLocations rl = MetaTableAccessor.getRegionLocations(result);
if (rl == null) {
LOG.error("No location found for " + result);
continue;
}
HRegionLocation[] locations = rl.getRegionLocations();
if (locations == null) {
LOG.error("No location found for " + rl);
continue;
}
for (HRegionLocation hrl : locations) {
if (hrl == null) {
continue;
}
HRegionInfo regionInfo = hrl.getRegionInfo();
if (regionInfo == null) {
LOG.error("No region info found " + hrl);
continue;
}
int replicaId = regionInfo.getReplicaId();
Cell cell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
RegionStateStore.getServerNameColumn(replicaId));
if (cell != null && cell.getValueLength() > 0) {
Delete delete =
new Delete(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
delete.addColumns(HConstants.CATALOG_FAMILY,
RegionStateStore.getServerNameColumn(replicaId));
redundantCQDeletes.add(delete);
}
cell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
RegionStateStore.getStateColumn(replicaId));
if (cell != null && cell.getValueLength() > 0) {
Delete delete =
new Delete(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
delete
.addColumns(HConstants.CATALOG_FAMILY, RegionStateStore.getStateColumn(replicaId));
redundantCQDeletes.add(delete);
}
}
}
if (!redundantCQDeletes.isEmpty()) {
LOG.info("Meta contains multiple info:sn and/or info:state values that are not required "
+ "for ZK based region assignment workflows. Preparing to delete these CQs. Number of"
+ " Deletes: " + redundantCQDeletes.size());
MetaTableAccessor.deleteFromMetaTable(server.getConnection(), redundantCQDeletes);
}
}
}

/**
* Recover the tables that were not fully moved to DISABLED state. These
* tables are in DISABLING state when the master restarted/switched.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -938,6 +938,8 @@ private void finishActiveMasterInitialization(MonitoredTask status)
// Set master as 'initialized'.
setInitialized(true);

this.assignmentManager.deleteNonZkBasedQualifiersForZkBasedAssignment();

assignmentManager.checkIfShouldMoveSystemRegionAsync();

status.setStatus("Starting quota manager");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,9 +73,9 @@ public class RegionStateStore {
* @return A ServerName instance or {@link HRegionInfo#getServerName(Result)}
* if necessary fields not found or empty.
*/
static ServerName getRegionServer(final Result r, int replicaId) {
static ServerName getRegionServer(final Result r, int replicaId, boolean isZKAssignmentInUse) {
Cell cell = r.getColumnLatestCell(HConstants.CATALOG_FAMILY, getServerNameColumn(replicaId));
if (cell == null || cell.getValueLength() == 0) {
if (cell == null || cell.getValueLength() == 0 || isZKAssignmentInUse) {
RegionLocations locations = MetaTableAccessor.getRegionLocations(r);
if (locations != null) {
HRegionLocation location = locations.getRegionLocation(replicaId);
Expand All @@ -89,7 +89,7 @@ static ServerName getRegionServer(final Result r, int replicaId) {
cell.getValueOffset(), cell.getValueLength()));
}

private static byte[] getServerNameColumn(int replicaId) {
static byte[] getServerNameColumn(int replicaId) {
return replicaId == 0
? HConstants.SERVERNAME_QUALIFIER
: Bytes.toBytes(HConstants.SERVERNAME_QUALIFIER_STR + META_REPLICA_ID_DELIMITER
Expand All @@ -101,14 +101,16 @@ private static byte[] getServerNameColumn(int replicaId) {
* @param r Result to pull the region state from
* @return the region state, or OPEN if there's no value written.
*/
static State getRegionState(final Result r, int replicaId) {
static State getRegionState(final Result r, int replicaId, boolean isZKAssignmentInUse) {
Cell cell = r.getColumnLatestCell(HConstants.CATALOG_FAMILY, getStateColumn(replicaId));
if (cell == null || cell.getValueLength() == 0) return State.OPEN;
return State.valueOf(Bytes.toString(cell.getValueArray(),
cell.getValueOffset(), cell.getValueLength()));
if (cell == null || cell.getValueLength() == 0 || isZKAssignmentInUse) {
return State.OPEN;
}
return State
.valueOf(Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
}

private static byte[] getStateColumn(int replicaId) {
static byte[] getStateColumn(int replicaId) {
return replicaId == 0
? HConstants.STATE_QUALIFIER
: Bytes.toBytes(HConstants.STATE_QUALIFIER_STR + META_REPLICA_ID_DELIMITER
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,4 +30,12 @@ public static boolean useZKForAssignment(Configuration conf) {
// To change the default, please also update ZooKeeperWatcher.java
return conf.getBoolean("hbase.assignment.usezk", true);
}

public static boolean isZKAssignmentInUse(Configuration conf) {
// ZK based region assignment is in use only if "hbase.assignment.usezk" is true
// and "hbase.assignment.usezk.migrating" is false.
return ConfigUtil.useZKForAssignment(conf) && !conf
.getBoolean("hbase.assignment.usezk.migrating", false);
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
/*
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.hadoop.hbase.client;

import java.util.List;
import org.junit.Assert;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.util.Bytes;

@Category({ LargeTests.class})
public class TestZKLessAssignment {

private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();

@Test
public void testZkLessAssignmentRollbackAndRollForward() throws Exception {
Configuration config = TEST_UTIL.getConfiguration();
config.setBoolean("hbase.assignment.usezk.migrating", true);
TEST_UTIL.startMiniCluster(2);
TableName tableName = TableName.valueOf("testZkLessAssignmentRollbackAndRollForward");
TEST_UTIL.createTable(tableName.getName(), new byte[][] { Bytes.toBytes("cf1") }, config)
.close();

try (Connection connection = ConnectionFactory.createConnection(config)) {
List<Result> results = MetaTableAccessor.fullScanOfMeta(connection);
boolean isSNQualifierExist = false;
boolean isStateQualifierExist = false;
for (Result result : results) {
Cell cell =
result.getColumnLatestCell(HConstants.CATALOG_FAMILY, HConstants.SERVERNAME_QUALIFIER);
if (cell != null && cell.getValueLength() > 0) {
isSNQualifierExist = true;
}
cell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY, HConstants.STATE_QUALIFIER);
if (cell != null && cell.getValueLength() > 0) {
isStateQualifierExist = true;
}
}
Assert.assertTrue(isSNQualifierExist);
Assert.assertTrue(isStateQualifierExist);
}

TEST_UTIL.shutdownMiniHBaseCluster();
config.unset("hbase.assignment.usezk.migrating");
TEST_UTIL.restartHBaseCluster(2);

try (Connection connection = ConnectionFactory.createConnection(config)) {
List<Result> results = MetaTableAccessor.fullScanOfMeta(connection);
boolean isSNQualifierExist = false;
boolean isStateQualifierExist = false;
for (Result result : results) {
Cell cell =
result.getColumnLatestCell(HConstants.CATALOG_FAMILY, HConstants.SERVERNAME_QUALIFIER);
if (cell != null && cell.getValueLength() > 0) {
isSNQualifierExist = true;
}
cell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY, HConstants.STATE_QUALIFIER);
if (cell != null && cell.getValueLength() > 0) {
isStateQualifierExist = true;
}
}
Assert.assertFalse(isSNQualifierExist);
Assert.assertFalse(isStateQualifierExist);
}
}

}

0 comments on commit 0d214ff

Please sign in to comment.