Skip to content

Commit

Permalink
HBASE-27250 MasterRpcService#setRegionStateInMeta does not support re…
Browse files Browse the repository at this point in the history 
…plica region encodedNames or region names

- Added sanity check to make sure input region encoded name or region name is valid
- Assignment improvements pertaining to read replica regions
- make several MetaTableAccessor methods more precise in their handling of replica regions
- hbck2 setRegionStateInMeta and HBCKServerCrashProcedure handle read replicas
- separate AM helper methods -- loading RegionInfo from cache vs. refreshing cache from meta
- AM helper method support loading RegionInfo from cache via either region name and encoded region
  name (both caches are maintained, and under lock)
- consolidate, extend tests to cover read replica regions

Co-authored-by: Huaxiang Sun <[email protected]>
Co-authored-by: Nick Dimiduk <[email protected]>
Signed-off-by: Peter Somogyi <[email protected]>
  • Loading branch information
@huaxiangsun @ndimiduk
3 people authored Feb 21, 2023
1 parent cf179d3 commit 22dbb7a
Show file tree
Hide file tree
Showing 13 changed files with 377 additions and 266 deletions.
49 changes: 36 additions & 13 deletions hbase-server/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,8 @@ public static HRegionLocation getRegionLocation(Connection connection, byte[] re
RegionLocations locations = CatalogFamilyFormat.getRegionLocations(r);
return locations == null
? null
: locations.getRegionLocation(parsedInfo == null ? 0 : parsedInfo.getReplicaId());
: locations.getRegionLocation(
parsedInfo == null ? RegionInfo.DEFAULT_REPLICA_ID : parsedInfo.getReplicaId());
}

/**
Expand Down Expand Up @@ -220,12 +221,12 @@ public static Result getCatalogFamilyRow(Connection connection, RegionInfo ri)
/**
* Gets the result in hbase:meta for the specified region.
* @param connection connection we're using
* @param regionName region we're looking for
* @param regionInfo region we're looking for
* @return result of the specified region
*/
public static Result getRegionResult(Connection connection, byte[] regionName)
public static Result getRegionResult(Connection connection, RegionInfo regionInfo)
throws IOException {
Get get = new Get(regionName);
Get get = new Get(CatalogFamilyFormat.getMetaKeyForRegion(regionInfo));
get.addFamily(HConstants.CATALOG_FAMILY);
try (Table t = getMetaHTable(connection)) {
return t.get(get);
Expand Down Expand Up @@ -621,11 +622,22 @@ public static void updateTableState(Connection conn, TableName tableName, TableS
////////////////////////
// Editing operations //
////////////////////////

/**
* Generates and returns a {@link Put} containing the {@link RegionInfo} for the catalog table.
* @throws IllegalArgumentException when the provided RegionInfo is not the default replica.
*/
public static Put makePutFromRegionInfo(RegionInfo regionInfo) throws IOException {
return makePutFromRegionInfo(regionInfo, EnvironmentEdgeManager.currentTime());
}

/**
* Generates and returns a Put containing the region into for the catalog table
* Generates and returns a {@link Put} containing the {@link RegionInfo} for the catalog table.
* @throws IllegalArgumentException when the provided RegionInfo is not the default replica.
*/
public static Put makePutFromRegionInfo(RegionInfo regionInfo, long ts) throws IOException {
return addRegionInfo(new Put(regionInfo.getRegionName(), ts), regionInfo);
return addRegionInfo(new Put(CatalogFamilyFormat.getMetaKeyForRegion(regionInfo), ts),
regionInfo);
}

/**
Expand All @@ -635,7 +647,11 @@ public static Delete makeDeleteFromRegionInfo(RegionInfo regionInfo, long ts) {
if (regionInfo == null) {
throw new IllegalArgumentException("Can't make a delete for null region");
}
Delete delete = new Delete(regionInfo.getRegionName());
if (regionInfo.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) {
throw new IllegalArgumentException(
"Can't make delete for a replica region. Operate on the primary");
}
Delete delete = new Delete(CatalogFamilyFormat.getMetaKeyForRegion(regionInfo));
delete.addFamily(HConstants.CATALOG_FAMILY, ts);
return delete;
}
Expand Down Expand Up @@ -726,9 +742,15 @@ private static void deleteFromMetaTable(final Connection connection, final List<
}
}

public static Put addRegionStateToPut(Put put, RegionState.State state) throws IOException {
/**
* Set the column value corresponding to this {@code replicaId}'s {@link RegionState} to the
* provided {@code state}. Mutates the provided {@link Put}.
*/
public static Put addRegionStateToPut(Put put, int replicaId, RegionState.State state)
throws IOException {
put.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(put.getRow())
.setFamily(HConstants.CATALOG_FAMILY).setQualifier(HConstants.STATE_QUALIFIER)
.setFamily(HConstants.CATALOG_FAMILY)
.setQualifier(CatalogFamilyFormat.getRegionStateColumn(replicaId))
.setTimestamp(put.getTimestamp()).setType(Cell.Type.Put).setValue(Bytes.toBytes(state.name()))
.build());
return put;
Expand All @@ -739,8 +761,9 @@ public static Put addRegionStateToPut(Put put, RegionState.State state) throws I
*/
public static void updateRegionState(Connection connection, RegionInfo ri,
RegionState.State state) throws IOException {
Put put = new Put(RegionReplicaUtil.getRegionInfoForDefaultReplica(ri).getRegionName());
putsToMetaTable(connection, Collections.singletonList(addRegionStateToPut(put, state)));
final Put put = makePutFromRegionInfo(ri);
addRegionStateToPut(put, ri.getReplicaId(), state);
putsToMetaTable(connection, Collections.singletonList(put));
}

/**
Expand All @@ -759,7 +782,7 @@ public static void updateRegionState(Connection connection, RegionInfo ri,
public static void addSplitsToParent(Connection connection, RegionInfo regionInfo,
RegionInfo splitA, RegionInfo splitB) throws IOException {
try (Table meta = getMetaHTable(connection)) {
Put put = makePutFromRegionInfo(regionInfo, EnvironmentEdgeManager.currentTime());
Put put = makePutFromRegionInfo(regionInfo);
addDaughtersToPut(put, splitA, splitB);
meta.put(put);
debugLogMutation(put);
Expand Down Expand Up @@ -797,7 +820,7 @@ public static void addRegionsToMeta(Connection connection, List<RegionInfo> regi
}
Put put = makePutFromRegionInfo(regionInfo, ts);
// New regions are added with initial state of CLOSED.
addRegionStateToPut(put, RegionState.State.CLOSED);
addRegionStateToPut(put, regionInfo.getReplicaId(), RegionState.State.CLOSED);
// Add empty locations for region replicas so that number of replicas can be cached
// whenever the primary region is looked up from meta
for (int i = 1; i < regionReplication; i++) {
Expand Down
157 changes: 73 additions & 84 deletions hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
Expand All @@ -49,7 +50,6 @@
import org.apache.hadoop.hbase.client.BalanceResponse;
import org.apache.hadoop.hbase.client.MasterSwitchType;
import org.apache.hadoop.hbase.client.NormalizeTableFilterParams;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableDescriptor;
Expand All @@ -66,6 +66,7 @@
import org.apache.hadoop.hbase.ipc.RpcServer.BlockingServiceAndInterface;
import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
import org.apache.hadoop.hbase.master.assignment.RegionStateNode;
import org.apache.hadoop.hbase.master.assignment.RegionStates;
import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
Expand Down Expand Up @@ -2549,64 +2550,69 @@ public SetRegionStateInMetaResponse setRegionStateInMeta(RpcController controlle
SetRegionStateInMetaRequest request) throws ServiceException {
rpcPreCheck("setRegionStateInMeta");
SetRegionStateInMetaResponse.Builder builder = SetRegionStateInMetaResponse.newBuilder();
final AssignmentManager am = server.getAssignmentManager();
try {
for (RegionSpecifierAndState s : request.getStatesList()) {
RegionSpecifier spec = s.getRegionSpecifier();
String encodedName;
if (spec.getType() == RegionSpecifierType.ENCODED_REGION_NAME) {
encodedName = spec.getValue().toStringUtf8();
} else {
// TODO: actually, a full region name can save a lot on meta scan, improve later.
encodedName = RegionInfo.encodeRegionName(spec.getValue().toByteArray());
final RegionSpecifier spec = s.getRegionSpecifier();
final RegionInfo targetRegionInfo = getRegionInfo(spec);
final RegionState.State targetState = RegionState.State.convert(s.getState());
final RegionState.State currentState = Optional.ofNullable(targetRegionInfo)
.map(info -> am.getRegionStates().getRegionState(info)).map(RegionState::getState)
.orElseThrow(
() -> new ServiceException("No existing state known for region '" + spec + "'."));
LOG.info("{} set region={} state from {} to {}", server.getClientIdAuditPrefix(),
targetRegionInfo, currentState, targetState);
if (currentState == targetState) {
LOG.debug("Proposed state matches current state. {}, {}", targetRegionInfo, currentState);
continue;
}
RegionInfo info = this.server.getAssignmentManager().loadRegionFromMeta(encodedName);
LOG.trace("region info loaded from meta table: {}", info);
RegionState prevState =
this.server.getAssignmentManager().getRegionStates().getRegionState(info);
RegionState.State newState = RegionState.State.convert(s.getState());
LOG.info("{} set region={} state from {} to {}", server.getClientIdAuditPrefix(), info,
prevState.getState(), newState);
Put metaPut =
MetaTableAccessor.makePutFromRegionInfo(info, EnvironmentEdgeManager.currentTime());
metaPut.addColumn(HConstants.CATALOG_FAMILY, HConstants.STATE_QUALIFIER,
Bytes.toBytes(newState.name()));
List<Put> putList = new ArrayList<>();
putList.add(metaPut);
MetaTableAccessor.putsToMetaTable(this.server.getConnection(), putList);
MetaTableAccessor.updateRegionState(server.getConnection(), targetRegionInfo, targetState);
// Loads from meta again to refresh AM cache with the new region state
this.server.getAssignmentManager().loadRegionFromMeta(encodedName);
am.populateRegionStatesFromMeta(targetRegionInfo);
builder.addStates(RegionSpecifierAndState.newBuilder().setRegionSpecifier(spec)
.setState(prevState.getState().convert()));
.setState(currentState.convert()));
}
} catch (Exception e) {
} catch (IOException e) {
throw new ServiceException(e);
}
return builder.build();
}

/**
* Get RegionInfo from Master using content of RegionSpecifier as key.
* @return RegionInfo found by decoding <code>rs</code> or null if none found
* Get {@link RegionInfo} from Master using content of {@link RegionSpecifier} as key.
* @return {@link RegionInfo} found by decoding {@code rs} or {@code null} if {@code rs} is
* unknown to the master.
* @throws ServiceException If some error occurs while querying META or parsing results.
*/
private RegionInfo getRegionInfo(HBaseProtos.RegionSpecifier rs) throws UnknownRegionException {
RegionInfo ri = null;
private RegionInfo getRegionInfo(HBaseProtos.RegionSpecifier rs) throws ServiceException {
// TODO: this doesn't handle MOB regions. Should it? See the public method #getRegionInfo
final AssignmentManager am = server.getAssignmentManager();
final String encodedRegionName;
final RegionInfo info;
// first try resolving from the AM's caches.
switch (rs.getType()) {
case REGION_NAME:
final byte[] regionName = rs.getValue().toByteArray();
ri = this.server.getAssignmentManager().getRegionInfo(regionName);
encodedRegionName = RegionInfo.encodeRegionName(regionName);
info = am.getRegionInfo(regionName);
break;
case ENCODED_REGION_NAME:
String encodedRegionName = Bytes.toString(rs.getValue().toByteArray());
RegionState regionState =
this.server.getAssignmentManager().getRegionStates().getRegionState(encodedRegionName);
ri = regionState == null
? this.server.getAssignmentManager().loadRegionFromMeta(encodedRegionName)
: regionState.getRegion();
encodedRegionName = rs.getValue().toStringUtf8();
info = am.getRegionInfo(encodedRegionName);
break;
default:
break;
throw new IllegalArgumentException("Unrecognized RegionSpecifierType " + rs.getType());
}
if (info != null) {
return info;
}
// fall back to a meta scan and check the cache again.
try {
am.populateRegionStatesFromMeta(encodedRegionName);
} catch (IOException e) {
throw new ServiceException(e);
}
return ri;
return am.getRegionInfo(encodedRegionName);
}

/**
Expand All @@ -2626,28 +2632,22 @@ private void checkMasterProcedureExecutor() throws ServiceException {
public MasterProtos.AssignsResponse assigns(RpcController controller,
MasterProtos.AssignsRequest request) throws ServiceException {
checkMasterProcedureExecutor();
final ProcedureExecutor<MasterProcedureEnv> pe = server.getMasterProcedureExecutor();
final AssignmentManager am = server.getAssignmentManager();
MasterProtos.AssignsResponse.Builder responseBuilder =
MasterProtos.AssignsResponse.newBuilder();
try {
boolean override = request.getOverride();
LOG.info("{} assigns, override={}", server.getClientIdAuditPrefix(), override);
for (HBaseProtos.RegionSpecifier rs : request.getRegionList()) {
long pid = Procedure.NO_PROC_ID;
RegionInfo ri = getRegionInfo(rs);
if (ri == null) {
LOG.info("Unknown={}", rs);
} else {
Procedure p = this.server.getAssignmentManager().createOneAssignProcedure(ri, override);
if (p != null) {
pid = this.server.getMasterProcedureExecutor().submitProcedure(p);
}
}
responseBuilder.addPid(pid);
final boolean override = request.getOverride();
LOG.info("{} assigns, override={}", server.getClientIdAuditPrefix(), override);
for (HBaseProtos.RegionSpecifier rs : request.getRegionList()) {
final RegionInfo info = getRegionInfo(rs);
if (info == null) {
LOG.info("Unknown region {}", rs);
continue;
}
return responseBuilder.build();
} catch (IOException ioe) {
throw new ServiceException(ioe);
responseBuilder.addPid(Optional.ofNullable(am.createOneAssignProcedure(info, override))
.map(pe::submitProcedure).orElse(Procedure.NO_PROC_ID));
}
return responseBuilder.build();
}

/**
Expand All @@ -2659,35 +2659,29 @@ public MasterProtos.AssignsResponse assigns(RpcController controller,
public MasterProtos.UnassignsResponse unassigns(RpcController controller,
MasterProtos.UnassignsRequest request) throws ServiceException {
checkMasterProcedureExecutor();
final ProcedureExecutor<MasterProcedureEnv> pe = server.getMasterProcedureExecutor();
final AssignmentManager am = server.getAssignmentManager();
MasterProtos.UnassignsResponse.Builder responseBuilder =
MasterProtos.UnassignsResponse.newBuilder();
try {
boolean override = request.getOverride();
LOG.info("{} unassigns, override={}", server.getClientIdAuditPrefix(), override);
for (HBaseProtos.RegionSpecifier rs : request.getRegionList()) {
long pid = Procedure.NO_PROC_ID;
RegionInfo ri = getRegionInfo(rs);
if (ri == null) {
LOG.info("Unknown={}", rs);
} else {
Procedure p = this.server.getAssignmentManager().createOneUnassignProcedure(ri, override);
if (p != null) {
pid = this.server.getMasterProcedureExecutor().submitProcedure(p);
}
}
responseBuilder.addPid(pid);
final boolean override = request.getOverride();
LOG.info("{} unassigns, override={}", server.getClientIdAuditPrefix(), override);
for (HBaseProtos.RegionSpecifier rs : request.getRegionList()) {
final RegionInfo info = getRegionInfo(rs);
if (info == null) {
LOG.info("Unknown region {}", rs);
continue;
}
return responseBuilder.build();
} catch (IOException ioe) {
throw new ServiceException(ioe);
responseBuilder.addPid(Optional.ofNullable(am.createOneUnassignProcedure(info, override))
.map(pe::submitProcedure).orElse(Procedure.NO_PROC_ID));
}
return responseBuilder.build();
}

/**
* Bypass specified procedure to completion. Procedure is marked completed but no actual work is
* done from the current state/ step onwards. Parents of the procedure are also marked for bypass.
* NOTE: this is a dangerous operation and may be used to unstuck buggy procedures. This may leave
* system in inconherent state. This may need to be followed by some cleanup steps/ actions by
* system in incoherent state. This may need to be followed by some cleanup steps/ actions by
* operator.
* @return BypassProcedureToCompletionResponse indicating success or failure
*/
Expand Down Expand Up @@ -3370,15 +3364,10 @@ public HBaseProtos.LogEntry getLogEntries(RpcController controller,
@QosPriority(priority = HConstants.ADMIN_QOS)
public GetRegionInfoResponse getRegionInfo(final RpcController controller,
final GetRegionInfoRequest request) throws ServiceException {
RegionInfo ri = null;
try {
ri = getRegionInfo(request.getRegion());
} catch (UnknownRegionException ure) {
throw new ServiceException(ure);
}
GetRegionInfoResponse.Builder builder = GetRegionInfoResponse.newBuilder();
if (ri != null) {
builder.setRegionInfo(ProtobufUtil.toRegionInfo(ri));
final GetRegionInfoResponse.Builder builder = GetRegionInfoResponse.newBuilder();
final RegionInfo info = getRegionInfo(request.getRegion());
if (info != null) {
builder.setRegionInfo(ProtobufUtil.toRegionInfo(info));
} else {
// Is it a MOB name? These work differently.
byte[] regionName = request.getRegion().getValue().toByteArray();
Expand Down
Loading

0 comments on commit 22dbb7a

Please sign in to comment.