Skip to content

Commit

Permalink
HBASE-22771 [HBCK2] fixMeta method and server-side support
Browse files Browse the repository at this point in the history 
This is a first cut at this patch. Implements hold fixing only
currently.

Add a fixMeta method to Hbck Interface.

M hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
 Bug fix. If hole is on end of last table, I wasn't seeing it.

A hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetaFixer.java
 Add a general meta fixer class. Explains up top why this stuff doesn't
 belong inside MetaTableAccessor.

M hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
 Break out the filesystem messing so don't have to copy it nor do more
 than is needed doing fixup for Region holes.

M hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
 Change behavious slightly. If directory exists, don't fail as we did
 but try and keep going and create .regioninfo file if missing (or
 overwrite if in place). This should make it idempotent. Can rerun
 command. Lets see if any repercussions in test suite.

A hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMetaFixer.java
 Add test.

Signed-off-by: Zheng Hu <[email protected]>
Signed-off-by: Guanghao Zhang <[email protected]>
  • Loading branch information
@saintstack
saintstack committed Aug 6, 2019
1 parent 946f5e6 commit db440d3
Show file tree
Hide file tree
Showing 13 changed files with 422 additions and 97 deletions.
3 changes: 1 addition & 2 deletions hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1809,9 +1809,8 @@ private static void updateLocation(Connection connection, RegionInfo regionInfo,
*/
public static void deleteRegionInfo(Connection connection, RegionInfo regionInfo)
throws IOException {
long time = EnvironmentEdgeManager.currentTime();
Delete delete = new Delete(regionInfo.getRegionName());
delete.addFamily(getCatalogFamily(), time);
delete.addFamily(getCatalogFamily(), HConstants.LATEST_TIMESTAMP);
deleteFromMetaTable(connection, delete);
LOG.info("Deleted " + regionInfo.getRegionNameAsString());
}
Expand Down
10 changes: 10 additions & 0 deletions hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseHbck.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.AssignsResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.BypassProcedureRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.BypassProcedureResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.FixMetaRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetTableStateResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.HbckService.BlockingInterface;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RunHbckChoreRequest;
Expand Down Expand Up @@ -186,4 +187,13 @@ public boolean runHbckChore() throws IOException {
throw new IOException(se);
}
}

@Override
public void fixMeta() throws IOException {
try {
this.hbck.fixMeta(rpcControllerFactory.newController(), FixMetaRequest.newBuilder().build());
} catch (ServiceException se) {
throw new IOException(se);
}
}
}
5 changes: 5 additions & 0 deletions hbase-client/src/main/java/org/apache/hadoop/hbase/client/Hbck.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,4 +114,9 @@ List<Long> scheduleServerCrashProcedure(List<HBaseProtos.ServerName> serverNames
* @throws IOException if a remote or network exception occurs
*/
boolean runHbckChore() throws IOException;

/**
* Fix Meta.
*/
void fixMeta() throws IOException;
}
8 changes: 8 additions & 0 deletions hbase-protocol-shaded/src/main/protobuf/Master.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1057,6 +1057,10 @@ message ScheduleServerCrashProcedureResponse {
repeated uint64 pid = 1;
}

message FixMetaRequest {}

message FixMetaResponse {}

service HbckService {
/** Update state of the table in meta only*/
rpc SetTableStateInMeta(SetTableStateInMetaRequest)
Expand Down Expand Up @@ -1093,4 +1097,8 @@ service HbckService {
*/
rpc RunHbckChore(RunHbckChoreRequest)
returns(RunHbckChoreResponse);

/** Schedule a fix meta run. */
rpc FixMeta(FixMetaRequest)
returns(FixMetaResponse);
}
97 changes: 36 additions & 61 deletions hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -457,9 +457,9 @@ public static class Report {
final Map<RegionInfo, Result> mergedRegions = new TreeMap<>(RegionInfo.COMPARATOR);
int count = 0;

private final List<Pair<MetaRow, MetaRow>> holes = new ArrayList<>();
private final List<Pair<MetaRow, MetaRow>> overlaps = new ArrayList<>();
private final List<Pair<MetaRow, ServerName>> unknownServers = new ArrayList<>();
private final List<Pair<RegionInfo, RegionInfo>> holes = new ArrayList<>();
private final List<Pair<RegionInfo, RegionInfo>> overlaps = new ArrayList<>();
private final List<Pair<RegionInfo, ServerName>> unknownServers = new ArrayList<>();
private final List<byte []> emptyRegionInfo = new ArrayList<>();

@VisibleForTesting
Expand All @@ -469,15 +469,15 @@ public long getCreateTime() {
return this.now;
}

public List<Pair<MetaRow, MetaRow>> getHoles() {
public List<Pair<RegionInfo, RegionInfo>> getHoles() {
return this.holes;
}

public List<Pair<MetaRow, MetaRow>> getOverlaps() {
public List<Pair<RegionInfo, RegionInfo>> getOverlaps() {
return this.overlaps;
}

public List<Pair<MetaRow, ServerName>> getUnknownServers() {
public List<Pair<RegionInfo, ServerName>> getUnknownServers() {
return unknownServers;
}

Expand All @@ -496,72 +496,37 @@ public boolean isEmpty() {
@Override
public String toString() {
StringBuffer sb = new StringBuffer();
for (Pair<MetaRow, MetaRow> p: this.holes) {
for (Pair<RegionInfo, RegionInfo> p: this.holes) {
if (sb.length() > 0) {
sb.append(", ");
}
sb.append("hole=" + Bytes.toStringBinary(p.getFirst().metaRow) + "/" +
Bytes.toStringBinary(p.getSecond().metaRow));
sb.append("hole=" + p.getFirst().getRegionNameAsString() + "/" +
p.getSecond().getRegionNameAsString());
}
for (Pair<MetaRow, MetaRow> p: this.overlaps) {
for (Pair<RegionInfo, RegionInfo> p: this.overlaps) {
if (sb.length() > 0) {
sb.append(", ");
}
sb.append("overlap=").append(Bytes.toStringBinary(p.getFirst().metaRow)).append("/").
append(Bytes.toStringBinary(p.getSecond().metaRow));
sb.append("overlap=" + p.getFirst().getRegionNameAsString() + "/" +
p.getSecond().getRegionNameAsString());
}
for (byte [] r: this.emptyRegionInfo) {
if (sb.length() > 0) {
sb.append(", ");
}
sb.append("empty=").append(Bytes.toStringBinary(r));
}
for (Pair<MetaRow, ServerName> p: this.unknownServers) {
for (Pair<RegionInfo, ServerName> p: this.unknownServers) {
if (sb.length() > 0) {
sb.append(", ");
}
sb.append("unknown_server=").append(p.getSecond()).append("/").
append(Bytes.toStringBinary(p.getFirst().metaRow));
append(p.getFirst().getRegionNameAsString());
}
return sb.toString();
}
}

/**
* Simple datastructure to hold a MetaRow content.
*/
public static class MetaRow {
/**
* A marker for use in case where there is a hole at the very
* first row in hbase:meta. Should never happen.
*/
private static final MetaRow UNDEFINED =
new MetaRow(HConstants.EMPTY_START_ROW, RegionInfo.UNDEFINED);

/**
* Row from hbase:meta table.
*/
private final byte [] metaRow;

/**
* The decoded RegionInfo gotten from hbase:meta.
*/
private final RegionInfo regionInfo;

MetaRow(byte [] metaRow, RegionInfo regionInfo) {
this.metaRow = metaRow;
this.regionInfo = regionInfo;
}

public RegionInfo getRegionInfo() {
return regionInfo;
}

public byte[] getMetaRow() {
return metaRow;
}
}

/**
* Visitor we use in here in CatalogJanitor to go against hbase:meta table.
* Generates a Report made of a collection of split parents and counts of rows
Expand All @@ -583,7 +548,7 @@ static class ReportMakingVisitor implements MetaTableAccessor.CloseableVisitor {
/**
* RegionInfo from previous row.
*/
private MetaRow previous = null;
private RegionInfo previous = null;

ReportMakingVisitor(MasterServices services) {
this.services = services;
Expand Down Expand Up @@ -641,36 +606,41 @@ private RegionInfo metaTableConsistencyCheck(Result metaTableRow) {
return ri;
}

MetaRow mrri = new MetaRow(metaTableRow.getRow(), ri);
if (!Bytes.equals(metaTableRow.getRow(), ri.getRegionName())) {
LOG.warn("INCONSISTENCY: Row name is not equal to serialized info:regioninfo content; " +
"row={} {}; See if RegionInfo is referenced in another hbase:meta row? Delete?",
Bytes.toStringBinary(metaTableRow.getRow()), ri.getRegionNameAsString());
return null;
}
// If table is disabled, skip integrity check.
if (!isTableDisabled(ri)) {
if (isTableTransition(ri)) {
// On table transition, look to see if last region was last in table
// and if this is the first. Report 'hole' if neither is true.
// HBCK1 used to have a special category for missing start or end keys.
// We'll just lump them in as 'holes'.
if ((this.previous != null && !this.previous.regionInfo.isLast()) || !ri.isFirst()) {
addHole(this.previous == null? MetaRow.UNDEFINED: this.previous, mrri);
if ((this.previous != null && !this.previous.isLast()) || !ri.isFirst()) {
addHole(this.previous == null? RegionInfo.UNDEFINED: this.previous, ri);
}
} else {
if (!this.previous.regionInfo.isNext(ri)) {
if (this.previous.regionInfo.isOverlap(ri)) {
addOverlap(this.previous, mrri);
if (!this.previous.isNext(ri)) {
if (this.previous.isOverlap(ri)) {
addOverlap(this.previous, ri);
} else {
addHole(this.previous, mrri);
addHole(this.previous, ri);
}
}
}
}
this.previous = mrri;
this.previous = ri;
return ri;
}

private void addOverlap(MetaRow a, MetaRow b) {
private void addOverlap(RegionInfo a, RegionInfo b) {
this.report.overlaps.add(new Pair<>(a, b));
}

private void addHole(MetaRow a, MetaRow b) {
private void addHole(RegionInfo a, RegionInfo b) {
this.report.holes.add(new Pair<>(a, b));
}

Expand Down Expand Up @@ -701,11 +671,16 @@ boolean isTableDisabled(RegionInfo ri) {
*/
private boolean isTableTransition(RegionInfo ri) {
return this.previous == null ||
!this.previous.regionInfo.getTable().equals(ri.getTable());
!this.previous.getTable().equals(ri.getTable());
}

@Override
public void close() throws IOException {
// This is a table transition... after the last region. Check previous.
// Should be last region. If not, its a hole on end of laster table.
if (this.previous != null && !this.previous.isLast()) {
addHole(this.previous, RegionInfo.UNDEFINED);
}
this.closed = true;
}
}
Expand Down
14 changes: 14 additions & 0 deletions hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,8 @@
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.EnableTableResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ExecProcedureRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ExecProcedureResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.FixMetaResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.FixMetaRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetClusterStatusRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetClusterStatusResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetCompletedSnapshotsRequest;
Expand Down Expand Up @@ -2502,4 +2504,16 @@ private boolean shouldSubmitSCP(ServerName serverName) {
}
return true;
}

@Override
public FixMetaResponse fixMeta(RpcController controller, FixMetaRequest request)
throws ServiceException {
try {
MetaFixer mf = new MetaFixer(this.master);
mf.fix();
return FixMetaResponse.newBuilder().build();
} catch (IOException ioe) {
throw new ServiceException(ioe);
}
}
}
Loading

0 comments on commit db440d3

Please sign in to comment.