HBASE-22777 Add a multi-region merge (for fixing overlaps)

Makes MergeTableRegionsProcedure do more than just two regions at a
time. Compatible as MTRP was done considering one day it'd do more than
two at a time.

Changes hardcoded assumption that merge parent regions are named
mergeA and mergeB in a column on the resultant region. Instead
can have N columns on the merged region, one for each parent
merged. Column qualifiers all being with 'merge'.

Most of code below is undoing the assumption that there are two
parents on a merge only.

hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionInfo.java

@@ -1,4 +1,4 @@
-/**
+/*
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
@@ -70,8 +70,7 @@
  */
 @InterfaceAudience.Public
 public interface RegionInfo {
-  public static final RegionInfo UNDEFINED =
-      RegionInfoBuilder.newBuilder(TableName.valueOf("__UNDEFINED__")).build();
+  RegionInfo UNDEFINED = RegionInfoBuilder.newBuilder(TableName.valueOf("__UNDEFINED__")).build();
   /**
    * Separator used to demarcate the encodedName in a region name
    * in the new format. See description on new format above.
@@ -141,11 +140,16 @@ public interface RegionInfo {
       }
 
       int replicaDiff = lhs.getReplicaId() - rhs.getReplicaId();
-      if (replicaDiff != 0) return replicaDiff;
+      if (replicaDiff != 0) {
+        return replicaDiff;
+      }
 
-      if (lhs.isOffline() == rhs.isOffline())
+      if (lhs.isOffline() == rhs.isOffline()) {
         return 0;
-      if (lhs.isOffline() == true) return -1;
+      }
+      if (lhs.isOffline()) {
+        return -1;
+      }
 
       return 1;
   };
@@ -224,8 +228,6 @@ public interface RegionInfo {
   boolean isMetaRegion();
 
   /**
-   * @param rangeStartKey
-   * @param rangeEndKey
    * @return true if the given inclusive range of rows is fully contained
    * by this region. For example, if the region is foo,a,g and this is
    * passed ["b","c"] or ["a","c"] it will return true, but if this is passed
@@ -235,7 +237,6 @@ public interface RegionInfo {
   boolean containsRange(byte[] rangeStartKey, byte[] rangeEndKey);
 
   /**
-   * @param row
    * @return true if the given row falls in this region.
    */
   boolean containsRow(byte[] row);
@@ -339,9 +340,7 @@ static TableName getTable(final byte [] regionName) {
 
   /**
    * Gets the start key from the specified region name.
-   * @param regionName
    * @return Start key.
-   * @throws java.io.IOException
    */
   static byte[] getStartKey(final byte[] regionName) throws IOException {
     return parseRegionName(regionName)[1];
@@ -362,7 +361,6 @@ static boolean isEncodedRegionName(byte[] regionName) throws IOException {
   }
 
   /**
-   * @param bytes
    * @return A deserialized {@link RegionInfo}
    * or null if we failed deserialize or passed bytes null
    */
@@ -373,9 +371,6 @@ static RegionInfo parseFromOrNull(final byte [] bytes) {
   }
 
   /**
-   * @param bytes
-   * @param offset
-   * @param len
    * @return A deserialized {@link RegionInfo} or null
    *  if we failed deserialize or passed bytes null
    */
@@ -392,7 +387,6 @@ static RegionInfo parseFromOrNull(final byte [] bytes, int offset, int len) {
   /**
    * @param bytes A pb RegionInfo serialized with a pb magic prefix.
    * @return A deserialized {@link RegionInfo}
-   * @throws DeserializationException
    */
   @InterfaceAudience.Private
   static RegionInfo parseFrom(final byte [] bytes) throws DeserializationException {
@@ -405,7 +399,6 @@ static RegionInfo parseFrom(final byte [] bytes) throws DeserializationException
    * @param offset starting point in the byte array
    * @param len length to read on the byte array
    * @return A deserialized {@link RegionInfo}
-   * @throws DeserializationException
    */
   @InterfaceAudience.Private
   static RegionInfo parseFrom(final byte [] bytes, int offset, int len)
@@ -426,30 +419,28 @@ static RegionInfo parseFrom(final byte [] bytes, int offset, int len)
   }
 
   /**
-   * Check whether two regions are adjacent
-   * @param regionA
-   * @param regionB
+   * Check whether two regions are adjacent; i.e. lies just before or just
+   * after in a table.
    * @return true if two regions are adjacent
    */
   static boolean areAdjacent(RegionInfo regionA, RegionInfo regionB) {
     if (regionA == null || regionB == null) {
       throw new IllegalArgumentException(
       "Can't check whether adjacent for null region");
     }
+    if (!regionA.getTable().equals(regionB.getTable())) {
+      return false;
+    }
     RegionInfo a = regionA;
     RegionInfo b = regionB;
     if (Bytes.compareTo(a.getStartKey(), b.getStartKey()) > 0) {
       a = regionB;
       b = regionA;
     }
-    if (Bytes.compareTo(a.getEndKey(), b.getStartKey()) == 0) {
-      return true;
-    }
-    return false;
+    return Bytes.equals(a.getEndKey(), b.getStartKey());
   }
 
   /**
-   * @param ri
    * @return This instance serialized as protobuf w/ a magic pb prefix.
    * @see #parseFrom(byte[])
    */
@@ -473,7 +464,6 @@ static String prettyPrint(final String encodedRegionName) {
 
   /**
    * Make a region name of passed parameters.
-   * @param tableName
    * @param startKey Can be null
    * @param regionid Region id (Usually timestamp from when region was created).
    * @param newFormat should we create the region name in the new format
@@ -487,7 +477,6 @@ static String prettyPrint(final String encodedRegionName) {
 
   /**
    * Make a region name of passed parameters.
-   * @param tableName
    * @param startKey Can be null
    * @param id Region id (Usually timestamp from when region was created).
    * @param newFormat should we create the region name in the new format
@@ -501,10 +490,8 @@ static String prettyPrint(final String encodedRegionName) {
 
   /**
    * Make a region name of passed parameters.
-   * @param tableName
    * @param startKey Can be null
    * @param regionid Region id (Usually timestamp from when region was created).
-   * @param replicaId
    * @param newFormat should we create the region name in the new format
    *                  (such that it contains its encoded name?).
    * @return Region name made of passed tableName, startKey, id and replicaId
@@ -517,7 +504,6 @@ static String prettyPrint(final String encodedRegionName) {
 
   /**
    * Make a region name of passed parameters.
-   * @param tableName
    * @param startKey Can be null
    * @param id Region id (Usually timestamp from when region was created).
    * @param newFormat should we create the region name in the new format
@@ -531,10 +517,8 @@ static String prettyPrint(final String encodedRegionName) {
 
   /**
    * Make a region name of passed parameters.
-   * @param tableName
    * @param startKey Can be null
    * @param id Region id (Usually timestamp from when region was created).
-   * @param replicaId
    * @param newFormat should we create the region name in the new format
    * @return Region name made of passed tableName, startKey, id and replicaId
    */
@@ -593,7 +577,7 @@ static String prettyPrint(final String encodedRegionName) {
       b[offset++] = ENC_SEPARATOR;
       System.arraycopy(md5HashBytes, 0, b, offset, MD5_HEX_LENGTH);
       offset += MD5_HEX_LENGTH;
-      b[offset++] = ENC_SEPARATOR;
+      b[offset] = ENC_SEPARATOR;
     }
 
     return b;
@@ -612,9 +596,7 @@ static RegionInfo createMobRegionInfo(TableName tableName) {
 
   /**
    * Separate elements of a regionName.
-   * @param regionName
    * @return Array of byte[] containing tableName, startKey and id
-   * @throws IOException
    */
   static byte [][] parseRegionName(final byte[] regionName)
   throws IOException {
@@ -693,7 +675,6 @@ static RegionInfo createMobRegionInfo(TableName tableName) {
    * be used to read back the instances.
    * @param infos RegionInfo objects to serialize
    * @return This instance serialized as a delimited protobuf w/ a magic pb prefix.
-   * @throws IOException
    */
   static byte[] toDelimitedByteArray(RegionInfo... infos) throws IOException {
     byte[][] bytes = new byte[infos.length][];
@@ -715,9 +696,7 @@ static byte[] toDelimitedByteArray(RegionInfo... infos) throws IOException {
   /**
    * Use this instead of {@link RegionInfo#toByteArray(RegionInfo)} when writing to a stream and you want to use
    * the pb mergeDelimitedFrom (w/o the delimiter, pb reads to EOF which may not be what you want).
-   * @param ri
    * @return This instance serialized as a delimied protobuf w/ a magic pb prefix.
-   * @throws IOException
    */
   static byte [] toDelimitedByteArray(RegionInfo ri) throws IOException {
     return ProtobufUtil.toDelimitedByteArray(ProtobufUtil.toRegionInfo(ri));
@@ -727,9 +706,7 @@ static byte[] toDelimitedByteArray(RegionInfo... infos) throws IOException {
    * Parses an RegionInfo instance from the passed in stream.
    * Presumes the RegionInfo was serialized to the stream with
    * {@link #toDelimitedByteArray(RegionInfo)}.
-   * @param in
    * @return An instance of RegionInfo.
-   * @throws IOException
    */
   static RegionInfo parseFrom(final DataInputStream in) throws IOException {
     // I need to be able to move back in the stream if this is not a pb
@@ -757,28 +734,23 @@ static RegionInfo parseFrom(final DataInputStream in) throws IOException {
    * @param offset the start offset into the byte[] buffer
    * @param length how far we should read into the byte[] buffer
    * @return All the RegionInfos that are in the byte array. Keeps reading till we hit the end.
-   * @throws IOException
    */
   static List<RegionInfo> parseDelimitedFrom(final byte[] bytes, final int offset,
                                              final int length) throws IOException {
     if (bytes == null) {
       throw new IllegalArgumentException("Can't build an object with empty bytes array");
     }
-    DataInputBuffer in = new DataInputBuffer();
     List<RegionInfo> ris = new ArrayList<>();
-    try {
+    try (DataInputBuffer in = new DataInputBuffer()) {
       in.reset(bytes, offset, length);
       while (in.available() > 0) {
         RegionInfo ri = parseFrom(in);
         ris.add(ri);
       }
-    } finally {
-      in.close();
     }
     return ris;
   }
 
-
   /**
    * @return True if this is first Region in Table
    */
@@ -794,10 +766,20 @@ default boolean isLast() {
   }
 
   /**
-   * @return True if regions are adjacent, if 'after' next. Does not do tablename compare.
+   * @return True if region is next, adjacent but 'after' this one.
+   * @see #isAdjacent(RegionInfo)
+   * @see #areAdjacent(RegionInfo, RegionInfo)
    */
   default boolean isNext(RegionInfo after) {
-    return Bytes.equals(getEndKey(), after.getStartKey());
+    return getTable().equals(after.getTable()) && Bytes.equals(getEndKey(), after.getStartKey());
+  }
+
+  /**
+   * @return True if region is adjacent, either just before or just after this one.
+   * @see #isNext(RegionInfo)
+   */
+  default boolean isAdjacent(RegionInfo other) {
+    return getTable().equals(other.getTable()) && areAdjacent(this, other);
   }
 
   /**
@@ -808,11 +790,13 @@ default boolean isDegenerate() {
   }
 
   /**
-   * @return True if an overlap in region range. Does not do tablename compare.
-   *   Does not check if <code>other</code> has degenerate range.
+   * @return True if an overlap in region range.
    * @see #isDegenerate()
    */
   default boolean isOverlap(RegionInfo other) {
+    if (!getTable().equals(other.getTable())) {
+      return false;
+    }
     int startKeyCompare = Bytes.compareTo(getStartKey(), other.getStartKey());
     if (startKeyCompare == 0) {
       return true;

hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java

@@ -527,11 +527,31 @@ public enum OperationStatusCode {
   /** The upper-half split region column qualifier */
   public static final byte [] SPLITB_QUALIFIER = Bytes.toBytes("splitB");
 
-  /** The lower-half merge region column qualifier */
-  public static final byte[] MERGEA_QUALIFIER = Bytes.toBytes("mergeA");
+  /**
+   * Merge qualifier prefix.
+   * We used to only allow two regions merge; mergeA and mergeB.
+   * Now we allow many to merge. Each region to merge will be referenced
+   * in a column whose qualifier starts with this define.
+   */
+  public static final String MERGE_QUALIFIER_PREFIX_STR = "merge";
+  public static final byte [] MERGE_QUALIFIER_PREFIX =
+      Bytes.toBytes(MERGE_QUALIFIER_PREFIX_STR);
 
-  /** The upper-half merge region column qualifier */
-  public static final byte[] MERGEB_QUALIFIER = Bytes.toBytes("mergeB");
+  /**
+   * The lower-half merge region column qualifier
+   * @deprecated Since 2.3.0 and 2.2.1. Not used anymore. Instead we look for
+   *   the {@link #MERGE_QUALIFIER_PREFIX_STR} prefix.
+   */
+  @Deprecated
+  public static final byte[] MERGEA_QUALIFIER = Bytes.toBytes(MERGE_QUALIFIER_PREFIX_STR + "A");
+
+  /**
+   * The upper-half merge region column qualifier
+   * @deprecated Since 2.3.0 and 2.2.1. Not used anymore. Instead we look for
+   *   the {@link #MERGE_QUALIFIER_PREFIX_STR} prefix.
+   */
+  @Deprecated
+  public static final byte[] MERGEB_QUALIFIER = Bytes.toBytes(MERGE_QUALIFIER_PREFIX_STR + "B");
 
   /** The catalog family as a string*/
   public static final String TABLE_FAMILY_STR = "table";

hbase-common/src/main/java/org/apache/hadoop/hbase/PrivateCellUtil.java

@@ -757,6 +757,25 @@ public static boolean matchingQualifier(final Cell left, final byte[] buf, final
       left.getQualifierLength(), buf, offset, length);
   }
 
+  /**
+   * Finds if the start of the qualifier part of the Cell matches <code>buf</code>
+   * @param left the cell with which we need to match the qualifier
+   * @param startsWith the serialized keyvalue format byte[]
+   * @return true if the qualifier have same staring characters, false otherwise
+   */
+  public static boolean qualifierStartsWith(final Cell left, final byte[] startsWith) {
+    if (startsWith == null || startsWith.length == 0) {
+      throw new IllegalArgumentException("Cannot pass an empty startsWith");
+    }
+    if (left instanceof ByteBufferExtendedCell) {
+      return ByteBufferUtils.equals(((ByteBufferExtendedCell) left).getQualifierByteBuffer(),
+          ((ByteBufferExtendedCell) left).getQualifierPosition(), startsWith.length,
+          startsWith, 0, startsWith.length);
+    }
+    return Bytes.equals(left.getQualifierArray(), left.getQualifierOffset(),
+        startsWith.length, startsWith, 0, startsWith.length);
+  }
+
   public static boolean matchingColumn(final Cell left, final byte[] fam, final int foffset,
       final int flength, final byte[] qual, final int qoffset, final int qlength) {
     if (!matchingFamily(left, fam, foffset, flength)) {

hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto

@@ -375,18 +375,27 @@ message GCRegionStateData {
   required RegionInfo region_info = 1;
 }
 
+// NOTE: This message is used by GCMergedRegionStateProcedure
+// AND GCMultipleMergedRegionStateProcedure.
 enum GCMergedRegionsState {
   GC_MERGED_REGIONS_PREPARE = 1;
   GC_MERGED_REGIONS_PURGE = 2;
   GC_REGION_EDIT_METADATA = 3;
 }
 
 message GCMergedRegionsStateData {
+  // Use GCMultipleMergedRegionsStateData instead.
+  option deprecated = true;
   required RegionInfo parent_a = 1;
   required RegionInfo parent_b = 2;
   required RegionInfo merged_child = 3;
 }
 
+message GCMultipleMergedRegionsStateData {
+  repeated RegionInfo parents = 1;
+  required RegionInfo merged_child = 2;
+}
+
 enum PeerModificationState {
   PRE_PEER_MODIFICATION = 1;
   UPDATE_PEER_STORAGE = 2;
@@ -544,4 +553,4 @@ enum SplitWALState {
   ACQUIRE_SPLIT_WAL_WORKER = 1;
   DISPATCH_WAL_TO_WORKER = 2;
   RELEASE_SPLIT_WORKER = 3;
-}
+}