HBASE-24586 Add table level locality in table.jsp

Closes #2048 Signed-off-by: Viraj Jasani <[email protected]>
apache · Jul 11, 2020 · fce1df9 · fce1df9
1 parent deff4c8
commit fce1df9
Show file tree

Hide file tree

Showing 8 changed files with 164 additions and 13 deletions.
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java
@@ -375,6 +375,21 @@ public float getDataLocalityForSsd() {
     return metrics.getDataLocalityForSsd();
   }
 
+  @Override
+  public long getBlocksLocalWeight() {
+    return metrics.getBlocksLocalWeight();
+  }
+
+  @Override
+  public long getBlocksLocalWithSsdWeight() {
+    return metrics.getBlocksLocalWithSsdWeight();
+  }
+
+  @Override
+  public long getBlocksTotalWeight() {
+    return metrics.getBlocksTotalWeight();
+  }
+
   /**
    * @see java.lang.Object#toString()
    */

diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java
@@ -160,4 +160,20 @@ default String getNameAsString() {
    * @return the data locality for ssd of region in the regionserver
    */
   float getDataLocalityForSsd();
+
+  /**
+   * @return the data at local weight of this region in the regionserver
+   */
+  long getBlocksLocalWeight();
+
+  /**
+   * Different from blocksLocalWeight,this metric's numerator only include the data stored on ssd
+   * @return the data at local with ssd weight of this region in the regionserver
+   */
+  long getBlocksLocalWithSsdWeight();
+
+  /**
+   * @return the block total weight of this region
+   */
+  long getBlocksTotalWeight();
 }
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java
@@ -53,6 +53,11 @@ public static RegionMetrics toRegionMetrics(ClusterStatusProtos.RegionLoad regio
         .setDataLocality(regionLoadPB.hasDataLocality() ? regionLoadPB.getDataLocality() : 0.0f)
         .setDataLocalityForSsd(regionLoadPB.hasDataLocalityForSsd() ?
           regionLoadPB.getDataLocalityForSsd() : 0.0f)
+        .setBlocksLocalWeight(regionLoadPB.hasBlocksLocalWeight() ?
+          regionLoadPB.getBlocksLocalWeight() : 0)
+        .setBlocksLocalWithSsdWeight(regionLoadPB.hasBlocksLocalWithSsdWeight() ?
+          regionLoadPB.getBlocksLocalWithSsdWeight() : 0)
+        .setBlocksTotalWeight(regionLoadPB.getBlocksTotalWeight())
         .setFilteredReadRequestCount(regionLoadPB.getFilteredReadRequestsCount())
         .setStoreFileUncompressedDataIndexSize(new Size(regionLoadPB.getTotalStaticIndexSizeKB(),
           Size.Unit.KILOBYTE))
@@ -148,6 +153,9 @@ public static RegionMetricsBuilder newBuilder(byte[] name) {
   private float dataLocality;
   private long lastMajorCompactionTimestamp;
   private float dataLocalityForSsd;
+  private long blocksLocalWeight;
+  private long blocksLocalWithSsdWeight;
+  private long blocksTotalWeight;
   private RegionMetricsBuilder(byte[] name) {
     this.name = name;
   }
@@ -236,7 +244,18 @@ public RegionMetricsBuilder setDataLocalityForSsd(float value) {
     this.dataLocalityForSsd = value;
     return this;
   }
-
+  public RegionMetricsBuilder setBlocksLocalWeight(long value) {
+    this.blocksLocalWeight = value;
+    return this;
+  }
+  public RegionMetricsBuilder setBlocksLocalWithSsdWeight(long value) {
+    this.blocksLocalWithSsdWeight = value;
+    return this;
+  }
+  public RegionMetricsBuilder setBlocksTotalWeight(long value) {
+    this.blocksTotalWeight = value;
+    return this;
+  }
   public RegionMetrics build() {
     return new RegionMetricsImpl(name,
         storeCount,
@@ -259,7 +278,10 @@ public RegionMetrics build() {
         storeSequenceIds,
         dataLocality,
         lastMajorCompactionTimestamp,
-        dataLocalityForSsd);
+        dataLocalityForSsd,
+        blocksLocalWeight,
+        blocksLocalWithSsdWeight,
+        blocksTotalWeight);
   }
 
   private static class RegionMetricsImpl implements RegionMetrics {
@@ -285,6 +307,9 @@ private static class RegionMetricsImpl implements RegionMetrics {
     private final float dataLocality;
     private final long lastMajorCompactionTimestamp;
     private final float dataLocalityForSsd;
+    private final long blocksLocalWeight;
+    private final long blocksLocalWithSsdWeight;
+    private final long blocksTotalWeight;
     RegionMetricsImpl(byte[] name,
         int storeCount,
         int storeFileCount,
@@ -306,7 +331,10 @@ private static class RegionMetricsImpl implements RegionMetrics {
         Map<byte[], Long> storeSequenceIds,
         float dataLocality,
         long lastMajorCompactionTimestamp,
-        float dataLocalityForSsd) {
+        float dataLocalityForSsd,
+        long blocksLocalWeight,
+        long blocksLocalWithSsdWeight,
+        long blocksTotalWeight) {
       this.name = Preconditions.checkNotNull(name);
       this.storeCount = storeCount;
       this.storeFileCount = storeFileCount;
@@ -329,6 +357,9 @@ private static class RegionMetricsImpl implements RegionMetrics {
       this.dataLocality = dataLocality;
       this.lastMajorCompactionTimestamp = lastMajorCompactionTimestamp;
       this.dataLocalityForSsd = dataLocalityForSsd;
+      this.blocksLocalWeight = blocksLocalWeight;
+      this.blocksLocalWithSsdWeight = blocksLocalWithSsdWeight;
+      this.blocksTotalWeight = blocksTotalWeight;
     }
 
     @Override
@@ -441,6 +472,21 @@ public float getDataLocalityForSsd() {
       return dataLocalityForSsd;
     }
 
+    @Override
+    public long getBlocksLocalWeight() {
+      return blocksLocalWeight;
+    }
+
+    @Override
+    public long getBlocksLocalWithSsdWeight() {
+      return blocksLocalWithSsdWeight;
+    }
+
+    @Override
+    public long getBlocksTotalWeight() {
+      return blocksTotalWeight;
+    }
+
     @Override
     public String toString() {
       StringBuilder sb = Strings.appendKeyValue(new StringBuilder(), "storeCount",
@@ -492,6 +538,12 @@ public String toString() {
           this.getDataLocality());
       Strings.appendKeyValue(sb, "dataLocalityForSsd",
           this.getDataLocalityForSsd());
+      Strings.appendKeyValue(sb, "blocksLocalWeight",
+        blocksLocalWeight);
+      Strings.appendKeyValue(sb, "blocksLocalWithSsdWeight",
+        blocksLocalWithSsdWeight);
+      Strings.appendKeyValue(sb, "blocksTotalWeight",
+        blocksTotalWeight);
       return sb.toString();
     }
   }

diff --git a/hbase-protocol-shaded/src/main/protobuf/ClusterStatus.proto b/hbase-protocol-shaded/src/main/protobuf/ClusterStatus.proto
@@ -158,6 +158,15 @@ message RegionLoad {
 
   /** The current data locality for ssd for region in the regionserver */
   optional float data_locality_for_ssd = 23;
+
+  /** The current blocks local weight for region in the regionserver */
+  optional uint64 blocks_local_weight = 24;
+
+  /** The current blocks local weight with ssd for region in the regionserver */
+  optional uint64 blocks_local_with_ssd_weight = 25;
+
+  /** The current blocks total weight for region in the regionserver */
+  optional uint64 blocks_total_weight = 26;
 }
 
 message UserLoad {

diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/HDFSBlocksDistribution.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/HDFSBlocksDistribution.java
@@ -51,7 +51,7 @@ public class HDFSBlocksDistribution {
    */
   public static class HostAndWeight {
 
-    private String host;
+    private final String host;
     private long weight;
     private long weightForSsd;
 
@@ -228,33 +228,57 @@ public long getUniqueBlocksTotalWeight() {
    * Implementations 'visit' hostAndWeight.
    */
   public interface Visitor {
-    float visit(final HostAndWeight hostAndWeight);
+    long visit(final HostAndWeight hostAndWeight);
   }
 
   /**
    * @param host the host name
    * @return the locality index of the given host
    */
   public float getBlockLocalityIndex(String host) {
-    return getBlockLocalityIndexInternal(host,
-      e -> (float) e.weight / (float) uniqueBlocksTotalWeight);
+    if (uniqueBlocksTotalWeight == 0) {
+      return 0.0f;
+    } else {
+      return (float) getBlocksLocalityWeightInternal(host, HostAndWeight::getWeight)
+        / (float) uniqueBlocksTotalWeight;
+    }
   }
 
   /**
    * @param host the host name
    * @return the locality index with ssd of the given host
    */
   public float getBlockLocalityIndexForSsd(String host) {
-    return getBlockLocalityIndexInternal(host,
-      e -> (float) e.weightForSsd / (float) uniqueBlocksTotalWeight);
+    if (uniqueBlocksTotalWeight == 0) {
+      return 0.0f;
+    } else {
+      return (float) getBlocksLocalityWeightInternal(host, HostAndWeight::getWeightForSsd)
+        / (float) uniqueBlocksTotalWeight;
+    }
+  }
+
+  /**
+   * @param host the host name
+   * @return the blocks local weight of the given host
+   */
+  public long getBlocksLocalWeight(String host) {
+    return getBlocksLocalityWeightInternal(host, HostAndWeight::getWeight);
+  }
+
+  /**
+   * @param host the host name
+   * @return the blocks local with ssd weight of the given host
+   */
+  public long getBlocksLocalWithSsdWeight(String host) {
+    return getBlocksLocalityWeightInternal(host, HostAndWeight::getWeightForSsd);
   }
 
   /**
    * @param host the host name
    * @return the locality index of the given host
    */
-  private float getBlockLocalityIndexInternal(String host, Visitor visitor) {
-    float localityIndex = 0;
+  private long getBlocksLocalityWeightInternal(String host, Visitor visitor) {
+    long localityIndex = 0;
     HostAndWeight hostAndWeight = this.hostAndWeights.get(host);
     if (hostAndWeight != null && uniqueBlocksTotalWeight != 0) {
       localityIndex = visitor.visit(hostAndWeight);

diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1708,6 +1708,9 @@ RegionLoad createRegionLoad(final HRegion r, RegionLoad.Builder regionLoadBldr,
     HDFSBlocksDistribution hdfsBd = r.getHDFSBlocksDistribution();
     float dataLocality = hdfsBd.getBlockLocalityIndex(serverName.getHostname());
     float dataLocalityForSsd = hdfsBd.getBlockLocalityIndexForSsd(serverName.getHostname());
+    long blocksTotalWeight = hdfsBd.getUniqueBlocksTotalWeight();
+    long blocksLocalWeight = hdfsBd.getBlocksLocalWeight(serverName.getHostname());
+    long blocksLocalWithSsdWeight = hdfsBd.getBlocksLocalWithSsdWeight(serverName.getHostname());
     if (regionLoadBldr == null) {
       regionLoadBldr = RegionLoad.newBuilder();
     }
@@ -1735,6 +1738,9 @@ RegionLoad createRegionLoad(final HRegion r, RegionLoad.Builder regionLoadBldr,
       .setCurrentCompactedKVs(currentCompactedKVs)
       .setDataLocality(dataLocality)
       .setDataLocalityForSsd(dataLocalityForSsd)
+      .setBlocksLocalWeight(blocksLocalWeight)
+      .setBlocksLocalWithSsdWeight(blocksLocalWithSsdWeight)
+      .setBlocksTotalWeight(blocksTotalWeight)
       .setLastMajorCompactionTs(r.getOldestHfileTs(true));
     r.setCompleteSequenceId(regionLoadBldr);
 

diff --git a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
@@ -763,9 +763,14 @@
       long totalMemSize = 0;
       long totalCompactingCells = 0;
       long totalCompactedCells = 0;
+      long totalBlocksTotalWeight = 0;
+      long totalBlocksLocalWeight = 0;
+      long totalBlocksLocalWithSsdWeight = 0;
       String totalCompactionProgress = "";
       String totalMemSizeStr = ZEROMB;
       String totalSizeStr = ZEROMB;
+      String totalLocality = "";
+      String totalLocalityForSsd = "";
       String urlRegionServer = null;
       Map<ServerName, Integer> regDistribution = new TreeMap<>();
       Map<ServerName, Integer> primaryRegDistribution = new TreeMap<>();
@@ -791,6 +796,9 @@
               totalStoreFileSizeMB += regionMetrics.getStoreFileSize().get(Size.Unit.MEGABYTE);
               totalCompactingCells += regionMetrics.getCompactingCellCount();
               totalCompactedCells += regionMetrics.getCompactedCellCount();
+              totalBlocksTotalWeight += regionMetrics.getBlocksTotalWeight();
+              totalBlocksLocalWeight += regionMetrics.getBlocksLocalWeight();
+              totalBlocksLocalWithSsdWeight += regionMetrics.getBlocksLocalWithSsdWeight();
             } else {
               RegionMetrics load0 = getEmptyRegionMetrics(regionInfo);
               regionsToLoad.put(regionInfo, load0);
@@ -814,6 +822,12 @@
         totalCompactionProgress = String.format("%.2f", 100 *
                 ((float) totalCompactedCells / totalCompactingCells)) + "%";
       }
+      if (totalBlocksTotalWeight > 0) {
+        totalLocality = String.format("%.1f",
+          ((float) totalBlocksLocalWeight / totalBlocksTotalWeight));
+        totalLocalityForSsd = String.format("%.1f",
+          ((float) totalBlocksLocalWithSsdWeight / totalBlocksTotalWeight));
+      }
       if(regions != null && regions.size() > 0) { %>
     <h2>Table Regions</h2>
     <div class="tabbable">
@@ -955,8 +969,8 @@
               <tr>
                 <th>Name(<%= String.format("%,1d", regions.size())%>)</th>
                 <th>Region Server</th>
-                <th>Locality</th>
-                <th>LocalityForSsd</th>
+                <th>Locality<br>(<%= totalLocality %>)</th>
+                <th>LocalityForSsd<br>(<%= totalLocalityForSsd %>)</th>
               </tr>
             </thead>
             <tbody>

diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionsRecoveryChore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionsRecoveryChore.java
@@ -492,6 +492,21 @@ public int getMaxCompactedStoreFileRefCount() {
       public float getDataLocalityForSsd() {
         return 0;
       }
+
+      @Override
+      public long getBlocksLocalWeight() {
+        return 0;
+      }
+
+      @Override
+      public long getBlocksLocalWithSsdWeight() {
+        return 0;
+      }
+
+      @Override
+      public long getBlocksTotalWeight() {
+        return 0;
+      }
     };
     return regionMetrics;
   }