From f329ea0fd5756104f492c0935833f2752001159c Mon Sep 17 00:00:00 2001 From: Bryan Beaudreault Date: Fri, 19 Apr 2024 10:28:56 -0400 Subject: [PATCH] [HubSpot] Add region server metrics for local and remote rack bytes read (#92) --- .../MetricsRegionServerSource.java | 8 ++++++ .../MetricsRegionServerWrapper.java | 4 +++ .../MetricsRegionServerSourceImpl.java | 4 +++ .../MetricsRegionServerWrapperImpl.java | 25 +++++++++++++++++++ .../MetricsRegionServerWrapperStub.java | 10 ++++++++ 5 files changed, 51 insertions(+) diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java index 75269e571816..1587f004f45a 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java @@ -527,6 +527,14 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo String ZEROCOPY_BYTES_READ = "zeroCopyBytesRead"; String ZEROCOPY_BYTES_READ_DESC = "The number of bytes read through HDFS zero copy"; + String LOCAL_RACK_BYTES_READ = "localRackBytesRead"; + String LOCAL_RACK_BYTES_READ_DESC = + "The number of bytes read from the same rack of the RegionServer, but not the local HDFS DataNode"; + + String REMOTE_RACK_BYTES_READ = "remoteRackBytesRead"; + String REMOTE_RACK_BYTES_READ_DESC = + "The number of bytes read from a different rack from that of the RegionServer"; + String BLOCKED_REQUESTS_COUNT = "blockedRequestCount"; String BLOCKED_REQUESTS_COUNT_DESC = "The number of blocked requests because of memstore size is " + "larger than blockingMemStoreSize"; diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java index 1e5c6c409ed8..30507330aafa 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java @@ -539,6 +539,10 @@ public interface MetricsRegionServerWrapper { /** Returns Number of bytes read from the local HDFS DataNode. */ long getLocalBytesRead(); + long getLocalRackBytesRead(); + + long getRemoteRackBytesRead(); + /** Returns Number of bytes read locally through HDFS short circuit. */ long getShortCircuitBytesRead(); diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java index 9e857a49997e..d0678b251325 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java @@ -557,6 +557,10 @@ private MetricsRecordBuilder addGaugesToMetricsRecordBuilder(MetricsRecordBuilde PERCENT_FILES_LOCAL_SECONDARY_REGIONS_DESC), rsWrap.getPercentFileLocalSecondaryRegions()) .addGauge(Interns.info(TOTAL_BYTES_READ, TOTAL_BYTES_READ_DESC), rsWrap.getTotalBytesRead()) .addGauge(Interns.info(LOCAL_BYTES_READ, LOCAL_BYTES_READ_DESC), rsWrap.getLocalBytesRead()) + .addGauge(Interns.info(LOCAL_RACK_BYTES_READ, LOCAL_RACK_BYTES_READ_DESC), + rsWrap.getLocalRackBytesRead()) + .addGauge(Interns.info(REMOTE_RACK_BYTES_READ, REMOTE_RACK_BYTES_READ_DESC), + rsWrap.getRemoteRackBytesRead()) .addGauge(Interns.info(SHORTCIRCUIT_BYTES_READ, SHORTCIRCUIT_BYTES_READ_DESC), rsWrap.getShortCircuitBytesRead()) .addGauge(Interns.info(ZEROCOPY_BYTES_READ, ZEROCOPY_BYTES_READ_DESC), diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java index 6efc4df23613..e5fc2923cd7a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java @@ -29,6 +29,8 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.fs.GlobalStorageStatistics; +import org.apache.hadoop.fs.StorageStatistics; import org.apache.hadoop.hbase.CompatibilitySingletonFactory; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HDFSBlocksDistribution; @@ -1037,6 +1039,29 @@ public long getLocalBytesRead() { return FSDataInputStreamWrapper.getLocalBytesRead(); } + @Override + public long getLocalRackBytesRead() { + return getGlobalStorageStatistic("bytesReadDistanceOfOneOrTwo"); + } + + @Override + public long getRemoteRackBytesRead() { + return getGlobalStorageStatistic("bytesReadDistanceOfThreeOrFour") + + getGlobalStorageStatistic("bytesReadDistanceOfFiveOrLarger"); + } + + private static long getGlobalStorageStatistic(String name) { + StorageStatistics stats = GlobalStorageStatistics.INSTANCE.get("hdfs"); + if (stats == null) { + return 0; + } + Long val = stats.getLong(name); + if (val == null) { + return 0; + } + return val; + } + @Override public long getShortCircuitBytesRead() { return FSDataInputStreamWrapper.getShortCircuitBytesRead(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java index 3ed0dab52e71..bd2595db63dd 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java @@ -532,6 +532,16 @@ public long getLocalBytesRead() { return 0; } + @Override + public long getLocalRackBytesRead() { + return 0; + } + + @Override + public long getRemoteRackBytesRead() { + return 0; + } + @Override public long getShortCircuitBytesRead() { return 0;