From 2e638de6a3fb8662b8977c118548a5c725e20666 Mon Sep 17 00:00:00 2001 From: Javier Akira Luca de Tena Date: Wed, 9 Sep 2020 10:45:46 +0900 Subject: [PATCH] HBASE-24994 Add hedgedReadOpsInCurThread metric (#2365) Signed-off-by: Duo Zhang --- .../hadoop/hbase/regionserver/MetricsRegionServerSource.java | 3 +++ .../hbase/regionserver/MetricsRegionServerSourceImpl.java | 2 ++ .../hbase/regionserver/MetricsRegionServerWrapper.java | 5 +++++ .../hbase/regionserver/MetricsRegionServerWrapperImpl.java | 5 +++++ .../hbase/regionserver/MetricsRegionServerWrapperStub.java | 5 +++++ src/main/asciidoc/_chapters/performance.adoc | 2 ++ 6 files changed, 22 insertions(+) diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java index 45269671ec56..afeccee504a7 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java @@ -486,6 +486,9 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo String HEDGED_READ_WINS = "hedgedReadWins"; String HEDGED_READ_WINS_DESC = "The number of times we started a hedged read and a hedged read won"; + String HEDGED_READ_IN_CUR_THREAD = "hedgedReadOpsInCurThread"; + String HEDGED_READ_IN_CUR_THREAD_DESC = + "The number of times we execute a hedged read in current thread as a fallback for task rejection"; String TOTAL_BYTES_READ = "totalBytesRead"; String TOTAL_BYTES_READ_DESC = "The total number of bytes read from HDFS"; diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java index b695ed2c49c3..32a2b1e99f4e 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java @@ -458,6 +458,8 @@ public void getMetrics(MetricsCollector metricsCollector, boolean all) { .addCounter(Interns.info(HEDGED_READS, HEDGED_READS_DESC), rsWrap.getHedgedReadOps()) .addCounter(Interns.info(HEDGED_READ_WINS, HEDGED_READ_WINS_DESC), rsWrap.getHedgedReadWins()) + .addCounter(Interns.info(HEDGED_READ_IN_CUR_THREAD, HEDGED_READ_IN_CUR_THREAD_DESC), + rsWrap.getHedgedReadOpsInCurThread()) .addCounter(Interns.info(BLOCKED_REQUESTS_COUNT, BLOCKED_REQUESTS_COUNT_DESC), rsWrap.getBlockedRequestsCount()) .tag(Interns.info(ZOOKEEPER_QUORUM_NAME, ZOOKEEPER_QUORUM_DESC), diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java index d36a6a024510..4ca2d0e85ad1 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java @@ -452,6 +452,11 @@ public interface MetricsRegionServerWrapper { */ long getHedgedReadWins(); + /** + * @return Count of times a hedged read executes in current thread + */ + long getHedgedReadOpsInCurThread(); + /** * @return Number of total bytes read from HDFS. */ diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java index 8231c2bb3fe2..c4328c410da4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java @@ -928,6 +928,11 @@ public long getHedgedReadWins() { return this.dfsHedgedReadMetrics == null? 0: this.dfsHedgedReadMetrics.getHedgedReadWins(); } + @Override + public long getHedgedReadOpsInCurThread() { + return this.dfsHedgedReadMetrics == null ? 0 : this.dfsHedgedReadMetrics.getHedgedReadOpsInCurThread(); + } + @Override public long getTotalBytesRead() { return FSDataInputStreamWrapper.getTotalBytesRead(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java index 80ed55d0d218..0a0485c1aafe 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java @@ -420,6 +420,11 @@ public long getHedgedReadWins() { return 10; } + @Override + public long getHedgedReadOpsInCurThread() { + return 5; + } + @Override public long getTotalBytesRead() { return 0; diff --git a/src/main/asciidoc/_chapters/performance.adoc b/src/main/asciidoc/_chapters/performance.adoc index ad6572dabccb..6d3f8e4b5d7d 100644 --- a/src/main/asciidoc/_chapters/performance.adoc +++ b/src/main/asciidoc/_chapters/performance.adoc @@ -801,6 +801,8 @@ See <> for more information. This could indicate that read requests are often slow, or that hedged reads are triggered too quickly. * hedgeReadOpsWin - the number of times the hedged read thread was faster than the original thread. This could indicate that a given RegionServer is having trouble servicing requests. +* hedgedReadOpsInCurThread - the number of times hedged read was rejected from executor and needed to fallback to be executed in current thread. + This could indicate that current hedged read thread pool size is not appropriate. [[perf.deleting]]