From 047f077f0b6117985eab9d02bb10bb8c328fabfb Mon Sep 17 00:00:00 2001 From: Fantasy-Jay <13631435453@163.com> Date: Thu, 6 Jul 2023 10:32:08 +0800 Subject: [PATCH] HBASE-27845 Distinguish the mutate types of rpc error in MetricsConnection. (#5224) Co-authored-by: fantasy <875282031@qq.com> Co-authored-by: jay.zhu Signed-off-by: Duo Zhang --- .../hbase/client/MetricsConnection.java | 25 +++++++++++- .../hbase/client/TestMetricsConnection.java | 39 ++++++++++++++----- 2 files changed, 52 insertions(+), 12 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java index dd3e571a8b7a..8a299dc4e5c1 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java @@ -652,7 +652,28 @@ public void updateRpc(MethodDescriptor method, Message param, CallStats stats, T concurrentCallsPerServerHist.update(callsPerServer); } // Update the counter that tracks RPCs by type. - final String methodName = method.getService().getName() + "_" + method.getName(); + StringBuilder methodName = new StringBuilder(); + methodName.append(method.getService().getName()).append("_").append(method.getName()); + // Distinguish mutate types. + if ("Mutate".equals(method.getName())) { + final MutationType type = ((MutateRequest) param).getMutation().getMutateType(); + switch (type) { + case APPEND: + methodName.append("(Append)"); + break; + case DELETE: + methodName.append("(Delete)"); + break; + case INCREMENT: + methodName.append("(Increment)"); + break; + case PUT: + methodName.append("(Put)"); + break; + default: + methodName.append("(Unknown)"); + } + } getMetric(CNT_BASE + methodName, rpcCounters, counterFactory).inc(); if (e != null) { getMetric(FAILURE_CNT_BASE + methodName, rpcCounters, counterFactory).inc(); @@ -729,7 +750,7 @@ public void updateRpc(MethodDescriptor method, Message param, CallStats stats, T } } // Fallback to dynamic registry lookup for DDL methods. - updateRpcGeneric(methodName, stats); + updateRpcGeneric(methodName.toString(), stats); } public void incrCacheDroppingExceptions(Object exception) { diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestMetricsConnection.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestMetricsConnection.java index d70d2cf60006..2afdc7ee558d 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestMetricsConnection.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestMetricsConnection.java @@ -99,7 +99,7 @@ public void testMetricsConnectionScope() throws IOException { } @Test - public void testMetricsWithMutiConnections() throws IOException { + public void testMetricsWithMultiConnections() throws IOException { Configuration conf = new Configuration(); conf.setBoolean(MetricsConnection.CLIENT_SIDE_METRICS_ENABLED_KEY, true); conf.set(MetricsConnection.METRICS_SCOPE_KEY, "unit-test"); @@ -178,7 +178,8 @@ public void testStaticMetrics() throws IOException { MutateRequest.newBuilder() .setMutation(ProtobufUtil.toMutation(MutationType.PUT, new Put(foo))).setRegion(region) .build(), - MetricsConnection.newCallStats(), null); + MetricsConnection.newCallStats(), + new CallTimeoutException("test with CallTimeoutException")); } final String rpcCountPrefix = "rpcCount_" + ClientService.getDescriptor().getName() + "_"; @@ -188,20 +189,38 @@ public void testStaticMetrics() throws IOException { long metricVal; Counter counter; - for (String method : new String[] { "Get", "Scan", "Multi", "Mutate" }) { + for (String method : new String[] { "Get", "Scan", "Multi" }) { metricKey = rpcCountPrefix + method; metricVal = METRICS.getRpcCounters().get(metricKey).getCount(); - assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal >= loop); + assertEquals("metric: " + metricKey + " val: " + metricVal, metricVal, loop); metricKey = rpcFailureCountPrefix + method; counter = METRICS.getRpcCounters().get(metricKey); metricVal = (counter != null) ? counter.getCount() : 0; - if (method.equals("Get") || method.equals("Mutate")) { + if (method.equals("Get")) { // no failure - assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal == 0); + assertEquals("metric: " + metricKey + " val: " + metricVal, 0, metricVal); } else { // has failure - assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal == loop); + assertEquals("metric: " + metricKey + " val: " + metricVal, metricVal, loop); + } + } + + String method = "Mutate"; + for (String mutationType : new String[] { "Append", "Delete", "Increment", "Put" }) { + metricKey = rpcCountPrefix + method + "(" + mutationType + ")"; + metricVal = METRICS.getRpcCounters().get(metricKey).getCount(); + assertEquals("metric: " + metricKey + " val: " + metricVal, metricVal, loop); + + metricKey = rpcFailureCountPrefix + method + "(" + mutationType + ")"; + counter = METRICS.getRpcCounters().get(metricKey); + metricVal = (counter != null) ? counter.getCount() : 0; + if (mutationType.equals("Put")) { + // has failure + assertEquals("metric: " + metricKey + " val: " + metricVal, metricVal, loop); + } else { + // no failure + assertEquals("metric: " + metricKey + " val: " + metricVal, 0, metricVal); } } @@ -209,19 +228,19 @@ public void testStaticMetrics() throws IOException { metricKey = "rpcRemoteExceptions_IOException"; counter = METRICS.getRpcCounters().get(metricKey); metricVal = (counter != null) ? counter.getCount() : 0; - assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal == loop); + assertEquals("metric: " + metricKey + " val: " + metricVal, metricVal, loop); // local exception metricKey = "rpcLocalExceptions_CallTimeoutException"; counter = METRICS.getRpcCounters().get(metricKey); metricVal = (counter != null) ? counter.getCount() : 0; - assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal == loop); + assertEquals("metric: " + metricKey + " val: " + metricVal, metricVal, loop * 2); // total exception metricKey = "rpcTotalExceptions"; counter = METRICS.getRpcCounters().get(metricKey); metricVal = (counter != null) ? counter.getCount() : 0; - assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal == loop * 2); + assertEquals("metric: " + metricKey + " val: " + metricVal, metricVal, loop * 3); for (MetricsConnection.CallTracker t : new MetricsConnection.CallTracker[] { METRICS.getGetTracker(), METRICS.getScanTracker(), METRICS.getMultiTracker(),