HBASE-26809: Report client backoff time for server overloaded (#4729)

Co-authored-by: Briana Augenreich <[email protected]>
Signed-off-by: Duo Zhang <[email protected]>

hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncBatchRpcRetryingCaller.java

@@ -487,6 +487,11 @@ private void tryResubmit(Stream<Action> actions, int tries, boolean immediately,
     } else {
       delayNs = getPauseTime(pauseNsToUse, tries - 1);
     }
+
+    if (isServerOverloaded) {
+      Optional<MetricsConnection> metrics = conn.getConnectionMetrics();
+      metrics.ifPresent(m -> m.incrementServerOverloadedBackoffTime(delayNs, TimeUnit.NANOSECONDS));
+    }
     retryTimer.newTimeout(t -> groupAndSend(actions, tries + 1), delayNs, TimeUnit.NANOSECONDS);
   }
 

hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRpcRetryingCaller.java

@@ -139,6 +139,10 @@ private void tryScheduleRetry(Throwable error) {
       delayNs = getPauseTime(pauseNsToUse, tries - 1);
     }
     tries++;
+    if (HBaseServerException.isServerOverloaded(error)) {
+      Optional<MetricsConnection> metrics = conn.getConnectionMetrics();
+      metrics.ifPresent(m -> m.incrementServerOverloadedBackoffTime(delayNs, TimeUnit.NANOSECONDS));
+    }
     retryTimer.newTimeout(t -> doCall(), delayNs, TimeUnit.NANOSECONDS);
   }
 

hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncScanSingleRegionRpcRetryingCaller.java

@@ -113,6 +113,8 @@ class AsyncScanSingleRegionRpcRetryingCaller {
 
   private final Runnable completeWhenNoMoreResultsInRegion;
 
+  private final AsyncConnectionImpl conn;
+
   private final CompletableFuture<Boolean> future;
 
   private final HBaseRpcController controller;
@@ -318,6 +320,7 @@ public AsyncScanSingleRegionRpcRetryingCaller(Timer retryTimer, AsyncConnectionI
     long pauseNsForServerOverloaded, int maxAttempts, long scanTimeoutNs, long rpcTimeoutNs,
     int startLogErrorsCnt) {
     this.retryTimer = retryTimer;
+    this.conn = conn;
     this.scan = scan;
     this.scanMetrics = scanMetrics;
     this.scannerId = scannerId;
@@ -441,6 +444,11 @@ private void onError(Throwable error) {
       return;
     }
     tries++;
+
+    if (HBaseServerException.isServerOverloaded(error)) {
+      Optional<MetricsConnection> metrics = conn.getConnectionMetrics();
+      metrics.ifPresent(m -> m.incrementServerOverloadedBackoffTime(delayNs, TimeUnit.NANOSECONDS));
+    }
     retryTimer.newTimeout(t -> call(), delayNs, TimeUnit.NANOSECONDS);
   }
 

hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java

@@ -315,6 +315,7 @@ public Counter newMetric(Class<?> clazz, String name, String scope) {
   protected final Histogram numActionsPerServerHist;
   protected final Counter nsLookups;
   protected final Counter nsLookupsFailed;
+  protected final Timer overloadedBackoffTimer;
 
   // dynamic metrics
 
@@ -376,6 +377,8 @@ protected Ratio getRatio() {
       registry.histogram(name(MetricsConnection.class, "numActionsPerServer", scope));
     this.nsLookups = registry.counter(name(this.getClass(), NS_LOOKUPS, scope));
     this.nsLookupsFailed = registry.counter(name(this.getClass(), NS_LOOKUPS_FAILED, scope));
+    this.overloadedBackoffTimer =
+      registry.timer(name(this.getClass(), "overloadedBackoffDurationMs", scope));
 
     this.reporter = JmxReporter.forRegistry(this.registry).build();
     this.reporter.start();
@@ -449,6 +452,11 @@ public void incrDelayRunnersAndUpdateDelayInterval(long interval) {
     this.runnerStats.updateDelayInterval(interval);
   }
 
+  /** Update the overloaded backoff time **/
+  public void incrementServerOverloadedBackoffTime(long time, TimeUnit timeUnit) {
+    overloadedBackoffTimer.update(time, timeUnit);
+  }
+
   /**
    * Get a metric for {@code key} from {@code map}, or create it with {@code factory}.
    */