diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRequestFutureImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRequestFutureImpl.java index de872f2e56ff..13c0c0975f90 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRequestFutureImpl.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRequestFutureImpl.java @@ -218,13 +218,13 @@ public void run() { } catch (IOException e) { // The service itself failed . It may be an error coming from the communication // layer, but, as well, a functional error raised by the server. - receiveGlobalFailure(multiAction, server, numAttempt, e); + receiveGlobalFailure(multiAction, server, numAttempt, e, true); return; } catch (Throwable t) { // This should not happen. Let's log & retry anyway. LOG.error("id=" + asyncProcess.id + ", caught throwable. Unexpected." + " Retrying. Server=" + server + ", tableName=" + tableName, t); - receiveGlobalFailure(multiAction, server, numAttempt, t); + receiveGlobalFailure(multiAction, server, numAttempt, t, true); return; } if (res.type() == AbstractResponse.ResponseType.MULTI) { @@ -520,6 +520,7 @@ private RegionLocations findAllLocationsOrFail(Action action, boolean useCache) */ void sendMultiAction(Map actionsByServer, int numAttempt, List actionsForReplicaThread, boolean reuseThread) { + boolean clearServerCache = true; // Run the last item on the same thread if we are already on a send thread. // We hope most of the time it will be the only item, so we can cut down on threads. int actionsRemaining = actionsByServer.size(); @@ -554,6 +555,8 @@ void sendMultiAction(Map actionsByServer, int numAttemp // let's secure this a little. LOG.warn("id=" + asyncProcess.id + ", task rejected by pool. Unexpected." + " Server=" + server.getServerName(), t); + // Do not update cache if exception is from failing to submit action to thread pool + clearServerCache = false; } else { // see #HBASE-14359 for more details LOG.warn("Caught unexpected exception/error: ", t); @@ -561,7 +564,7 @@ void sendMultiAction(Map actionsByServer, int numAttemp asyncProcess.decTaskCounters(multiAction.getRegions(), server); // We're likely to fail again, but this will increment the attempt counter, // so it will finish. - receiveGlobalFailure(multiAction, server, numAttempt, t); + receiveGlobalFailure(multiAction, server, numAttempt, t, clearServerCache); } } } @@ -711,11 +714,15 @@ private void failAll(MultiAction actions, ServerName server, int numAttempt, * @param t the throwable (if any) that caused the resubmit */ private void receiveGlobalFailure(MultiAction rsActions, ServerName server, int numAttempt, - Throwable t) { + Throwable t, boolean clearServerCache) { errorsByServer.reportServerError(server); Retry canRetry = errorsByServer.canTryMore(numAttempt) ? Retry.YES : Retry.NO_RETRIES_EXHAUSTED; - cleanServerCache(server, t); + // Do not update cache if exception is from failing to submit action to thread pool + if (clearServerCache) { + cleanServerCache(server, t); + } + int failed = 0; int stopped = 0; List toReplay = new ArrayList<>(); @@ -723,9 +730,12 @@ private void receiveGlobalFailure(MultiAction rsActions, ServerName server, int byte[] regionName = e.getKey(); byte[] row = e.getValue().get(0).getAction().getRow(); // Do not use the exception for updating cache because it might be coming from - // any of the regions in the MultiAction. - updateCachedLocations(server, regionName, row, - ClientExceptionsUtil.isMetaClearingException(t) ? null : t); + // any of the regions in the MultiAction and do not update cache if exception is + // from failing to submit action to thread pool + if (clearServerCache) { + updateCachedLocations(server, regionName, row, + ClientExceptionsUtil.isMetaClearingException(t) ? null : t); + } for (Action action : e.getValue()) { Retry retry = manageError(action.getOriginalIndex(), action.getAction(), canRetry, t, server);