diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionLocatorHelper.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionLocatorHelper.java index 5f4bc9f999fa..a4191d9e6229 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionLocatorHelper.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionLocatorHelper.java @@ -19,6 +19,7 @@ import static org.apache.hadoop.hbase.exceptions.ClientExceptionsUtil.findException; import static org.apache.hadoop.hbase.exceptions.ClientExceptionsUtil.isMetaClearingException; +import static org.apache.hadoop.hbase.exceptions.ClientExceptionsUtil.isRegionServerOverloadedException; import java.util.Arrays; import java.util.Optional; @@ -67,7 +68,8 @@ static void updateCachedLocationOnError(HRegionLocation loc, Throwable exception LOG.debug("The actual exception when updating {} is {}", loc, cause != null ? cause.toString() : "none"); } - if (cause == null || !isMetaClearingException(cause)) { + if (cause == null || !isMetaClearingException(cause) + || isRegionServerOverloadedException(cause)) { LOG.debug("Will not update {} because the exception is null or not the one we care about", loc); return; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRequestFutureImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRequestFutureImpl.java index e46a50e76199..c76cd0b135ea 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRequestFutureImpl.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRequestFutureImpl.java @@ -711,7 +711,8 @@ private void receiveGlobalFailure( // Do not use the exception for updating cache because it might be coming from // any of the regions in the MultiAction. updateCachedLocations(server, regionName, row, - ClientExceptionsUtil.isMetaClearingException(t) ? null : t); + ClientExceptionsUtil.isMetaClearingException(t) && + !ClientExceptionsUtil.isRegionServerOverloadedException(t) ? null : t); for (Action action : e.getValue()) { Retry retry = manageError( action.getOriginalIndex(), action.getAction(), canRetry, t, server); @@ -913,7 +914,8 @@ private void invokeCallBack(byte[] regionName, byte[] row, CResult result) { } private void cleanServerCache(ServerName server, Throwable regionException) { - if (ClientExceptionsUtil.isMetaClearingException(regionException)) { + if (ClientExceptionsUtil.isMetaClearingException(regionException) + && !ClientExceptionsUtil.isRegionServerOverloadedException(regionException)) { // We want to make sure to clear the cache in case there were location-related exceptions. // We don't to clear the cache for every possible exception that comes through, however. asyncProcess.connection.clearCaches(server); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java index af8429eda117..43fd65d3d900 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java @@ -1911,7 +1911,8 @@ public void updateCachedLocations(final TableName tableName, byte[] regionName, RegionInfo regionInfo = oldLocation.getRegion(); Throwable cause = ClientExceptionsUtil.findException(exception); if (cause != null) { - if (!ClientExceptionsUtil.isMetaClearingException(cause)) { + if (!ClientExceptionsUtil.isMetaClearingException(cause) + || ClientExceptionsUtil.isRegionServerOverloadedException(cause)) { // We know that the region is still on this region server return; } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/exceptions/ClientExceptionsUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/exceptions/ClientExceptionsUtil.java index 6b1e251953b9..e330c3c2239e 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/exceptions/ClientExceptionsUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/exceptions/ClientExceptionsUtil.java @@ -32,7 +32,6 @@ import org.apache.hadoop.hbase.CallDroppedException; import org.apache.hadoop.hbase.CallQueueTooBigException; import org.apache.hadoop.hbase.DoNotRetryIOException; -import org.apache.hadoop.hbase.MultiActionResultTooLarge; import org.apache.hadoop.hbase.NotServingRegionException; import org.apache.hadoop.hbase.RegionTooBusyException; import org.apache.hadoop.hbase.RetryImmediatelyException; @@ -59,18 +58,28 @@ public static boolean isMetaClearingException(Throwable cur) { if (cur == null) { return true; } - return !isSpecialException(cur) || (cur instanceof RegionMovedException) - || cur instanceof NotServingRegionException; + return !isMetaCachePreservingException(cur); } - public static boolean isSpecialException(Throwable cur) { - return (cur instanceof RegionMovedException || cur instanceof RegionOpeningException - || cur instanceof RegionTooBusyException || cur instanceof RpcThrottlingException - || cur instanceof MultiActionResultTooLarge || cur instanceof RetryImmediatelyException - || cur instanceof CallQueueTooBigException || cur instanceof CallDroppedException - || cur instanceof NotServingRegionException || cur instanceof RequestTooBigException); + public static boolean isRegionServerOverloadedException(Throwable t) { + t = findException(t); + return isInstanceOfRegionServerOverloadedException(t); + } + + private static boolean isInstanceOfRegionServerOverloadedException(Throwable t) { + return t instanceof CallQueueTooBigException || t instanceof CallDroppedException; } + private static boolean isMetaCachePreservingException(Throwable t) { + return t instanceof RegionOpeningException || t instanceof RegionTooBusyException + || t instanceof RpcThrottlingException || t instanceof RetryImmediatelyException + || t instanceof RequestTooBigException; + } + + private static boolean isExceptionWeCare(Throwable t) { + return isMetaCachePreservingException(t) || isInstanceOfRegionServerOverloadedException(t) + || t instanceof NotServingRegionException; + } /** * Look for an exception we know in the remote exception: @@ -87,7 +96,7 @@ public static Throwable findException(Object exception) { } Throwable cur = (Throwable) exception; while (cur != null) { - if (isSpecialException(cur)) { + if (isExceptionWeCare(cur)) { return cur; } if (cur instanceof RemoteException) { @@ -95,7 +104,7 @@ public static Throwable findException(Object exception) { cur = re.unwrapRemoteException(); // unwrapRemoteException can return the exception given as a parameter when it cannot - // unwrap it. In this case, there is no need to look further + // unwrap it. In this case, there is no need to look further // noinspection ObjectEquality if (cur == re) { return cur; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaCache.java index 3870244f4cf1..f159ab851bba 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaCache.java @@ -27,6 +27,7 @@ import java.util.ArrayList; import java.util.List; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CallDroppedException; import org.apache.hadoop.hbase.CallQueueTooBigException; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; @@ -40,6 +41,7 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.exceptions.ClientExceptionsUtil; import org.apache.hadoop.hbase.exceptions.RegionOpeningException; +import org.apache.hadoop.hbase.exceptions.RequestTooBigException; import org.apache.hadoop.hbase.quotas.RpcThrottlingException; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.regionserver.RSRpcServices; @@ -143,12 +145,14 @@ public void testPreserveMetaCacheOnException() throws Exception { table.mutateRow(mutations); } catch (IOException ex) { // Only keep track of the last exception that updated the meta cache - if (ClientExceptionsUtil.isMetaClearingException(ex) || success) { + if ((ClientExceptionsUtil.isMetaClearingException(ex) + && !ClientExceptionsUtil.isRegionServerOverloadedException(ex)) || success) { exp = ex; } } // Do not test if we did not touch the meta cache in this iteration. - if (exp != null && ClientExceptionsUtil.isMetaClearingException(exp)) { + if (exp != null && ClientExceptionsUtil.isMetaClearingException(exp) + && !ClientExceptionsUtil.isRegionServerOverloadedException(exp)) { assertNull(conn.getCachedLocation(TABLE_NAME, row)); } else if (success) { assertNotNull(conn.getCachedLocation(TABLE_NAME, row)); @@ -207,7 +211,9 @@ public static List metaCachePreservingExceptions() { add(new RpcThrottlingException(" ")); add(new MultiActionResultTooLarge(" ")); add(new RetryImmediatelyException(" ")); + add(new RequestTooBigException()); add(new CallQueueTooBigException()); + add(new CallDroppedException()); }}; }