From ee68227a5203eba8eb481306e9c6be0ed0b1ba7b Mon Sep 17 00:00:00 2001 From: David Roazen Date: Tue, 14 Aug 2018 10:44:28 -0400 Subject: [PATCH] google-cloud-nio: retry on 502 errors, and increase max depth when doing channel reopens We've frequently encountered transient 502 errors in the wild when using google-cloud-nio (see https://github.com/broadinstitute/gatk/issues/4888), implying that this error should be added to the list of retryable errors. I also increased the maximum depth when inspecting nested exceptions looking for reopenable errors from 10 to 20, as we've seen chains of exceptions that come very close to the current limit of 10. --- .../cloud/storage/contrib/nio/CloudStorageRetryHandler.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageRetryHandler.java b/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageRetryHandler.java index c0b78172c112..694618875e45 100644 --- a/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageRetryHandler.java +++ b/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageRetryHandler.java @@ -21,6 +21,7 @@ import java.io.EOFException; import java.net.SocketException; import java.net.SocketTimeoutException; +import java.net.UnknownHostException; import javax.net.ssl.SSLException; /** @@ -144,7 +145,7 @@ void sleepForAttempt(int attempt) { * @return true if exs is a retryable error, otherwise false */ private static boolean isRetryable(final StorageException exs) { - return exs.isRetryable() || exs.getCode() == 500 || exs.getCode() == 503; + return exs.isRetryable() || exs.getCode() == 500 || exs.getCode() == 502 || exs.getCode() == 503; } /** @@ -154,7 +155,7 @@ private static boolean isRetryable(final StorageException exs) { private static boolean isReopenable(final StorageException exs) { Throwable throwable = exs; // ensures finite iteration - int maxDepth = 10; + int maxDepth = 20; while (throwable != null && maxDepth-- > 0) { if ((throwable.getMessage() != null && throwable.getMessage().contains("Connection closed prematurely"))