From 3929bca6598fc3ca547521927e3195d0267cdabe Mon Sep 17 00:00:00 2001 From: Louis Bergelson Date: Fri, 8 Dec 2023 18:05:15 -0500 Subject: [PATCH 1/2] Update http-nio and wire its new settings * it now has a retry mechanism so we're asking for a lot of retries --- build.gradle | 2 +- .../hellbender/utils/gcs/BucketUtils.java | 34 +++++++++++++++---- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/build.gradle b/build.gradle index fa68235fae8..55bb2dca57c 100644 --- a/build.gradle +++ b/build.gradle @@ -336,7 +336,7 @@ dependencies { implementation 'org.broadinstitute:gatk-bwamem-jni:1.0.4' implementation 'org.broadinstitute:gatk-fermilite-jni:1.2.0' - implementation 'org.broadinstitute:http-nio:0.1.0-rc1' + implementation 'org.broadinstitute:http-nio:1.0.0' // Required for COSMIC Funcotator data source: implementation 'org.xerial:sqlite-jdbc:3.44.1.0' diff --git a/src/main/java/org/broadinstitute/hellbender/utils/gcs/BucketUtils.java b/src/main/java/org/broadinstitute/hellbender/utils/gcs/BucketUtils.java index 43f4ab688fd..d36c36986d0 100644 --- a/src/main/java/org/broadinstitute/hellbender/utils/gcs/BucketUtils.java +++ b/src/main/java/org/broadinstitute/hellbender/utils/gcs/BucketUtils.java @@ -25,18 +25,17 @@ import org.broadinstitute.hellbender.utils.Utils; import org.broadinstitute.hellbender.utils.io.IOUtils; import org.broadinstitute.http.nio.HttpFileSystemProvider; +import org.broadinstitute.http.nio.HttpFileSystemProviderSettings; import org.broadinstitute.http.nio.HttpsFileSystemProvider; import com.google.api.gax.retrying.RetrySettings; import com.google.auth.oauth2.GoogleCredentials; import com.google.cloud.http.HttpTransportOptions; +import org.broadinstitute.http.nio.RetryHandler; import org.threeten.bp.Duration; import java.io.*; import java.net.URL; import java.nio.channels.SeekableByteChannel; -import java.net.MalformedURLException; -import java.net.URISyntaxException; -import java.net.URL; import java.nio.file.Files; import java.util.Arrays; import java.util.UUID; @@ -55,6 +54,8 @@ public final class BucketUtils { // slashes omitted since hdfs paths seem to only have 1 slash which would be weirder to include than no slashes public static final String FILE_PREFIX = "file:"; + public static final int MAX_TIMEOUT = 120000; + public static final int MAX_ATTEMPTS = 15; private BucketUtils(){} //private so that no one will instantiate this class @@ -387,7 +388,7 @@ public static String getPathWithoutBucket(String path) { * Sets max_reopens, requester_pays, and generous timeouts as the global default. * These will apply even to library code that creates its own paths to access with NIO. * - * @param maxReopens If the GCS bucket channel errors out, how many times it will attempt to + * @param maxReopens If the channel errors out, how many times it will attempt to * re-initiate the connection. * @param requesterProject Project to bill when accessing "requester pays" buckets. If unset, * these buckets cannot be accessed. @@ -395,6 +396,25 @@ public static String getPathWithoutBucket(String path) { public static void setGlobalNIODefaultOptions(int maxReopens, String requesterProject) { CloudStorageFileSystemProvider.setDefaultCloudStorageConfiguration(getCloudStorageConfiguration(maxReopens, requesterProject)); CloudStorageFileSystemProvider.setStorageOptions(setGenerousTimeouts(StorageOptions.newBuilder()).build()); + setGlobalHttpNioDefaultOptions(maxReopens); + } + + /** + * Set the options for http-nio to have generous timeouts and defaults + */ + public static void setGlobalHttpNioDefaultOptions(final int maxRetries) { + final HttpFileSystemProviderSettings.RetrySettings retrySettings = new HttpFileSystemProviderSettings.RetrySettings( + maxRetries, + RetryHandler.DEFAULT_RETRYABLE_HTTP_CODES, + RetryHandler.DEFAULT_RETRYABLE_EXCEPTIONS, + RetryHandler.DEFALT_RETRYABLE_MESSAGES, + e -> false); + final HttpFileSystemProviderSettings settings = new HttpFileSystemProviderSettings( + java.time.Duration.ofMillis(MAX_TIMEOUT), + HttpFileSystemProviderSettings.DEFAULT_SETTINGS.redirect(), + retrySettings); + + HttpFileSystemProvider.setSettings(settings); } /** @@ -437,11 +457,11 @@ public static CloudStorageConfiguration getCloudStorageConfiguration(int maxReop private static StorageOptions.Builder setGenerousTimeouts(StorageOptions.Builder builder) { return builder .setTransportOptions(HttpTransportOptions.newBuilder() - .setConnectTimeout(120000) - .setReadTimeout(120000) + .setConnectTimeout(MAX_TIMEOUT) + .setReadTimeout(MAX_TIMEOUT) .build()) .setRetrySettings(RetrySettings.newBuilder() - .setMaxAttempts(15) + .setMaxAttempts(MAX_ATTEMPTS) .setMaxRetryDelay(Duration.ofMillis(256_000L)) .setTotalTimeout(Duration.ofMillis(4000_000L)) .setInitialRetryDelay(Duration.ofMillis(1000L)) From c830e7816ef7f06efbab54ec1bb8479a5a5fbe67 Mon Sep 17 00:00:00 2001 From: Louis Bergelson Date: Fri, 8 Dec 2023 18:18:23 -0500 Subject: [PATCH 2/2] inline --- .../org/broadinstitute/hellbender/utils/gcs/BucketUtils.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/org/broadinstitute/hellbender/utils/gcs/BucketUtils.java b/src/main/java/org/broadinstitute/hellbender/utils/gcs/BucketUtils.java index d36c36986d0..9eeaffb6223 100644 --- a/src/main/java/org/broadinstitute/hellbender/utils/gcs/BucketUtils.java +++ b/src/main/java/org/broadinstitute/hellbender/utils/gcs/BucketUtils.java @@ -55,7 +55,6 @@ public final class BucketUtils { // slashes omitted since hdfs paths seem to only have 1 slash which would be weirder to include than no slashes public static final String FILE_PREFIX = "file:"; public static final int MAX_TIMEOUT = 120000; - public static final int MAX_ATTEMPTS = 15; private BucketUtils(){} //private so that no one will instantiate this class @@ -461,7 +460,7 @@ private static StorageOptions.Builder setGenerousTimeouts(StorageOptions.Builder .setReadTimeout(MAX_TIMEOUT) .build()) .setRetrySettings(RetrySettings.newBuilder() - .setMaxAttempts(MAX_ATTEMPTS) + .setMaxAttempts(15) .setMaxRetryDelay(Duration.ofMillis(256_000L)) .setTotalTimeout(Duration.ofMillis(4000_000L)) .setInitialRetryDelay(Duration.ofMillis(1000L))