[Distributed/Snapshot/Restore] Add configurations: use_path_style && use_chunked_encoding

docs/plugins/repository-s3.asciidoc

@@ -145,6 +145,20 @@ settings belong in the `elasticsearch.yml` file.
     Whether retries should be throttled (i.e. should back off). Must be `true`
     or `false`. Defaults to `true`.
 
+`use_path_style`::
+    Whether path style should be used to access a bucket. Must be `true` or
+    `false`. Defaults to `true`.
+    See
+    https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html#access-bucket-intro[AWS
+    documentation] for more details about path style or virtual hosted style.
+
+`use_chunked_encoding`::
+    Whether chunked encoding should be used or not. Must be `true` or
+    `false`. Defaults to `true`.
+    See
+    https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html[AWS
+    documentation] for more details about this.
+
 [float]
 [[repository-s3-compatible-services]]
 ===== S3-compatible services

plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java

@@ -95,6 +95,14 @@ final class S3ClientSettings {
     static final Setting.AffixSetting<Boolean> USE_THROTTLE_RETRIES_SETTING = Setting.affixKeySetting(PREFIX, "use_throttle_retries",
         key -> Setting.boolSetting(key, ClientConfiguration.DEFAULT_THROTTLE_RETRIES, Property.NodeScope));
 
+    /** Whether path style should be used or not (Default is true). */
+    static final Setting.AffixSetting<Boolean> USE_PATH_STYLE_SETTING = Setting.affixKeySetting(PREFIX, "use_path_style",
+        key -> Setting.boolSetting(key, true, Property.NodeScope));
+
+    /** Whether chunked encoding should be used or not (Default is true). */
+    static final Setting.AffixSetting<Boolean> USE_CHUNKED_ENCODING_SETTING = Setting.affixKeySetting(PREFIX, "use_chunked_encoding",
+        key -> Setting.boolSetting(key, true, Property.NodeScope));
+
     /** Credentials to authenticate with s3. */
     final S3BasicCredentials credentials;
 
@@ -127,9 +135,16 @@ final class S3ClientSettings {
     /** Whether the s3 client should use an exponential backoff retry policy. */
     final boolean throttleRetries;
 
+    /** Whether path style should be used or not. */
+    final boolean usePathStyle;
+
+    /** Whether chunked encoding should be used or not. */
+    final boolean useChunkedEncoding;
+
     private S3ClientSettings(S3BasicCredentials credentials, String endpoint, Protocol protocol,
                              String proxyHost, int proxyPort, String proxyUsername, String proxyPassword,
-                             int readTimeoutMillis, int maxRetries, boolean throttleRetries) {
+                             int readTimeoutMillis, int maxRetries, boolean throttleRetries,
+                             boolean usePathStyle, boolean useChunkedEncoding) {
         this.credentials = credentials;
         this.endpoint = endpoint;
         this.protocol = protocol;
@@ -140,6 +155,8 @@ private S3ClientSettings(S3BasicCredentials credentials, String endpoint, Protoc
         this.readTimeoutMillis = readTimeoutMillis;
         this.maxRetries = maxRetries;
         this.throttleRetries = throttleRetries;
+        this.usePathStyle = usePathStyle;
+        this.useChunkedEncoding = useChunkedEncoding;
     }
 
     /**
@@ -168,9 +185,12 @@ S3ClientSettings refine(RepositoryMetaData metadata) {
         } else {
             newCredentials = credentials;
         }
+        final boolean newUsePathStyle = getRepoSettingOrDefault(USE_PATH_STYLE_SETTING, normalizedSettings, usePathStyle);
+        final boolean newUseChunkedEncoding = getRepoSettingOrDefault(USE_CHUNKED_ENCODING_SETTING, normalizedSettings, useChunkedEncoding);
         if (Objects.equals(endpoint, newEndpoint) && protocol == newProtocol && Objects.equals(proxyHost, newProxyHost)
             && proxyPort == newProxyPort && newReadTimeoutMillis == readTimeoutMillis && maxRetries == newMaxRetries
-            && newThrottleRetries == throttleRetries && Objects.equals(credentials, newCredentials)) {
+            && newThrottleRetries == throttleRetries && Objects.equals(credentials, newCredentials)
+            && newUsePathStyle == usePathStyle && newUseChunkedEncoding == useChunkedEncoding) {
             return this;
         }
         return new S3ClientSettings(
@@ -183,7 +203,9 @@ S3ClientSettings refine(RepositoryMetaData metadata) {
             proxyPassword,
             newReadTimeoutMillis,
             newMaxRetries,
-            newThrottleRetries
+            newThrottleRetries,
+            newUsePathStyle,
+            newUseChunkedEncoding
         );
     }
 
@@ -270,7 +292,9 @@ static S3ClientSettings getClientSettings(final Settings settings, final String
                 proxyPassword.toString(),
                 Math.toIntExact(getConfigValue(settings, clientName, READ_TIMEOUT_SETTING).millis()),
                 getConfigValue(settings, clientName, MAX_RETRIES_SETTING),
-                getConfigValue(settings, clientName, USE_THROTTLE_RETRIES_SETTING)
+                getConfigValue(settings, clientName, USE_THROTTLE_RETRIES_SETTING),
+                getConfigValue(settings, clientName, USE_PATH_STYLE_SETTING),
+                getConfigValue(settings, clientName, USE_CHUNKED_ENCODING_SETTING)
             );
         }
     }
@@ -293,13 +317,15 @@ public boolean equals(final Object o) {
             protocol == that.protocol &&
             Objects.equals(proxyHost, that.proxyHost) &&
             Objects.equals(proxyUsername, that.proxyUsername) &&
-            Objects.equals(proxyPassword, that.proxyPassword);
+            Objects.equals(proxyPassword, that.proxyPassword) &&
+            Objects.equals(usePathStyle, that.usePathStyle) &&
+            Objects.equals(useChunkedEncoding, that.useChunkedEncoding);
     }
 
     @Override
     public int hashCode() {
         return Objects.hash(credentials, endpoint, protocol, proxyHost, proxyPort, proxyUsername, proxyPassword,
-            readTimeoutMillis, maxRetries, throttleRetries);
+            readTimeoutMillis, maxRetries, throttleRetries, usePathStyle, useChunkedEncoding);
     }
 
     private static <T> T getConfigValue(Settings settings, String clientName,

plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java

@@ -154,7 +154,8 @@ AmazonS3 buildClient(final S3ClientSettings clientSettings) {
         // We do this because directly constructing the client is deprecated (was already deprecated in 1.1.223 too)
         // so this change removes that usage of a deprecated API.
         builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, null))
-            .enablePathStyleAccess();
+            .withPathStyleAccessEnabled(clientSettings.usePathStyle)
+            .withChunkedEncodingDisabled(!clientSettings.useChunkedEncoding);
 
         return builder.build();
     }

plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java

@@ -49,27 +49,39 @@ public void testThereIsADefaultClientByDefault() {
         assertThat(defaultSettings.readTimeoutMillis, is(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT));
         assertThat(defaultSettings.maxRetries, is(ClientConfiguration.DEFAULT_RETRY_POLICY.getMaxErrorRetry()));
         assertThat(defaultSettings.throttleRetries, is(ClientConfiguration.DEFAULT_THROTTLE_RETRIES));
+        assertThat(defaultSettings.usePathStyle, is(true));
+        assertThat(defaultSettings.useChunkedEncoding, is(true));
     }
 
     public void testDefaultClientSettingsCanBeSet() {
         final Map<String, S3ClientSettings> settings = S3ClientSettings.load(Settings.builder()
-            .put("s3.client.default.max_retries", 10).build());
+            .put("s3.client.default.max_retries", 10)
+            .put("s3.client.default.use_path_style", false)
+            .put("s3.client.default.use_chunked_encoding", false).build());
         assertThat(settings.keySet(), contains("default"));
 
         final S3ClientSettings defaultSettings = settings.get("default");
         assertThat(defaultSettings.maxRetries, is(10));
+        assertThat(defaultSettings.usePathStyle, is(false));
+        assertThat(defaultSettings.useChunkedEncoding, is(false));
     }
 
     public void testNondefaultClientCreatedBySettingItsSettings() {
         final Map<String, S3ClientSettings> settings = S3ClientSettings.load(Settings.builder()
-            .put("s3.client.another_client.max_retries", 10).build());
+            .put("s3.client.another_client.max_retries", 10)
+            .put("s3.client.another_client.use_path_style", false)
+            .put("s3.client.another_client.use_chunked_encoding", false).build());
         assertThat(settings.keySet(), contains("default", "another_client"));
 
         final S3ClientSettings defaultSettings = settings.get("default");
         assertThat(defaultSettings.maxRetries, is(ClientConfiguration.DEFAULT_RETRY_POLICY.getMaxErrorRetry()));
+        assertThat(defaultSettings.usePathStyle, is(true));
+        assertThat(defaultSettings.useChunkedEncoding, is(true));
 
         final S3ClientSettings anotherClientSettings = settings.get("another_client");
         assertThat(anotherClientSettings.maxRetries, is(10));
+        assertThat(anotherClientSettings.usePathStyle, is(false));
+        assertThat(anotherClientSettings.useChunkedEncoding, is(false));
     }
 
     public void testRejectionOfLoneAccessKey() {