From d789584190ff373f8b270c8d6ae567fde9d36051 Mon Sep 17 00:00:00 2001 From: wujinhu Date: Fri, 28 Jun 2019 17:53:09 +0800 Subject: [PATCH 1/2] add configurations(use_path_style && use_chunked_encoding) --- docs/plugins/repository-s3.asciidoc | 14 +++++++ .../repositories/s3/S3ClientSettings.java | 38 ++++++++++++++++--- .../repositories/s3/S3Service.java | 3 +- .../s3/S3ClientSettingsTests.java | 16 +++++++- 4 files changed, 62 insertions(+), 9 deletions(-) diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc index e2e489bb93eea..117dec2307ed5 100644 --- a/docs/plugins/repository-s3.asciidoc +++ b/docs/plugins/repository-s3.asciidoc @@ -145,6 +145,20 @@ settings belong in the `elasticsearch.yml` file. Whether retries should be throttled (i.e. should back off). Must be `true` or `false`. Defaults to `true`. +`use_path_style`:: + Whether path style should be used to access a bucket. Must be `true` or + `false`. Defaults to `true`. + See + https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html#access-bucket-intro[AWS + documentation] for more details about path style or virtual hosted style. + +`use_chunked_encoding`:: + Whether chunked encoding should be used or not. Must be `true` or + `false`. Defaults to `true`. + See + https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html[AWS + documentation] for more details about this. + [float] [[repository-s3-compatible-services]] ===== S3-compatible services diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java index ea45fbaf93dd3..2eb9db803b6fb 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java @@ -95,6 +95,14 @@ final class S3ClientSettings { static final Setting.AffixSetting USE_THROTTLE_RETRIES_SETTING = Setting.affixKeySetting(PREFIX, "use_throttle_retries", key -> Setting.boolSetting(key, ClientConfiguration.DEFAULT_THROTTLE_RETRIES, Property.NodeScope)); + /** Whether path style should be used or not (Default is true). */ + static final Setting.AffixSetting USE_PATH_STYLE_SETTING = Setting.affixKeySetting(PREFIX, "use_path_style", + key -> Setting.boolSetting(key, true, Property.NodeScope)); + + /** Whether chunked encoding should be used or not (Default is true). */ + static final Setting.AffixSetting USE_CHUNKED_ENCODING = Setting.affixKeySetting(PREFIX, "use_chunked_encoding", + key -> Setting.boolSetting(key, true, Property.NodeScope)); + /** Credentials to authenticate with s3. */ final S3BasicCredentials credentials; @@ -127,9 +135,16 @@ final class S3ClientSettings { /** Whether the s3 client should use an exponential backoff retry policy. */ final boolean throttleRetries; + /** Whether path style should be used or not. */ + final boolean usePathStyle; + + /** Whether chunked encoding should be used or not. */ + final boolean useChunkedEncoding; + private S3ClientSettings(S3BasicCredentials credentials, String endpoint, Protocol protocol, String proxyHost, int proxyPort, String proxyUsername, String proxyPassword, - int readTimeoutMillis, int maxRetries, boolean throttleRetries) { + int readTimeoutMillis, int maxRetries, boolean throttleRetries, + boolean usePathStyle, boolean useChunkedEncoding) { this.credentials = credentials; this.endpoint = endpoint; this.protocol = protocol; @@ -140,6 +155,8 @@ private S3ClientSettings(S3BasicCredentials credentials, String endpoint, Protoc this.readTimeoutMillis = readTimeoutMillis; this.maxRetries = maxRetries; this.throttleRetries = throttleRetries; + this.usePathStyle = usePathStyle; + this.useChunkedEncoding = useChunkedEncoding; } /** @@ -168,9 +185,12 @@ S3ClientSettings refine(RepositoryMetaData metadata) { } else { newCredentials = credentials; } + final boolean newUsePathStyle = getRepoSettingOrDefault(USE_PATH_STYLE_SETTING, normalizedSettings, usePathStyle); + final boolean newUseChunkedEncoding = getRepoSettingOrDefault(USE_CHUNKED_ENCODING, normalizedSettings, useChunkedEncoding); if (Objects.equals(endpoint, newEndpoint) && protocol == newProtocol && Objects.equals(proxyHost, newProxyHost) && proxyPort == newProxyPort && newReadTimeoutMillis == readTimeoutMillis && maxRetries == newMaxRetries - && newThrottleRetries == throttleRetries && Objects.equals(credentials, newCredentials)) { + && newThrottleRetries == throttleRetries && Objects.equals(credentials, newCredentials) + && newUsePathStyle == usePathStyle && newUseChunkedEncoding == useChunkedEncoding) { return this; } return new S3ClientSettings( @@ -183,7 +203,9 @@ S3ClientSettings refine(RepositoryMetaData metadata) { proxyPassword, newReadTimeoutMillis, newMaxRetries, - newThrottleRetries + newThrottleRetries, + newUsePathStyle, + newUseChunkedEncoding ); } @@ -270,7 +292,9 @@ static S3ClientSettings getClientSettings(final Settings settings, final String proxyPassword.toString(), Math.toIntExact(getConfigValue(settings, clientName, READ_TIMEOUT_SETTING).millis()), getConfigValue(settings, clientName, MAX_RETRIES_SETTING), - getConfigValue(settings, clientName, USE_THROTTLE_RETRIES_SETTING) + getConfigValue(settings, clientName, USE_THROTTLE_RETRIES_SETTING), + getConfigValue(settings, clientName, USE_PATH_STYLE_SETTING), + getConfigValue(settings, clientName, USE_CHUNKED_ENCODING) ); } } @@ -293,13 +317,15 @@ public boolean equals(final Object o) { protocol == that.protocol && Objects.equals(proxyHost, that.proxyHost) && Objects.equals(proxyUsername, that.proxyUsername) && - Objects.equals(proxyPassword, that.proxyPassword); + Objects.equals(proxyPassword, that.proxyPassword) && + Objects.equals(usePathStyle, that.usePathStyle) && + Objects.equals(useChunkedEncoding, that.useChunkedEncoding); } @Override public int hashCode() { return Objects.hash(credentials, endpoint, protocol, proxyHost, proxyPort, proxyUsername, proxyPassword, - readTimeoutMillis, maxRetries, throttleRetries); + readTimeoutMillis, maxRetries, throttleRetries, usePathStyle, useChunkedEncoding); } private static T getConfigValue(Settings settings, String clientName, diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java index 89afc7eefeee3..7f1cf5356b066 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java @@ -154,7 +154,8 @@ AmazonS3 buildClient(final S3ClientSettings clientSettings) { // We do this because directly constructing the client is deprecated (was already deprecated in 1.1.223 too) // so this change removes that usage of a deprecated API. builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, null)) - .enablePathStyleAccess(); + .withPathStyleAccessEnabled(clientSettings.usePathStyle) + .withChunkedEncodingDisabled(!clientSettings.useChunkedEncoding); return builder.build(); } diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java index 53740672df329..10cf18ba8d0b0 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java @@ -49,27 +49,39 @@ public void testThereIsADefaultClientByDefault() { assertThat(defaultSettings.readTimeoutMillis, is(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT)); assertThat(defaultSettings.maxRetries, is(ClientConfiguration.DEFAULT_RETRY_POLICY.getMaxErrorRetry())); assertThat(defaultSettings.throttleRetries, is(ClientConfiguration.DEFAULT_THROTTLE_RETRIES)); + assertThat(defaultSettings.usePathStyle, is(true)); + assertThat(defaultSettings.useChunkedEncoding, is(true)); } public void testDefaultClientSettingsCanBeSet() { final Map settings = S3ClientSettings.load(Settings.builder() - .put("s3.client.default.max_retries", 10).build()); + .put("s3.client.default.max_retries", 10) + .put("s3.client.default.use_path_style", false) + .put("s3.client.default.use_chunked_encoding", false).build()); assertThat(settings.keySet(), contains("default")); final S3ClientSettings defaultSettings = settings.get("default"); assertThat(defaultSettings.maxRetries, is(10)); + assertThat(defaultSettings.usePathStyle, is(false)); + assertThat(defaultSettings.useChunkedEncoding, is(false)); } public void testNondefaultClientCreatedBySettingItsSettings() { final Map settings = S3ClientSettings.load(Settings.builder() - .put("s3.client.another_client.max_retries", 10).build()); + .put("s3.client.another_client.max_retries", 10) + .put("s3.client.another_client.use_path_style", false) + .put("s3.client.another_client.use_chunked_encoding", false).build()); assertThat(settings.keySet(), contains("default", "another_client")); final S3ClientSettings defaultSettings = settings.get("default"); assertThat(defaultSettings.maxRetries, is(ClientConfiguration.DEFAULT_RETRY_POLICY.getMaxErrorRetry())); + assertThat(defaultSettings.usePathStyle, is(true)); + assertThat(defaultSettings.useChunkedEncoding, is(true)); final S3ClientSettings anotherClientSettings = settings.get("another_client"); assertThat(anotherClientSettings.maxRetries, is(10)); + assertThat(anotherClientSettings.usePathStyle, is(false)); + assertThat(anotherClientSettings.useChunkedEncoding, is(false)); } public void testRejectionOfLoneAccessKey() { From 185decd081ed8823f673a649fd8e0343ba603ea6 Mon Sep 17 00:00:00 2001 From: wujinhu Date: Fri, 28 Jun 2019 19:45:55 +0800 Subject: [PATCH 2/2] add configurations(use_path_style && use_chunked_encoding) --- .../org/elasticsearch/repositories/s3/S3ClientSettings.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java index 2eb9db803b6fb..d594517fb3a9a 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java @@ -100,7 +100,7 @@ final class S3ClientSettings { key -> Setting.boolSetting(key, true, Property.NodeScope)); /** Whether chunked encoding should be used or not (Default is true). */ - static final Setting.AffixSetting USE_CHUNKED_ENCODING = Setting.affixKeySetting(PREFIX, "use_chunked_encoding", + static final Setting.AffixSetting USE_CHUNKED_ENCODING_SETTING = Setting.affixKeySetting(PREFIX, "use_chunked_encoding", key -> Setting.boolSetting(key, true, Property.NodeScope)); /** Credentials to authenticate with s3. */ @@ -186,7 +186,7 @@ S3ClientSettings refine(RepositoryMetaData metadata) { newCredentials = credentials; } final boolean newUsePathStyle = getRepoSettingOrDefault(USE_PATH_STYLE_SETTING, normalizedSettings, usePathStyle); - final boolean newUseChunkedEncoding = getRepoSettingOrDefault(USE_CHUNKED_ENCODING, normalizedSettings, useChunkedEncoding); + final boolean newUseChunkedEncoding = getRepoSettingOrDefault(USE_CHUNKED_ENCODING_SETTING, normalizedSettings, useChunkedEncoding); if (Objects.equals(endpoint, newEndpoint) && protocol == newProtocol && Objects.equals(proxyHost, newProxyHost) && proxyPort == newProxyPort && newReadTimeoutMillis == readTimeoutMillis && maxRetries == newMaxRetries && newThrottleRetries == throttleRetries && Objects.equals(credentials, newCredentials) @@ -294,7 +294,7 @@ static S3ClientSettings getClientSettings(final Settings settings, final String getConfigValue(settings, clientName, MAX_RETRIES_SETTING), getConfigValue(settings, clientName, USE_THROTTLE_RETRIES_SETTING), getConfigValue(settings, clientName, USE_PATH_STYLE_SETTING), - getConfigValue(settings, clientName, USE_CHUNKED_ENCODING) + getConfigValue(settings, clientName, USE_CHUNKED_ENCODING_SETTING) ); } }