Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove deprecated S3 settings #24445

Merged
merged 8 commits into from
May 11, 2017
Merged

Remove deprecated S3 settings #24445

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
1761524
S3 Repository: Remove deprecated settings
rjernst May 2, 2017
5a15dae
add migration doc
rjernst May 9, 2017
70dea73
cleanup
rjernst May 9, 2017
aa8d020
fix compile
rjernst May 9, 2017
b39b418
Merge branch 'master' into remove_s3_deprecations
rjernst May 9, 2017
0b31d2b
Merge branch 'master' into remove_s3_deprecations
rjernst May 9, 2017
e9db3d6
Merge branch 'master' into remove_s3_deprecations
rjernst May 10, 2017
04c708f
Merge branch 'master' into remove_s3_deprecations
rjernst May 11, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 18 additions & 3 deletions docs/reference/migration/migrate_6_0/plugins.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,24 @@ the region of the configured bucket.

* Specifying s3 signer type has been removed, including `cloud.aws.signer` and `cloud.aws.s3.signer`.

* All `cloud.aws` and `repositories.s3` settings have been removed. Use `s3.client.*` settings instead.

* All repository level client settings have been removed. Use `s3.client.*` settings instead.
* Global repositories settings have been removed. This includes `repositories.s3.bucket`,
`repositories.s3.server_side_encryption`, `repositories.s3.buffer_size`,
`repositories.s3.max_retries`, `repositories.s3.use_throttle_retries`,
`repositories.s3.chunk_size`, `repositories.s3.compress`, `repositories.s3.storage_class`,
`repositories.s3.canned_acl`, `repositories.s3.base_path`, and
`repositories.s3.path_style_access`. Instead, these settings should be set directly in the
settings per repository.
See {plugins}/repository-s3-repository.html[S3 Repository settings].

* Shared client settings have been removed. This includes `cloud.aws.access_key`,
`cloud.aws.secret_key`, `cloud.aws.protocol`, `cloud.aws.proxy.host`,
`cloud.aws.proxy.port`, `cloud.aws.proxy.username`, `cloud.aws.proxy.password`,
`cloud.aws.signer`, `cloud.aws.read_timeout`, `cloud.aws.s3.access_key`,
`cloud.aws.s3.secret_key`, `cloud.aws.s3.protocol`, `cloud.aws.s3.proxy.host`,
`cloud.aws.s3.proxy.port`, `cloud.aws.s3.proxy.username`, `cloud.aws.s3.proxy.password`,
`cloud.aws.s3.signer`, `cloud.aws.s3.read_timeout`, `repositories.s3.access_key`,
`repositories.s3.secret_key`, `repositories.s3.endpoint` and `repositories.s3.protocol`.
Instead, use the new named client settings under `s3.client.CLIENT_NAME.*`.

==== Azure Repository plugin

Expand Down
45 changes: 4 additions & 41 deletions ...s/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,27 +24,20 @@
import java.util.function.Function;

import com.amazonaws.ClientConfiguration;
import com.amazonaws.Protocol;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.auth.InstanceProfileCredentialsProvider;
import com.amazonaws.http.IdleConnectionReaper;
import com.amazonaws.internal.StaticCredentialsProvider;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.S3ClientOptions;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.cluster.metadata.RepositoryMetaData;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.component.AbstractLifecycleComponent;
import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.common.settings.SecureString;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;

import static org.elasticsearch.repositories.s3.S3Repository.getValue;

class InternalAwsS3Service extends AbstractLifecycleComponent implements AwsS3Service {

Expand All @@ -70,33 +63,17 @@ public synchronized AmazonS3 client(Settings repositorySettings) {

S3ClientSettings clientSettings = clientsSettings.get(clientName);
if (clientSettings == null) {
throw new IllegalArgumentException("Unknown s3 client name [" + clientName + "]. " +
"Existing client configs: " +
throw new IllegalArgumentException("Unknown s3 client name [" + clientName + "]. Existing client configs: " +
Strings.collectionToDelimitedString(clientsSettings.keySet(), ","));
}

// If the user defined a path style access setting, we rely on it,
// otherwise we use the default value set by the SDK
Boolean pathStyleAccess = null;
if (S3Repository.Repository.PATH_STYLE_ACCESS_SETTING.exists(repositorySettings) ||
S3Repository.Repositories.PATH_STYLE_ACCESS_SETTING.exists(settings)) {
pathStyleAccess = getValue(repositorySettings, settings,
S3Repository.Repository.PATH_STYLE_ACCESS_SETTING,
S3Repository.Repositories.PATH_STYLE_ACCESS_SETTING);
}

logger.debug("creating S3 client with client_name [{}], endpoint [{}], path_style_access [{}]",
clientName, clientSettings.endpoint, pathStyleAccess);
logger.debug("creating S3 client with client_name [{}], endpoint [{}]", clientName, clientSettings.endpoint);

AWSCredentialsProvider credentials = buildCredentials(logger, clientSettings);
ClientConfiguration configuration = buildConfiguration(clientSettings, repositorySettings);

client = new AmazonS3Client(credentials, configuration);

if (pathStyleAccess != null) {
client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(pathStyleAccess));
}

if (Strings.hasText(clientSettings.endpoint)) {
client.setEndpoint(clientSettings.endpoint);
}
Expand All @@ -121,14 +98,8 @@ static ClientConfiguration buildConfiguration(S3ClientSettings clientSettings, S
clientConfiguration.setProxyPassword(clientSettings.proxyPassword);
}

Integer maxRetries = getRepoValue(repositorySettings, S3Repository.Repository.MAX_RETRIES_SETTING, clientSettings.maxRetries);
if (maxRetries != null) {
// If not explicitly set, default to 3 with exponential backoff policy
clientConfiguration.setMaxErrorRetry(maxRetries);
}
boolean useThrottleRetries = getRepoValue(repositorySettings,
S3Repository.Repository.USE_THROTTLE_RETRIES_SETTING, clientSettings.throttleRetries);
clientConfiguration.setUseThrottleRetries(useThrottleRetries);
clientConfiguration.setMaxErrorRetry(clientSettings.maxRetries);
clientConfiguration.setUseThrottleRetries(clientSettings.throttleRetries);
clientConfiguration.setSocketTimeout(clientSettings.readTimeoutMillis);

return clientConfiguration;
Expand All @@ -145,14 +116,6 @@ static AWSCredentialsProvider buildCredentials(Logger logger, S3ClientSettings c
}
}

/** Returns the value for a given setting from the repository, or returns the fallback value. */
private static <T> T getRepoValue(Settings repositorySettings, Setting<T> repositorySetting, T fallback) {
if (repositorySetting.exists(repositorySettings)) {
return repositorySetting.get(repositorySettings);
}
return fallback;
}

@Override
protected void doStart() throws ElasticsearchException {
}
Expand Down
4 changes: 2 additions & 2 deletions plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,11 +81,11 @@ class S3ClientSettings {

/** The number of retries to use when an s3 request fails. */
static final Setting.AffixSetting<Integer> MAX_RETRIES_SETTING = Setting.affixKeySetting(PREFIX, "max_retries",
key -> Setting.intSetting(key, S3Repository.Repositories.MAX_RETRIES_SETTING, 0, Property.NodeScope));
key -> Setting.intSetting(key, ClientConfiguration.DEFAULT_RETRY_POLICY.getMaxErrorRetry(), 0, Property.NodeScope));

/** Whether retries should be throttled (ie use backoff). */
static final Setting.AffixSetting<Boolean> USE_THROTTLE_RETRIES_SETTING = Setting.affixKeySetting(PREFIX, "use_throttle_retries",
key -> Setting.boolSetting(key, S3Repository.Repositories.USE_THROTTLE_RETRIES_SETTING, Property.NodeScope));
key -> Setting.boolSetting(key, ClientConfiguration.DEFAULT_THROTTLE_RETRIES, Property.NodeScope));

/** Credentials to authenticate with s3. */
final BasicAWSCredentials credentials;
Expand Down
Loading