From 0278c882221a91578e36e8d48f485fad924c8e40 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Tue, 10 Jul 2018 13:27:20 +0200 Subject: [PATCH 01/12] add EC2 credential test --- .../gradle/test/ClusterConfiguration.groovy | 4 +- .../gradle/test/ClusterFormationTasks.groovy | 33 ++- .../elasticsearch/gradle/test/NodeInfo.groovy | 10 +- plugins/repository-s3/build.gradle | 8 +- .../repositories/s3/AmazonS3Fixture.java | 72 +++++- .../40_repository_ec2_credentials.yml | 243 ++++++++++++++++++ 6 files changed, 341 insertions(+), 29 deletions(-) create mode 100644 plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy index 5c363ac043aff..e6ce2744f1857 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy @@ -137,7 +137,7 @@ class ClusterConfiguration { this.project = project } - Map systemProperties = new HashMap<>() + Map systemProperties = new HashMap<>() Map settings = new HashMap<>() @@ -157,7 +157,7 @@ class ClusterConfiguration { List dependencies = new ArrayList<>() @Input - void systemProperty(String property, String value) { + void systemProperty(String property, Object value) { systemProperties.put(property, value) } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index be0fb3a07c699..24aee5c21d26d 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -202,6 +202,7 @@ class ClusterFormationTasks { setup = configureCreateKeystoreTask(taskName(prefix, node, 'createKeystore'), project, setup, node) setup = configureAddKeystoreSettingTasks(prefix, project, setup, node) setup = configureAddKeystoreFileTasks(prefix, project, setup, node) + setup = configureESJavaOpts(prefix, project, setup, node) if (node.config.plugins.isEmpty() == false) { if (node.nodeVersion == VersionProperties.elasticsearch) { @@ -412,6 +413,31 @@ class ClusterFormationTasks { return parentTask } + /** Configure ES JAVA OPTS - adds system properties, assertion flags, remote debug etc */ + static Task configureESJavaOpts(String parent, Project project, Task setup, NodeInfo node) { + return project.tasks.create(name: taskName(parent, node, 'configureESJavaOpts'), type: DefaultTask, dependsOn: setup) { + doLast { + String collectedSystemProperties = node.config.systemProperties.collect { key, value -> "-D${key}=${value}" }.join(" ") + List esJavaOpts = [collectedSystemProperties] + esJavaOpts.add(node.config.jvmArgs) + if (Boolean.parseBoolean(System.getProperty('tests.asserts', 'true'))) { + // put the enable assertions options before other options to allow + // flexibility to disable assertions for specific packages or classes + // in the cluster-specific options + esJavaOpts.add("-ea") + esJavaOpts.add("-esa") + } + // we must add debug options inside the closure so the config is read at execution time, as + // gradle task options are not processed until the end of the configuration phase + if (node.config.debug) { + println 'Running elasticsearch in debug mode, suspending until connected on port 8000' + esJavaOpts.add('-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=8000') + } + node.env['ES_JAVA_OPTS'] = esJavaOpts.join(" ") + } + } + } + static Task configureExtraConfigFilesTask(String name, Project project, Task setup, NodeInfo node) { if (node.config.extraConfigFiles.isEmpty()) { return setup @@ -624,13 +650,6 @@ class ClusterFormationTasks { node.writeWrapperScript() } - // we must add debug options inside the closure so the config is read at execution time, as - // gradle task options are not processed until the end of the configuration phase - if (node.config.debug) { - println 'Running elasticsearch in debug mode, suspending until connected on port 8000' - node.env['ES_JAVA_OPTS'] = '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=8000' - } - node.getCommandString().eachLine { line -> logger.info(line) } if (logger.isInfoEnabled() || node.config.daemonize == false) { diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy index 5e67dfa55cfd4..7844ea77fc18f 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy @@ -180,15 +180,7 @@ class NodeInfo { } args.addAll("-E", "node.portsfile=true") - String collectedSystemProperties = config.systemProperties.collect { key, value -> "-D${key}=${value}" }.join(" ") - String esJavaOpts = config.jvmArgs.isEmpty() ? collectedSystemProperties : collectedSystemProperties + " " + config.jvmArgs - if (Boolean.parseBoolean(System.getProperty('tests.asserts', 'true'))) { - // put the enable assertions options before other options to allow - // flexibility to disable assertions for specific packages or classes - // in the cluster-specific options - esJavaOpts = String.join(" ", "-ea", "-esa", esJavaOpts) - } - env = ['ES_JAVA_OPTS': esJavaOpts] + env = [:] for (Map.Entry property : System.properties.entrySet()) { if (property.key.startsWith('tests.es.')) { args.add("-E") diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index dc2140a6086a4..7273dc9903958 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -288,7 +288,9 @@ Map expansions = [ 'permanent_bucket': s3PermanentBucket, 'permanent_base_path': s3PermanentBasePath, 'temporary_bucket': s3TemporaryBucket, - 'temporary_base_path': s3TemporaryBasePath + 'temporary_base_path': s3TemporaryBasePath, + 'ec2_bucket': s3TemporaryBucket, + 'ec2_base_path': s3TemporaryBasePath ] processTestResources { @@ -309,6 +311,10 @@ integTestCluster { /* Use a closure on the string to delay evaluation until tests are executed */ setting 's3.client.integration_test_permanent.endpoint', "http://${-> s3Fixture.addressAndPort}" setting 's3.client.integration_test_temporary.endpoint', "http://${-> s3Fixture.addressAndPort}" + setting 's3.client.integration_test_ec2.endpoint', "http://${-> s3Fixture.addressAndPort}" + + // to redirect InstanceProfileCredentialsProvider to custom auth point + systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", "http://${-> s3Fixture.addressAndPort}" } else { println "Using an external service to test the repository-s3 plugin" } diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java index 26bc7c962c450..5234643e9fd0c 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java @@ -37,6 +37,8 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; import static java.nio.charset.StandardCharsets.UTF_8; @@ -70,17 +72,24 @@ private AmazonS3Fixture(final String workingDir, final String permanentBucketNam @Override protected Response handle(final Request request) throws IOException { - final RequestHandler handler = handlers.retrieve(request.getMethod() + " " + request.getPath(), request.getParameters()); + final String nonAuthorizedPath = "* " + request.getMethod() + " " + request.getPath(); + final RequestHandler nonAuthorizedHandler = handlers.retrieve(nonAuthorizedPath, request.getParameters()); + if (nonAuthorizedHandler != null) { + return nonAuthorizedHandler.handle(request); + } + + final String authorizedPath = "A " + request.getMethod() + " " + request.getPath(); + final RequestHandler handler = handlers.retrieve(authorizedPath, request.getParameters()); if (handler != null) { final String authorization = request.getHeader("Authorization"); final String permittedBucket; - if (authorization.contains("s3_integration_test_permanent_access_key")) { + if (authorization != null && authorization.contains("s3_integration_test_permanent_access_key")) { final String sessionToken = request.getHeader("x-amz-security-token"); if (sessionToken != null) { return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Unexpected session token", ""); } permittedBucket = permanentBucketName; - } else if (authorization.contains("s3_integration_test_temporary_access_key")) { + } else if (authorization != null && authorization.contains("s3_integration_test_temporary_access_key")) { final String sessionToken = request.getHeader("x-amz-security-token"); if (sessionToken == null) { return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "No session token", ""); @@ -89,6 +98,15 @@ protected Response handle(final Request request) throws IOException { return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad session token", ""); } permittedBucket = temporaryBucketName; + } else if (authorization != null && authorization.contains("securitycredentials42_KEYID")) { + final String sessionToken = request.getHeader("x-amz-security-token"); + if (sessionToken == null) { + return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "No session token", ""); + } + if (sessionToken.equals("securitycredentials42_TKN") == false) { + return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad session token", ""); + } + permittedBucket = temporaryBucketName; } else { return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad access key", ""); } @@ -129,7 +147,7 @@ private static PathTrie defaultHandlers(final Map + objectsPaths("A HEAD /{bucket}").forEach(path -> handlers.insert(path, (request) -> { final String bucketName = request.getParam("bucket"); @@ -151,7 +169,7 @@ private static PathTrie defaultHandlers(final Map + objectsPaths("A PUT /{bucket}").forEach(path -> handlers.insert(path, (request) -> { final String destBucketName = request.getParam("bucket"); @@ -201,7 +219,7 @@ private static PathTrie defaultHandlers(final Map + objectsPaths("A DELETE /{bucket}").forEach(path -> handlers.insert(path, (request) -> { final String bucketName = request.getParam("bucket"); @@ -219,7 +237,7 @@ private static PathTrie defaultHandlers(final Map + objectsPaths("A GET /{bucket}").forEach(path -> handlers.insert(path, (request) -> { final String bucketName = request.getParam("bucket"); @@ -240,7 +258,7 @@ private static PathTrie defaultHandlers(final Map { + handlers.insert("A HEAD /{bucket}", (request) -> { String bucket = request.getParam("bucket"); if (Strings.hasText(bucket) && buckets.containsKey(bucket)) { return new Response(RestStatus.OK.getStatus(), TEXT_PLAIN_CONTENT_TYPE, EMPTY_BYTE); @@ -252,7 +270,7 @@ private static PathTrie defaultHandlers(final Map { + handlers.insert("A GET /{bucket}/", (request) -> { final String bucketName = request.getParam("bucket"); final Bucket bucket = buckets.get(bucketName); @@ -270,7 +288,7 @@ private static PathTrie defaultHandlers(final Map { + handlers.insert("A POST /", (request) -> { final List deletes = new ArrayList<>(); final List errors = new ArrayList<>(); @@ -312,6 +330,40 @@ private static PathTrie defaultHandlers(final Map credentialResponseFunction = prefix -> { + final Date expiration = new Date(new Date().getTime() + TimeUnit.DAYS.toMillis(1)); + final String response = "{" + + "\"AccessKeyId\": \"" + prefix + "_KEYID" + "\"," + + "\"Expiration\": \"" + DateUtils.formatISO8601Date(expiration) + "\"," + + "\"RoleArn\": \"" + prefix + "_ROLE" + "\"," + + "\"SecretAccessKey\": \"" + prefix + "_SCR_KEY" + "\"," + + "\"Token\": \"" + prefix + "_TKN" + "\"" + + "}"; + + final Map headers = new HashMap<>(contentType("application/json")); + return new Response(RestStatus.OK.getStatus(), headers, response.getBytes(UTF_8)); + }; + + // GET + // + // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html + handlers.insert("* GET /latest/meta-data/iam/security-credentials/", (request) -> { + final String response = "securitycredentials42"; + + final Map headers = new HashMap<>(contentType("text/plain")); + return new Response(RestStatus.OK.getStatus(), headers, response.getBytes(UTF_8)); + }); + + // GET + // + // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html + handlers.insert("* GET /latest/meta-data/iam/security-credentials/{credentials}", (request) -> { + final String credentials = request.getParam("credentials"); + return credentialResponseFunction.apply(credentials); + }); + return handlers; } diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml new file mode 100644 index 0000000000000..2df3b8290a19b --- /dev/null +++ b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml @@ -0,0 +1,243 @@ +# Integration tests for repository-s3 + +--- +setup: + + # Register repository with ec2 credentials + - do: + snapshot.create_repository: + repository: repository_ec2 + body: + type: s3 + settings: + bucket: ${ec2_bucket} + client: integration_test_ec2 + base_path: ${ec2_base_path} + canned_acl: private + storage_class: standard + +--- +"Snapshot and Restore with repository-s3 using ec2 credentials": + + # Get repository + - do: + snapshot.get_repository: + repository: repository_ec2 + + - match: { repository_ec2.settings.bucket : ${ec2_bucket} } + - match: { repository_ec2.settings.client : "integration_test_ec2" } + - match: { repository_ec2.settings.base_path : ${ec2_base_path} } + - match: { repository_ec2.settings.canned_acl : "private" } + - match: { repository_ec2.settings.storage_class : "standard" } + - is_false: repository_ec2.settings.access_key + - is_false: repository_ec2.settings.secret_key + - is_false: repository_ec2.settings.session_token + + # Index documents + - do: + bulk: + refresh: true + body: + - index: + _index: docs + _type: doc + _id: 1 + - snapshot: one + - index: + _index: docs + _type: doc + _id: 2 + - snapshot: one + - index: + _index: docs + _type: doc + _id: 3 + - snapshot: one + + - do: + count: + index: docs + + - match: {count: 3} + + # Create a first snapshot + - do: + snapshot.create: + repository: repository_ec2 + snapshot: snapshot-one + wait_for_completion: true + + - match: { snapshot.snapshot: snapshot-one } + - match: { snapshot.state : SUCCESS } + - match: { snapshot.include_global_state: true } + - match: { snapshot.shards.failed : 0 } + + - do: + snapshot.status: + repository: repository_ec2 + snapshot: snapshot-one + + - is_true: snapshots + - match: { snapshots.0.snapshot: snapshot-one } + - match: { snapshots.0.state : SUCCESS } + + # Index more documents + - do: + bulk: + refresh: true + body: + - index: + _index: docs + _type: doc + _id: 4 + - snapshot: two + - index: + _index: docs + _type: doc + _id: 5 + - snapshot: two + - index: + _index: docs + _type: doc + _id: 6 + - snapshot: two + - index: + _index: docs + _type: doc + _id: 7 + - snapshot: two + + - do: + count: + index: docs + + - match: {count: 7} + + # Create a second snapshot + - do: + snapshot.create: + repository: repository_ec2 + snapshot: snapshot-two + wait_for_completion: true + + - match: { snapshot.snapshot: snapshot-two } + - match: { snapshot.state : SUCCESS } + - match: { snapshot.shards.failed : 0 } + + - do: + snapshot.get: + repository: repository_ec2 + snapshot: snapshot-one,snapshot-two + + - is_true: snapshots + - match: { snapshots.0.state : SUCCESS } + - match: { snapshots.1.state : SUCCESS } + + # Delete the index + - do: + indices.delete: + index: docs + + # Restore the second snapshot + - do: + snapshot.restore: + repository: repository_ec2 + snapshot: snapshot-two + wait_for_completion: true + + - do: + count: + index: docs + + - match: {count: 7} + + # Delete the index again + - do: + indices.delete: + index: docs + + # Restore the first snapshot + - do: + snapshot.restore: + repository: repository_ec2 + snapshot: snapshot-one + wait_for_completion: true + + - do: + count: + index: docs + + - match: {count: 3} + + # Remove the snapshots + - do: + snapshot.delete: + repository: repository_ec2 + snapshot: snapshot-two + + - do: + snapshot.delete: + repository: repository_ec2 + snapshot: snapshot-one + +--- +"Register a repository with a non existing bucket": + + - do: + catch: /repository_exception/ + snapshot.create_repository: + repository: repository_ec2 + body: + type: s3 + settings: + bucket: zHHkfSqlbnBsbpSgvCYtxrEfFLqghXtyPvvvKPNBnRCicNHQLE + client: integration_test_temporary + +--- +"Register a repository with a non existing client": + + - do: + catch: /repository_exception/ + snapshot.create_repository: + repository: repository_ec2 + body: + type: s3 + settings: + bucket: repository_ec2 + client: unknown + +--- +"Get a non existing snapshot": + + - do: + catch: /snapshot_missing_exception/ + snapshot.get: + repository: repository_ec2 + snapshot: missing + +--- +"Delete a non existing snapshot": + + - do: + catch: /snapshot_missing_exception/ + snapshot.delete: + repository: repository_ec2 + snapshot: missing + +--- +"Restore a non existing snapshot": + + - do: + catch: /snapshot_restore_exception/ + snapshot.restore: + repository: repository_ec2 + snapshot: missing + wait_for_completion: true + +--- +teardown: + + # Remove our repository + - do: + snapshot.delete_repository: + repository: repository_ec2 From a56d1cd5d95098bcaf3d3fb949a7fc4531d03bbb Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Tue, 10 Jul 2018 14:41:35 +0200 Subject: [PATCH 02/12] excluded ec2_credentials from minio tests --- plugins/repository-s3/build.gradle | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 7273dc9903958..17d247583b524 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -271,7 +271,10 @@ if (useFixture && minioDistribution) { integTestMinioRunner.dependsOn(startMinio) integTestMinioRunner.finalizedBy(stopMinio) // Minio only supports a single access key, see https://github.com/minio/minio/pull/5968 - integTestMinioRunner.systemProperty 'tests.rest.blacklist', 'repository_s3/30_repository_temporary_credentials/*' + integTestMinioRunner.systemProperty 'tests.rest.blacklist', [ + 'repository_s3/30_repository_temporary_credentials/*', + 'repository_s3/40_repository_ec2_credentials/*' + ].join(",") project.check.dependsOn(integTestMinio) } From 5fe42270561cf2c8eaf1af753bdfc79d39aeafe6 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Tue, 10 Jul 2018 14:42:46 +0200 Subject: [PATCH 03/12] some gradle amendments on Alpar's review --- .../gradle/test/ClusterConfiguration.groovy | 3 ++ .../gradle/test/ClusterFormationTasks.groovy | 48 ++++++++----------- 2 files changed, 24 insertions(+), 27 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy index e6ce2744f1857..d6477e05b15d5 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy @@ -137,6 +137,9 @@ class ClusterConfiguration { this.project = project } + // **Note** for systemProperties, settings, keystoreFiles etc: + // value could be a GString that is evaluated to just a String + // there are cases when value depends on task that is not executed yet on configuration stage Map systemProperties = new HashMap<>() Map settings = new HashMap<>() diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index 24aee5c21d26d..3d86c7cf45cf4 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -202,7 +202,6 @@ class ClusterFormationTasks { setup = configureCreateKeystoreTask(taskName(prefix, node, 'createKeystore'), project, setup, node) setup = configureAddKeystoreSettingTasks(prefix, project, setup, node) setup = configureAddKeystoreFileTasks(prefix, project, setup, node) - setup = configureESJavaOpts(prefix, project, setup, node) if (node.config.plugins.isEmpty() == false) { if (node.nodeVersion == VersionProperties.elasticsearch) { @@ -413,31 +412,6 @@ class ClusterFormationTasks { return parentTask } - /** Configure ES JAVA OPTS - adds system properties, assertion flags, remote debug etc */ - static Task configureESJavaOpts(String parent, Project project, Task setup, NodeInfo node) { - return project.tasks.create(name: taskName(parent, node, 'configureESJavaOpts'), type: DefaultTask, dependsOn: setup) { - doLast { - String collectedSystemProperties = node.config.systemProperties.collect { key, value -> "-D${key}=${value}" }.join(" ") - List esJavaOpts = [collectedSystemProperties] - esJavaOpts.add(node.config.jvmArgs) - if (Boolean.parseBoolean(System.getProperty('tests.asserts', 'true'))) { - // put the enable assertions options before other options to allow - // flexibility to disable assertions for specific packages or classes - // in the cluster-specific options - esJavaOpts.add("-ea") - esJavaOpts.add("-esa") - } - // we must add debug options inside the closure so the config is read at execution time, as - // gradle task options are not processed until the end of the configuration phase - if (node.config.debug) { - println 'Running elasticsearch in debug mode, suspending until connected on port 8000' - esJavaOpts.add('-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=8000') - } - node.env['ES_JAVA_OPTS'] = esJavaOpts.join(" ") - } - } - } - static Task configureExtraConfigFilesTask(String name, Project project, Task setup, NodeInfo node) { if (node.config.extraConfigFiles.isEmpty()) { return setup @@ -629,7 +603,6 @@ class ClusterFormationTasks { /** Adds a task to start an elasticsearch node with the given configuration */ static Task configureStartTask(String name, Project project, Task setup, NodeInfo node) { - // this closure is converted into ant nodes by groovy's AntBuilder Closure antRunner = { AntBuilder ant -> ant.exec(executable: node.executable, spawn: node.config.daemonize, dir: node.cwd, taskname: 'elasticsearch') { @@ -667,6 +640,27 @@ class ClusterFormationTasks { } start.doLast(elasticsearchRunner) start.doFirst { + // Configure ES JAVA OPTS - adds system properties, assertion flags, remote debug etc + List esJavaOpts = [node.env.get('ES_JAVA_OPTS', '')] + String collectedSystemProperties = node.config.systemProperties.collect { key, value -> "-D${key}=${value}" }.join(" ") + esJavaOpts.add(collectedSystemProperties) + esJavaOpts.add(node.config.jvmArgs) + if (Boolean.parseBoolean(System.getProperty('tests.asserts', 'true'))) { + // put the enable assertions options before other options to allow + // flexibility to disable assertions for specific packages or classes + // in the cluster-specific options + esJavaOpts.add("-ea") + esJavaOpts.add("-esa") + } + // we must add debug options inside the closure so the config is read at execution time, as + // gradle task options are not processed until the end of the configuration phase + if (node.config.debug) { + println 'Running elasticsearch in debug mode, suspending until connected on port 8000' + esJavaOpts.add('-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=8000') + } + node.env['ES_JAVA_OPTS'] = esJavaOpts.join(" ") + + // project.logger.info("Starting node in ${node.clusterName} distribution: ${node.config.distribution}") } return start From 30e86c826aaa4bcf7439d9cb32b6e313d81796f5 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Tue, 10 Jul 2018 16:14:15 +0200 Subject: [PATCH 04/12] changes due to David's review: use own bucket/basepath for EC2 test; s3fixture clean up --- plugins/repository-s3/build.gradle | 18 ++++-- .../repositories/s3/AmazonS3Fixture.java | 58 ++++++++++++------- 2 files changed, 50 insertions(+), 26 deletions(-) diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 17d247583b524..aa0c7291a054b 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -89,11 +89,15 @@ String s3TemporarySessionToken = System.getenv("amazon_s3_session_token_temporar String s3TemporaryBucket = System.getenv("amazon_s3_bucket_temporary") String s3TemporaryBasePath = System.getenv("amazon_s3_base_path_temporary") +String s3EC2Bucket = System.getenv("amazon_s3_bucket_ec2") +String s3EC2BasePath = System.getenv("amazon_s3_base_path_ec2") + // If all these variables are missing then we are testing against the internal fixture instead, which has the following // credentials hard-coded in. if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3PermanentBasePath - && !s3TemporaryAccessKey && !s3TemporarySecretKey && !s3TemporaryBucket && !s3TemporaryBasePath && !s3TemporarySessionToken) { + && !s3TemporaryAccessKey && !s3TemporarySecretKey && !s3TemporaryBucket && !s3TemporaryBasePath && !s3TemporarySessionToken + && !s3EC2Bucket && !s3EC2BasePath) { s3PermanentAccessKey = 's3_integration_test_permanent_access_key' s3PermanentSecretKey = 's3_integration_test_permanent_secret_key' @@ -106,9 +110,13 @@ if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3P s3TemporaryBasePath = 'integration_test' s3TemporarySessionToken = 's3_integration_test_temporary_session_token' + s3EC2Bucket = 'ec2-bucket-test' + s3EC2BasePath = 'integration_test' + useFixture = true } else if (!s3PermanentAccessKey || !s3PermanentSecretKey || !s3PermanentBucket || !s3PermanentBasePath - || !s3TemporaryAccessKey || !s3TemporarySecretKey || !s3TemporaryBucket || !s3TemporaryBasePath || !s3TemporarySessionToken) { + || !s3TemporaryAccessKey || !s3TemporarySecretKey || !s3TemporaryBucket || !s3TemporaryBasePath || !s3TemporarySessionToken + || !s3EC2Bucket || !s3EC2BasePath) { throw new IllegalArgumentException("not all options specified to run against external S3 service") } @@ -284,7 +292,7 @@ task s3Fixture(type: AntFixture) { dependsOn testClasses env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }" executable = new File(project.runtimeJavaHome, 'bin/java') - args 'org.elasticsearch.repositories.s3.AmazonS3Fixture', baseDir, s3PermanentBucket, s3TemporaryBucket + args 'org.elasticsearch.repositories.s3.AmazonS3Fixture', baseDir, s3PermanentBucket, s3TemporaryBucket, s3EC2Bucket } Map expansions = [ @@ -292,8 +300,8 @@ Map expansions = [ 'permanent_base_path': s3PermanentBasePath, 'temporary_bucket': s3TemporaryBucket, 'temporary_base_path': s3TemporaryBasePath, - 'ec2_bucket': s3TemporaryBucket, - 'ec2_base_path': s3TemporaryBasePath + 'ec2_bucket': s3EC2Bucket, + 'ec2_base_path': s3EC2BasePath ] processTestResources { diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java index 5234643e9fd0c..ffab15dde8e98 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java @@ -48,6 +48,9 @@ * he implementation is based on official documentation available at https://docs.aws.amazon.com/AmazonS3/latest/API/. */ public class AmazonS3Fixture extends AbstractHttpFixture { + private static final String AUTH = "AUTH"; + private static final String NON_AUTH = "NON_AUTH"; + private static final String EC2_AUTH_PREFIX = "securitycredentials42"; /** List of the buckets stored on this test server **/ private final Map buckets = ConcurrentCollections.newConcurrentMap(); @@ -56,40 +59,47 @@ public class AmazonS3Fixture extends AbstractHttpFixture { private final PathTrie handlers; private final String permanentBucketName; private final String temporaryBucketName; + private final String ec2BucketName; /** * Creates a {@link AmazonS3Fixture} */ - private AmazonS3Fixture(final String workingDir, final String permanentBucketName, final String temporaryBucketName) { + private AmazonS3Fixture(final String workingDir, final String permanentBucketName, + final String temporaryBucketName, final String ec2BucketName) { super(workingDir); this.permanentBucketName = permanentBucketName; this.temporaryBucketName = temporaryBucketName; + this.ec2BucketName = ec2BucketName; this.buckets.put(permanentBucketName, new Bucket(permanentBucketName)); this.buckets.put(temporaryBucketName, new Bucket(temporaryBucketName)); + this.buckets.put(ec2BucketName, new Bucket(ec2BucketName)); this.handlers = defaultHandlers(buckets); } @Override protected Response handle(final Request request) throws IOException { - final String nonAuthorizedPath = "* " + request.getMethod() + " " + request.getPath(); + final String nonAuthorizedPath = NON_AUTH + " " + request.getMethod() + " " + request.getPath(); final RequestHandler nonAuthorizedHandler = handlers.retrieve(nonAuthorizedPath, request.getParameters()); if (nonAuthorizedHandler != null) { return nonAuthorizedHandler.handle(request); } - final String authorizedPath = "A " + request.getMethod() + " " + request.getPath(); + final String authorizedPath = AUTH + " " + request.getMethod() + " " + request.getPath(); final RequestHandler handler = handlers.retrieve(authorizedPath, request.getParameters()); if (handler != null) { final String authorization = request.getHeader("Authorization"); + if (authorization == null) { + return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad access key", ""); + } final String permittedBucket; - if (authorization != null && authorization.contains("s3_integration_test_permanent_access_key")) { + if (authorization.contains("s3_integration_test_permanent_access_key")) { final String sessionToken = request.getHeader("x-amz-security-token"); if (sessionToken != null) { return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Unexpected session token", ""); } permittedBucket = permanentBucketName; - } else if (authorization != null && authorization.contains("s3_integration_test_temporary_access_key")) { + } else if (authorization.contains("s3_integration_test_temporary_access_key")) { final String sessionToken = request.getHeader("x-amz-security-token"); if (sessionToken == null) { return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "No session token", ""); @@ -98,15 +108,15 @@ protected Response handle(final Request request) throws IOException { return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad session token", ""); } permittedBucket = temporaryBucketName; - } else if (authorization != null && authorization.contains("securitycredentials42_KEYID")) { + } else if (authorization.contains(EC2_AUTH_PREFIX + "_KEYID")) { final String sessionToken = request.getHeader("x-amz-security-token"); if (sessionToken == null) { return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "No session token", ""); } - if (sessionToken.equals("securitycredentials42_TKN") == false) { + if (sessionToken.equals(EC2_AUTH_PREFIX + "_TKN") == false) { return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad session token", ""); } - permittedBucket = temporaryBucketName; + permittedBucket = ec2BucketName; } else { return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad access key", ""); } @@ -131,12 +141,15 @@ protected Response handle(final Request request) throws IOException { } public static void main(final String[] args) throws Exception { - if (args == null || args.length != 3) { + if (args == null || args.length != 4) { throw new IllegalArgumentException( - "AmazonS3Fixture "); + "AmazonS3Fixture " + + " " + + " " + + " "); } - final AmazonS3Fixture fixture = new AmazonS3Fixture(args[0], args[1], args[2]); + final AmazonS3Fixture fixture = new AmazonS3Fixture(args[0], args[1], args[2], args[3]); fixture.listen(); } @@ -147,7 +160,7 @@ private static PathTrie defaultHandlers(final Map + objectsPaths(AUTH + " HEAD /{bucket}").forEach(path -> handlers.insert(path, (request) -> { final String bucketName = request.getParam("bucket"); @@ -169,7 +182,7 @@ private static PathTrie defaultHandlers(final Map + objectsPaths(AUTH + " PUT /{bucket}").forEach(path -> handlers.insert(path, (request) -> { final String destBucketName = request.getParam("bucket"); @@ -219,7 +232,7 @@ private static PathTrie defaultHandlers(final Map + objectsPaths(AUTH + " DELETE /{bucket}").forEach(path -> handlers.insert(path, (request) -> { final String bucketName = request.getParam("bucket"); @@ -237,7 +250,7 @@ private static PathTrie defaultHandlers(final Map + objectsPaths(AUTH + " GET /{bucket}").forEach(path -> handlers.insert(path, (request) -> { final String bucketName = request.getParam("bucket"); @@ -258,7 +271,7 @@ private static PathTrie defaultHandlers(final Map { + handlers.insert(AUTH + " HEAD /{bucket}", (request) -> { String bucket = request.getParam("bucket"); if (Strings.hasText(bucket) && buckets.containsKey(bucket)) { return new Response(RestStatus.OK.getStatus(), TEXT_PLAIN_CONTENT_TYPE, EMPTY_BYTE); @@ -270,7 +283,7 @@ private static PathTrie defaultHandlers(final Map { + handlers.insert(AUTH + " GET /{bucket}/", (request) -> { final String bucketName = request.getParam("bucket"); final Bucket bucket = buckets.get(bucketName); @@ -288,7 +301,7 @@ private static PathTrie defaultHandlers(final Map { + handlers.insert(AUTH + " POST /", (request) -> { final List deletes = new ArrayList<>(); final List errors = new ArrayList<>(); @@ -349,8 +362,8 @@ private static PathTrie defaultHandlers(final Map { - final String response = "securitycredentials42"; + handlers.insert(NON_AUTH + " GET /latest/meta-data/iam/security-credentials/", (request) -> { + final String response = EC2_AUTH_PREFIX; final Map headers = new HashMap<>(contentType("text/plain")); return new Response(RestStatus.OK.getStatus(), headers, response.getBytes(UTF_8)); @@ -359,8 +372,11 @@ private static PathTrie defaultHandlers(final Map { + handlers.insert(NON_AUTH + " GET /latest/meta-data/iam/security-credentials/{credentials}", (request) -> { final String credentials = request.getParam("credentials"); + if (EC2_AUTH_PREFIX.equals(credentials) == false) { + return new Response(RestStatus.NOT_FOUND.getStatus(), new HashMap<>(), "unknown credentials".getBytes(UTF_8)); + } return credentialResponseFunction.apply(credentials); }); From 46dded931764911ab1dc9445c89eb84c17db39b0 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Tue, 10 Jul 2018 21:15:26 +0200 Subject: [PATCH 05/12] clean AmazonS3Fixture from hard-coded keys --- plugins/repository-s3/build.gradle | 19 +++++- .../repositories/s3/AmazonS3Fixture.java | 61 +++++++++++-------- 2 files changed, 50 insertions(+), 30 deletions(-) diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index aa0c7291a054b..1cddbff9628ae 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -91,13 +91,15 @@ String s3TemporaryBasePath = System.getenv("amazon_s3_base_path_temporary") String s3EC2Bucket = System.getenv("amazon_s3_bucket_ec2") String s3EC2BasePath = System.getenv("amazon_s3_base_path_ec2") +String s3EC2AccessKey = System.getenv("amazon_s3_access_key_ec2") +String s3EC2Token = System.getenv("amazon_s3_token_ec2") // If all these variables are missing then we are testing against the internal fixture instead, which has the following // credentials hard-coded in. if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3PermanentBasePath && !s3TemporaryAccessKey && !s3TemporarySecretKey && !s3TemporaryBucket && !s3TemporaryBasePath && !s3TemporarySessionToken - && !s3EC2Bucket && !s3EC2BasePath) { + && !s3EC2Bucket && !s3EC2BasePath && !s3EC2AccessKey) { s3PermanentAccessKey = 's3_integration_test_permanent_access_key' s3PermanentSecretKey = 's3_integration_test_permanent_secret_key' @@ -112,11 +114,13 @@ if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3P s3EC2Bucket = 'ec2-bucket-test' s3EC2BasePath = 'integration_test' + s3EC2AccessKey = 'ec2_accessKey' + s3EC2Token = 'ec2_tkn' useFixture = true } else if (!s3PermanentAccessKey || !s3PermanentSecretKey || !s3PermanentBucket || !s3PermanentBasePath || !s3TemporaryAccessKey || !s3TemporarySecretKey || !s3TemporaryBucket || !s3TemporaryBasePath || !s3TemporarySessionToken - || !s3EC2Bucket || !s3EC2BasePath) { + || !s3EC2Bucket || !s3EC2BasePath || !s3EC2AccessKey) { throw new IllegalArgumentException("not all options specified to run against external S3 service") } @@ -291,8 +295,17 @@ if (useFixture && minioDistribution) { task s3Fixture(type: AntFixture) { dependsOn testClasses env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }" + env 'S3FIXTURE_WORKINGDIR', "${baseDir}" + env 'S3FIXTURE_PERMANENT_BUCKET_NAME', "${s3PermanentBucket}" + env 'S3FIXTURE_PERMANENT_KEY', "${s3PermanentAccessKey}" + env 'S3FIXTURE_TEMPORARY_BUCKET_NAME', "${s3TemporaryBucket}" + env 'S3FIXTURE_TEMPORARY_KEY', "${s3TemporaryAccessKey}" + env 'S3FIXTURE_TEMPORARY_SESSION_TOKEN', "${s3TemporarySessionToken}" + env 'S3FIXTURE_EC2_BUCKET_NAME', "${s3EC2Bucket}" + env 'S3FIXTURE_EC2_KEY', "${s3EC2AccessKey}" + env 'S3FIXTURE_EC2_TOKEN', "${s3EC2Token}" executable = new File(project.runtimeJavaHome, 'bin/java') - args 'org.elasticsearch.repositories.s3.AmazonS3Fixture', baseDir, s3PermanentBucket, s3TemporaryBucket, s3EC2Bucket + args 'org.elasticsearch.repositories.s3.AmazonS3Fixture', baseDir } Map expansions = [ diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java index ffab15dde8e98..3602105b27350 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.repositories.s3; +import org.elasticsearch.common.TriFunction; import org.elasticsearch.test.fixture.AbstractHttpFixture; import com.amazonaws.util.DateUtils; import org.elasticsearch.common.Strings; @@ -38,7 +39,6 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.TimeUnit; -import java.util.function.Function; import static java.nio.charset.StandardCharsets.UTF_8; @@ -50,7 +50,6 @@ public class AmazonS3Fixture extends AbstractHttpFixture { private static final String AUTH = "AUTH"; private static final String NON_AUTH = "NON_AUTH"; - private static final String EC2_AUTH_PREFIX = "securitycredentials42"; /** List of the buckets stored on this test server **/ private final Map buckets = ConcurrentCollections.newConcurrentMap(); @@ -58,18 +57,27 @@ public class AmazonS3Fixture extends AbstractHttpFixture { /** Request handlers for the requests made by the S3 client **/ private final PathTrie handlers; private final String permanentBucketName; + private final String permanentKey; private final String temporaryBucketName; + private final String temporaryKey; + private final String temporarySessionToken; private final String ec2BucketName; + private final String ec2Key; + private final String ec2Token; /** * Creates a {@link AmazonS3Fixture} */ - private AmazonS3Fixture(final String workingDir, final String permanentBucketName, - final String temporaryBucketName, final String ec2BucketName) { + private AmazonS3Fixture(final String workingDir) { super(workingDir); - this.permanentBucketName = permanentBucketName; - this.temporaryBucketName = temporaryBucketName; - this.ec2BucketName = ec2BucketName; + this.permanentBucketName = envVar("S3FIXTURE_PERMANENT_BUCKET_NAME"); + this.permanentKey = envVar("S3FIXTURE_PERMANENT_KEY"); + this.temporaryBucketName = envVar("S3FIXTURE_TEMPORARY_BUCKET_NAME"); + this.temporaryKey = envVar("S3FIXTURE_TEMPORARY_KEY"); + this.temporarySessionToken = envVar("S3FIXTURE_TEMPORARY_SESSION_TOKEN"); + this.ec2BucketName = envVar("S3FIXTURE_EC2_BUCKET_NAME"); + this.ec2Key = envVar("S3FIXTURE_EC2_KEY"); + this.ec2Token = envVar("S3FIXTURE_EC2_TOKEN"); this.buckets.put(permanentBucketName, new Bucket(permanentBucketName)); this.buckets.put(temporaryBucketName, new Bucket(temporaryBucketName)); @@ -77,6 +85,10 @@ private AmazonS3Fixture(final String workingDir, final String permanentBucketNam this.handlers = defaultHandlers(buckets); } + private String envVar(String varName) { + return Objects.requireNonNull(System.getenv(varName), "env variable '" + varName + "' is missing"); + } + @Override protected Response handle(final Request request) throws IOException { final String nonAuthorizedPath = NON_AUTH + " " + request.getMethod() + " " + request.getPath(); @@ -93,27 +105,27 @@ protected Response handle(final Request request) throws IOException { return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad access key", ""); } final String permittedBucket; - if (authorization.contains("s3_integration_test_permanent_access_key")) { + if (authorization.contains(permanentKey)) { final String sessionToken = request.getHeader("x-amz-security-token"); if (sessionToken != null) { return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Unexpected session token", ""); } permittedBucket = permanentBucketName; - } else if (authorization.contains("s3_integration_test_temporary_access_key")) { + } else if (authorization.contains(temporaryKey)) { final String sessionToken = request.getHeader("x-amz-security-token"); if (sessionToken == null) { return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "No session token", ""); } - if (sessionToken.equals("s3_integration_test_temporary_session_token") == false) { + if (sessionToken.equals(temporarySessionToken) == false) { return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad session token", ""); } permittedBucket = temporaryBucketName; - } else if (authorization.contains(EC2_AUTH_PREFIX + "_KEYID")) { + } else if (authorization.contains(ec2Key)) { final String sessionToken = request.getHeader("x-amz-security-token"); if (sessionToken == null) { return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "No session token", ""); } - if (sessionToken.equals(EC2_AUTH_PREFIX + "_TKN") == false) { + if (sessionToken.equals(ec2Token) == false) { return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad session token", ""); } permittedBucket = ec2BucketName; @@ -141,20 +153,15 @@ protected Response handle(final Request request) throws IOException { } public static void main(final String[] args) throws Exception { - if (args == null || args.length != 4) { - throw new IllegalArgumentException( - "AmazonS3Fixture " - + " " - + " " - + " "); + if (args == null || args.length != 1) { + throw new IllegalArgumentException("AmazonS3Fixture defaultHandlers(final Map buckets) { + private PathTrie defaultHandlers(final Map buckets) { final PathTrie handlers = new PathTrie<>(RestUtils.REST_DECODER); // HEAD Object @@ -345,14 +352,14 @@ private static PathTrie defaultHandlers(final Map credentialResponseFunction = prefix -> { + TriFunction credentialResponseFunction = (prefix, key, token) -> { final Date expiration = new Date(new Date().getTime() + TimeUnit.DAYS.toMillis(1)); final String response = "{" - + "\"AccessKeyId\": \"" + prefix + "_KEYID" + "\"," + + "\"AccessKeyId\": \"" + key + "\"," + "\"Expiration\": \"" + DateUtils.formatISO8601Date(expiration) + "\"," + "\"RoleArn\": \"" + prefix + "_ROLE" + "\"," + "\"SecretAccessKey\": \"" + prefix + "_SCR_KEY" + "\"," - + "\"Token\": \"" + prefix + "_TKN" + "\"" + + "\"Token\": \"" + token + "\"" + "}"; final Map headers = new HashMap<>(contentType("application/json")); @@ -363,7 +370,7 @@ private static PathTrie defaultHandlers(final Map { - final String response = EC2_AUTH_PREFIX; + final String response = ec2Key; final Map headers = new HashMap<>(contentType("text/plain")); return new Response(RestStatus.OK.getStatus(), headers, response.getBytes(UTF_8)); @@ -374,10 +381,10 @@ private static PathTrie defaultHandlers(final Map { final String credentials = request.getParam("credentials"); - if (EC2_AUTH_PREFIX.equals(credentials) == false) { + if (ec2Key.equals(credentials) == false) { return new Response(RestStatus.NOT_FOUND.getStatus(), new HashMap<>(), "unknown credentials".getBytes(UTF_8)); } - return credentialResponseFunction.apply(credentials); + return credentialResponseFunction.apply(credentials, ec2Key, ec2Token); }); return handlers; From ccfa0c5e7bfe0f1d51fc1f275ab6a6022debf99a Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Wed, 11 Jul 2018 14:18:16 +0200 Subject: [PATCH 06/12] added s3EC2ProfileName to gradle; added nonAuthPath / authPath to AmazonS3Fixture --- plugins/repository-s3/build.gradle | 23 ++++--- .../repositories/s3/AmazonS3Fixture.java | 69 ++++++++++++------- 2 files changed, 59 insertions(+), 33 deletions(-) diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 1cddbff9628ae..6907eb047b98a 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -90,6 +90,7 @@ String s3TemporaryBucket = System.getenv("amazon_s3_bucket_temporary") String s3TemporaryBasePath = System.getenv("amazon_s3_base_path_temporary") String s3EC2Bucket = System.getenv("amazon_s3_bucket_ec2") +String s3EC2ProfileName = System.getenv("amazon_s3_profile_ec2") String s3EC2BasePath = System.getenv("amazon_s3_base_path_ec2") String s3EC2AccessKey = System.getenv("amazon_s3_access_key_ec2") String s3EC2Token = System.getenv("amazon_s3_token_ec2") @@ -99,7 +100,7 @@ String s3EC2Token = System.getenv("amazon_s3_token_ec2") if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3PermanentBasePath && !s3TemporaryAccessKey && !s3TemporarySecretKey && !s3TemporaryBucket && !s3TemporaryBasePath && !s3TemporarySessionToken - && !s3EC2Bucket && !s3EC2BasePath && !s3EC2AccessKey) { + && !s3EC2Bucket && !s3EC2ProfileName && !s3EC2BasePath && !s3EC2AccessKey && !s3EC2Token) { s3PermanentAccessKey = 's3_integration_test_permanent_access_key' s3PermanentSecretKey = 's3_integration_test_permanent_secret_key' @@ -113,6 +114,7 @@ if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3P s3TemporarySessionToken = 's3_integration_test_temporary_session_token' s3EC2Bucket = 'ec2-bucket-test' + s3EC2ProfileName = 'ec2_profile' s3EC2BasePath = 'integration_test' s3EC2AccessKey = 'ec2_accessKey' s3EC2Token = 'ec2_tkn' @@ -120,7 +122,7 @@ if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3P useFixture = true } else if (!s3PermanentAccessKey || !s3PermanentSecretKey || !s3PermanentBucket || !s3PermanentBasePath || !s3TemporaryAccessKey || !s3TemporarySecretKey || !s3TemporaryBucket || !s3TemporaryBasePath || !s3TemporarySessionToken - || !s3EC2Bucket || !s3EC2BasePath || !s3EC2AccessKey) { + || !s3EC2Bucket || !s3EC2ProfileName || !s3EC2BasePath || !s3EC2AccessKey || !s3EC2Token) { throw new IllegalArgumentException("not all options specified to run against external S3 service") } @@ -296,14 +298,15 @@ task s3Fixture(type: AntFixture) { dependsOn testClasses env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }" env 'S3FIXTURE_WORKINGDIR', "${baseDir}" - env 'S3FIXTURE_PERMANENT_BUCKET_NAME', "${s3PermanentBucket}" - env 'S3FIXTURE_PERMANENT_KEY', "${s3PermanentAccessKey}" - env 'S3FIXTURE_TEMPORARY_BUCKET_NAME', "${s3TemporaryBucket}" - env 'S3FIXTURE_TEMPORARY_KEY', "${s3TemporaryAccessKey}" - env 'S3FIXTURE_TEMPORARY_SESSION_TOKEN', "${s3TemporarySessionToken}" - env 'S3FIXTURE_EC2_BUCKET_NAME', "${s3EC2Bucket}" - env 'S3FIXTURE_EC2_KEY', "${s3EC2AccessKey}" - env 'S3FIXTURE_EC2_TOKEN', "${s3EC2Token}" + env 'S3FIXTURE_PERMANENT_BUCKET_NAME', s3PermanentBucket + env 'S3FIXTURE_PERMANENT_KEY', s3PermanentAccessKey + env 'S3FIXTURE_TEMPORARY_BUCKET_NAME', s3TemporaryBucket + env 'S3FIXTURE_TEMPORARY_KEY', s3TemporaryAccessKey + env 'S3FIXTURE_TEMPORARY_SESSION_TOKEN', s3TemporarySessionToken + env 'S3FIXTURE_EC2_BUCKET_NAME', s3EC2Bucket + env 'S3FIXTURE_EC2_PROFILE_NAME', s3EC2ProfileName + env 'S3FIXTURE_EC2_KEY', s3EC2AccessKey + env 'S3FIXTURE_EC2_TOKEN', s3EC2Token executable = new File(project.runtimeJavaHome, 'bin/java') args 'org.elasticsearch.repositories.s3.AmazonS3Fixture', baseDir } diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java index 3602105b27350..c880943af724b 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java @@ -18,6 +18,11 @@ */ package org.elasticsearch.repositories.s3; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpHead; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; import org.elasticsearch.common.TriFunction; import org.elasticsearch.test.fixture.AbstractHttpFixture; import com.amazonaws.util.DateUtils; @@ -62,6 +67,7 @@ public class AmazonS3Fixture extends AbstractHttpFixture { private final String temporaryKey; private final String temporarySessionToken; private final String ec2BucketName; + private final String ec2ProfileName; private final String ec2Key; private final String ec2Token; @@ -76,28 +82,45 @@ private AmazonS3Fixture(final String workingDir) { this.temporaryKey = envVar("S3FIXTURE_TEMPORARY_KEY"); this.temporarySessionToken = envVar("S3FIXTURE_TEMPORARY_SESSION_TOKEN"); this.ec2BucketName = envVar("S3FIXTURE_EC2_BUCKET_NAME"); + this.ec2ProfileName = envVar("S3FIXTURE_EC2_PROFILE_NAME"); this.ec2Key = envVar("S3FIXTURE_EC2_KEY"); this.ec2Token = envVar("S3FIXTURE_EC2_TOKEN"); - this.buckets.put(permanentBucketName, new Bucket(permanentBucketName)); - this.buckets.put(temporaryBucketName, new Bucket(temporaryBucketName)); - this.buckets.put(ec2BucketName, new Bucket(ec2BucketName)); + for (String bucketName : new String[]{permanentBucketName, temporaryBucketName, ec2BucketName}) { + this.buckets.put(bucketName, new Bucket(bucketName)); + } this.handlers = defaultHandlers(buckets); } - private String envVar(String varName) { + private static String envVar(String varName) { return Objects.requireNonNull(System.getenv(varName), "env variable '" + varName + "' is missing"); } + private static String nonAuthPath(Request request) { + return nonAuthPath(request.getMethod(), request.getPath()); + } + + private static String nonAuthPath(String method, String path) { + return NON_AUTH + " " + method + " " + path; + } + + private static String authPath(Request request) { + return authPath(request.getMethod(), request.getPath()); + } + + private static String authPath(String method, String path) { + return AUTH + " " + method + " " + path; + } + @Override protected Response handle(final Request request) throws IOException { - final String nonAuthorizedPath = NON_AUTH + " " + request.getMethod() + " " + request.getPath(); + final String nonAuthorizedPath = nonAuthPath(request); final RequestHandler nonAuthorizedHandler = handlers.retrieve(nonAuthorizedPath, request.getParameters()); if (nonAuthorizedHandler != null) { return nonAuthorizedHandler.handle(request); } - final String authorizedPath = AUTH + " " + request.getMethod() + " " + request.getPath(); + final String authorizedPath = authPath(request); final RequestHandler handler = handlers.retrieve(authorizedPath, request.getParameters()); if (handler != null) { final String authorization = request.getHeader("Authorization"); @@ -154,7 +177,7 @@ protected Response handle(final Request request) throws IOException { public static void main(final String[] args) throws Exception { if (args == null || args.length != 1) { - throw new IllegalArgumentException("AmazonS3Fixture "); } final AmazonS3Fixture fixture = new AmazonS3Fixture(args[0]); fixture.listen(); @@ -167,7 +190,7 @@ private PathTrie defaultHandlers(final Map bucke // HEAD Object // // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectHEAD.html - objectsPaths(AUTH + " HEAD /{bucket}").forEach(path -> + objectsPaths(authPath(HttpHead.METHOD_NAME, "/{bucket}")).forEach(path -> handlers.insert(path, (request) -> { final String bucketName = request.getParam("bucket"); @@ -189,7 +212,7 @@ private PathTrie defaultHandlers(final Map bucke // PUT Object // // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html - objectsPaths(AUTH + " PUT /{bucket}").forEach(path -> + objectsPaths(authPath(HttpPut.METHOD_NAME, "/{bucket}")).forEach(path -> handlers.insert(path, (request) -> { final String destBucketName = request.getParam("bucket"); @@ -239,7 +262,7 @@ private PathTrie defaultHandlers(final Map bucke // DELETE Object // // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectDELETE.html - objectsPaths(AUTH + " DELETE /{bucket}").forEach(path -> + objectsPaths(authPath(HttpDelete.METHOD_NAME, "/{bucket}")).forEach(path -> handlers.insert(path, (request) -> { final String bucketName = request.getParam("bucket"); @@ -257,7 +280,7 @@ private PathTrie defaultHandlers(final Map bucke // GET Object // // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html - objectsPaths(AUTH + " GET /{bucket}").forEach(path -> + objectsPaths(authPath(HttpGet.METHOD_NAME, "/{bucket}")).forEach(path -> handlers.insert(path, (request) -> { final String bucketName = request.getParam("bucket"); @@ -278,7 +301,7 @@ private PathTrie defaultHandlers(final Map bucke // HEAD Bucket // // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketHEAD.html - handlers.insert(AUTH + " HEAD /{bucket}", (request) -> { + handlers.insert(authPath(HttpHead.METHOD_NAME, "/{bucket}"), (request) -> { String bucket = request.getParam("bucket"); if (Strings.hasText(bucket) && buckets.containsKey(bucket)) { return new Response(RestStatus.OK.getStatus(), TEXT_PLAIN_CONTENT_TYPE, EMPTY_BYTE); @@ -290,7 +313,7 @@ private PathTrie defaultHandlers(final Map bucke // GET Bucket (List Objects) Version 1 // // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGET.html - handlers.insert(AUTH + " GET /{bucket}/", (request) -> { + handlers.insert(authPath(HttpGet.METHOD_NAME, "/{bucket}/"), (request) -> { final String bucketName = request.getParam("bucket"); final Bucket bucket = buckets.get(bucketName); @@ -308,7 +331,7 @@ private PathTrie defaultHandlers(final Map bucke // Delete Multiple Objects // // https://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html - handlers.insert(AUTH + " POST /", (request) -> { + handlers.insert(authPath(HttpPost.METHOD_NAME, "/"), (request) -> { final List deletes = new ArrayList<>(); final List errors = new ArrayList<>(); @@ -352,13 +375,13 @@ private PathTrie defaultHandlers(final Map bucke // non-authorized requests - TriFunction credentialResponseFunction = (prefix, key, token) -> { + TriFunction credentialResponseFunction = (profileName, key, token) -> { final Date expiration = new Date(new Date().getTime() + TimeUnit.DAYS.toMillis(1)); final String response = "{" + "\"AccessKeyId\": \"" + key + "\"," + "\"Expiration\": \"" + DateUtils.formatISO8601Date(expiration) + "\"," - + "\"RoleArn\": \"" + prefix + "_ROLE" + "\"," - + "\"SecretAccessKey\": \"" + prefix + "_SCR_KEY" + "\"," + + "\"RoleArn\": \"" + profileName + "_ROLE" + "\"," + + "\"SecretAccessKey\": \"" + profileName + "_SCR_KEY" + "\"," + "\"Token\": \"" + token + "\"" + "}"; @@ -369,8 +392,8 @@ private PathTrie defaultHandlers(final Map bucke // GET // // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html - handlers.insert(NON_AUTH + " GET /latest/meta-data/iam/security-credentials/", (request) -> { - final String response = ec2Key; + handlers.insert(nonAuthPath(HttpGet.METHOD_NAME, "/latest/meta-data/iam/security-credentials/"), (request) -> { + final String response = ec2ProfileName; final Map headers = new HashMap<>(contentType("text/plain")); return new Response(RestStatus.OK.getStatus(), headers, response.getBytes(UTF_8)); @@ -379,12 +402,12 @@ private PathTrie defaultHandlers(final Map bucke // GET // // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html - handlers.insert(NON_AUTH + " GET /latest/meta-data/iam/security-credentials/{credentials}", (request) -> { - final String credentials = request.getParam("credentials"); - if (ec2Key.equals(credentials) == false) { + handlers.insert(nonAuthPath(HttpGet.METHOD_NAME, "/latest/meta-data/iam/security-credentials/{profileName}"), (request) -> { + final String profileName = request.getParam("profileName"); + if (ec2ProfileName.equals(profileName) == false) { return new Response(RestStatus.NOT_FOUND.getStatus(), new HashMap<>(), "unknown credentials".getBytes(UTF_8)); } - return credentialResponseFunction.apply(credentials, ec2Key, ec2Token); + return credentialResponseFunction.apply(profileName, ec2Key, ec2Token); }); return handlers; From c7af0b647163ce8d6cbf90e00ed359f57013c5b4 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Wed, 11 Jul 2018 17:44:38 +0200 Subject: [PATCH 07/12] polish of AmazonS3Fixture --- plugins/repository-s3/build.gradle | 7 +- .../repositories/s3/AmazonS3Fixture.java | 111 +++++++++--------- 2 files changed, 58 insertions(+), 60 deletions(-) diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 6907eb047b98a..31ea83c4764d5 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -90,7 +90,6 @@ String s3TemporaryBucket = System.getenv("amazon_s3_bucket_temporary") String s3TemporaryBasePath = System.getenv("amazon_s3_base_path_temporary") String s3EC2Bucket = System.getenv("amazon_s3_bucket_ec2") -String s3EC2ProfileName = System.getenv("amazon_s3_profile_ec2") String s3EC2BasePath = System.getenv("amazon_s3_base_path_ec2") String s3EC2AccessKey = System.getenv("amazon_s3_access_key_ec2") String s3EC2Token = System.getenv("amazon_s3_token_ec2") @@ -100,7 +99,7 @@ String s3EC2Token = System.getenv("amazon_s3_token_ec2") if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3PermanentBasePath && !s3TemporaryAccessKey && !s3TemporarySecretKey && !s3TemporaryBucket && !s3TemporaryBasePath && !s3TemporarySessionToken - && !s3EC2Bucket && !s3EC2ProfileName && !s3EC2BasePath && !s3EC2AccessKey && !s3EC2Token) { + && !s3EC2Bucket && !s3EC2BasePath && !s3EC2AccessKey && !s3EC2Token) { s3PermanentAccessKey = 's3_integration_test_permanent_access_key' s3PermanentSecretKey = 's3_integration_test_permanent_secret_key' @@ -114,7 +113,6 @@ if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3P s3TemporarySessionToken = 's3_integration_test_temporary_session_token' s3EC2Bucket = 'ec2-bucket-test' - s3EC2ProfileName = 'ec2_profile' s3EC2BasePath = 'integration_test' s3EC2AccessKey = 'ec2_accessKey' s3EC2Token = 'ec2_tkn' @@ -122,7 +120,7 @@ if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3P useFixture = true } else if (!s3PermanentAccessKey || !s3PermanentSecretKey || !s3PermanentBucket || !s3PermanentBasePath || !s3TemporaryAccessKey || !s3TemporarySecretKey || !s3TemporaryBucket || !s3TemporaryBasePath || !s3TemporarySessionToken - || !s3EC2Bucket || !s3EC2ProfileName || !s3EC2BasePath || !s3EC2AccessKey || !s3EC2Token) { + || !s3EC2Bucket || !s3EC2BasePath || !s3EC2AccessKey || !s3EC2Token) { throw new IllegalArgumentException("not all options specified to run against external S3 service") } @@ -304,7 +302,6 @@ task s3Fixture(type: AntFixture) { env 'S3FIXTURE_TEMPORARY_KEY', s3TemporaryAccessKey env 'S3FIXTURE_TEMPORARY_SESSION_TOKEN', s3TemporarySessionToken env 'S3FIXTURE_EC2_BUCKET_NAME', s3EC2Bucket - env 'S3FIXTURE_EC2_PROFILE_NAME', s3EC2ProfileName env 'S3FIXTURE_EC2_KEY', s3EC2AccessKey env 'S3FIXTURE_EC2_TOKEN', s3EC2Token executable = new File(project.runtimeJavaHome, 'bin/java') diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java index c880943af724b..2e45401490aa6 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java @@ -38,6 +38,7 @@ import java.io.IOException; import java.io.InputStreamReader; import java.util.ArrayList; +import java.util.Collection; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -56,46 +57,33 @@ public class AmazonS3Fixture extends AbstractHttpFixture { private static final String AUTH = "AUTH"; private static final String NON_AUTH = "NON_AUTH"; + private static final String EC2_PROFILE = "ec2Profile"; + /** List of the buckets stored on this test server **/ private final Map buckets = ConcurrentCollections.newConcurrentMap(); /** Request handlers for the requests made by the S3 client **/ private final PathTrie handlers; - private final String permanentBucketName; - private final String permanentKey; - private final String temporaryBucketName; - private final String temporaryKey; - private final String temporarySessionToken; - private final String ec2BucketName; - private final String ec2ProfileName; - private final String ec2Key; - private final String ec2Token; + + private final Bucket permanentBucket; + private final Bucket temporaryBucket; + private final Bucket ec2Bucket; /** * Creates a {@link AmazonS3Fixture} */ private AmazonS3Fixture(final String workingDir) { super(workingDir); - this.permanentBucketName = envVar("S3FIXTURE_PERMANENT_BUCKET_NAME"); - this.permanentKey = envVar("S3FIXTURE_PERMANENT_KEY"); - this.temporaryBucketName = envVar("S3FIXTURE_TEMPORARY_BUCKET_NAME"); - this.temporaryKey = envVar("S3FIXTURE_TEMPORARY_KEY"); - this.temporarySessionToken = envVar("S3FIXTURE_TEMPORARY_SESSION_TOKEN"); - this.ec2BucketName = envVar("S3FIXTURE_EC2_BUCKET_NAME"); - this.ec2ProfileName = envVar("S3FIXTURE_EC2_PROFILE_NAME"); - this.ec2Key = envVar("S3FIXTURE_EC2_KEY"); - this.ec2Token = envVar("S3FIXTURE_EC2_TOKEN"); - - for (String bucketName : new String[]{permanentBucketName, temporaryBucketName, ec2BucketName}) { - this.buckets.put(bucketName, new Bucket(bucketName)); + this.permanentBucket = new Bucket("S3FIXTURE_PERMANENT", false); + this.temporaryBucket = new Bucket("S3FIXTURE_TEMPORARY"); + this.ec2Bucket = new Bucket("S3FIXTURE_EC2"); + + for (Bucket bucket : new Bucket[]{permanentBucket, temporaryBucket, ec2Bucket}) { + this.buckets.put(bucket.name, bucket); } this.handlers = defaultHandlers(buckets); } - private static String envVar(String varName) { - return Objects.requireNonNull(System.getenv(varName), "env variable '" + varName + "' is missing"); - } - private static String nonAuthPath(Request request) { return nonAuthPath(request.getMethod(), request.getPath()); } @@ -127,32 +115,29 @@ protected Response handle(final Request request) throws IOException { if (authorization == null) { return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad access key", ""); } - final String permittedBucket; - if (authorization.contains(permanentKey)) { - final String sessionToken = request.getHeader("x-amz-security-token"); - if (sessionToken != null) { - return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Unexpected session token", ""); - } - permittedBucket = permanentBucketName; - } else if (authorization.contains(temporaryKey)) { - final String sessionToken = request.getHeader("x-amz-security-token"); - if (sessionToken == null) { - return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "No session token", ""); - } - if (sessionToken.equals(temporarySessionToken) == false) { - return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad session token", ""); - } - permittedBucket = temporaryBucketName; - } else if (authorization.contains(ec2Key)) { - final String sessionToken = request.getHeader("x-amz-security-token"); - if (sessionToken == null) { - return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "No session token", ""); - } - if (sessionToken.equals(ec2Token) == false) { - return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad session token", ""); + final Collection values = buckets.values(); + String permittedBucket = null; + for (final Bucket bucket : values) { + if (authorization.contains(bucket.key)) { + final String sessionToken = request.getHeader("x-amz-security-token"); + if (bucket.token == null) { + if (sessionToken != null) { + return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Unexpected session token", ""); + } + } else { + if (sessionToken == null) { + return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "No session token", ""); + } + if (sessionToken.equals(bucket.token) == false) { + return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad session token", ""); + } + } + permittedBucket = bucket.name; + break; } - permittedBucket = ec2BucketName; - } else { + } + + if (permittedBucket == null) { return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad access key", ""); } @@ -393,7 +378,7 @@ private PathTrie defaultHandlers(final Map bucke // // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html handlers.insert(nonAuthPath(HttpGet.METHOD_NAME, "/latest/meta-data/iam/security-credentials/"), (request) -> { - final String response = ec2ProfileName; + final String response = EC2_PROFILE; final Map headers = new HashMap<>(contentType("text/plain")); return new Response(RestStatus.OK.getStatus(), headers, response.getBytes(UTF_8)); @@ -404,15 +389,19 @@ private PathTrie defaultHandlers(final Map bucke // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html handlers.insert(nonAuthPath(HttpGet.METHOD_NAME, "/latest/meta-data/iam/security-credentials/{profileName}"), (request) -> { final String profileName = request.getParam("profileName"); - if (ec2ProfileName.equals(profileName) == false) { + if (EC2_PROFILE.equals(profileName) == false) { return new Response(RestStatus.NOT_FOUND.getStatus(), new HashMap<>(), "unknown credentials".getBytes(UTF_8)); } - return credentialResponseFunction.apply(profileName, ec2Key, ec2Token); + return credentialResponseFunction.apply(profileName, ec2Bucket.key, ec2Bucket.token); }); return handlers; } + private static String envVar(String varName) { + return Objects.requireNonNull(System.getenv(varName), "env variable '" + varName + "' is missing"); + } + /** * Represents a S3 bucket. */ @@ -421,11 +410,23 @@ static class Bucket { /** Bucket name **/ final String name; + final String key; + + final String token; + /** Blobs contained in the bucket **/ final Map objects; - Bucket(final String name) { - this.name = Objects.requireNonNull(name); + Bucket(final String envPrefix) { + this(envPrefix, true); + } + + Bucket(final String envPrefix, final boolean tokenRequired) { + this.name = envVar(envPrefix + "_BUCKET_NAME"); + this.key = envVar(envPrefix + "_KEY"); + this.token = tokenRequired ? System.getenv(envPrefix + "_TOKEN") != null + ? envVar(envPrefix + "_TOKEN") : envVar(envPrefix + "_SESSION_TOKEN") + : null; this.objects = ConcurrentCollections.newConcurrentMap(); } } From fd285799a07596811b6ca6de376aa27c851d2cf9 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Thu, 12 Jul 2018 19:26:44 +0200 Subject: [PATCH 08/12] moved to properties file for s3Fixture --- plugins/repository-s3/build.gradle | 27 +++++++---- .../repositories/s3/AmazonS3Fixture.java | 47 +++++++++++-------- 2 files changed, 45 insertions(+), 29 deletions(-) diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 1043c04e3db18..8b76a67777d35 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -297,18 +297,25 @@ if (useFixture && minioDistribution) { /** A task to start the AmazonS3Fixture which emulates an S3 service **/ task s3Fixture(type: AntFixture) { dependsOn testClasses + + def s3FixtureOptions = [ + "s3Fixture.permanent_bucket_name": s3PermanentBucket, + "s3Fixture.permanent_key": s3PermanentAccessKey, + "s3Fixture.temporary_bucket_name": s3TemporaryBucket, + "s3Fixture.temporary_key": s3TemporaryAccessKey, + "s3Fixture.temporary_session_token": s3TemporarySessionToken, + "s3Fixture.ec2_bucket_name": s3EC2Bucket, + "s3Fixture.ec2_key": s3EC2AccessKey, + "s3Fixture.ec2_token": s3EC2Token + ] + + File parentFixtures = new File(project.buildDir, "fixtures") + File s3FixtureFile = new File(parentFixtures, 's3Fixture.properties') + s3FixtureFile.setText(s3FixtureOptions.collect{ k, v -> "$k = $v"}.join("\n"), 'UTF-8') + env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }" - env 'S3FIXTURE_WORKINGDIR', "${baseDir}" - env 'S3FIXTURE_PERMANENT_BUCKET_NAME', s3PermanentBucket - env 'S3FIXTURE_PERMANENT_KEY', s3PermanentAccessKey - env 'S3FIXTURE_TEMPORARY_BUCKET_NAME', s3TemporaryBucket - env 'S3FIXTURE_TEMPORARY_KEY', s3TemporaryAccessKey - env 'S3FIXTURE_TEMPORARY_SESSION_TOKEN', s3TemporarySessionToken - env 'S3FIXTURE_EC2_BUCKET_NAME', s3EC2Bucket - env 'S3FIXTURE_EC2_KEY', s3EC2AccessKey - env 'S3FIXTURE_EC2_TOKEN', s3EC2Token executable = new File(project.runtimeJavaHome, 'bin/java') - args 'org.elasticsearch.repositories.s3.AmazonS3Fixture', baseDir + args 'org.elasticsearch.repositories.s3.AmazonS3Fixture', baseDir, s3FixtureFile.getAbsolutePath() } Map expansions = [ diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java index 2e45401490aa6..b565fa4bca6d2 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java @@ -24,6 +24,7 @@ import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPut; import org.elasticsearch.common.TriFunction; +import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.test.fixture.AbstractHttpFixture; import com.amazonaws.util.DateUtils; import org.elasticsearch.common.Strings; @@ -36,7 +37,9 @@ import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; import java.io.InputStreamReader; +import java.nio.file.Files; import java.util.ArrayList; import java.util.Collection; import java.util.Date; @@ -44,6 +47,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Properties; import java.util.concurrent.TimeUnit; import static java.nio.charset.StandardCharsets.UTF_8; @@ -72,15 +76,12 @@ public class AmazonS3Fixture extends AbstractHttpFixture { /** * Creates a {@link AmazonS3Fixture} */ - private AmazonS3Fixture(final String workingDir) { + private AmazonS3Fixture(final String workingDir, Properties properties) { super(workingDir); - this.permanentBucket = new Bucket("S3FIXTURE_PERMANENT", false); - this.temporaryBucket = new Bucket("S3FIXTURE_TEMPORARY"); - this.ec2Bucket = new Bucket("S3FIXTURE_EC2"); + this.permanentBucket = new Bucket(properties, buckets, "s3Fixture.permanent", false); + this.temporaryBucket = new Bucket(properties, buckets, "s3Fixture.temporary"); + this.ec2Bucket = new Bucket(properties, buckets, "s3Fixture.ec2"); - for (Bucket bucket : new Bucket[]{permanentBucket, temporaryBucket, ec2Bucket}) { - this.buckets.put(bucket.name, bucket); - } this.handlers = defaultHandlers(buckets); } @@ -161,10 +162,14 @@ protected Response handle(final Request request) throws IOException { } public static void main(final String[] args) throws Exception { - if (args == null || args.length != 1) { - throw new IllegalArgumentException("AmazonS3Fixture "); + if (args == null || args.length != 2) { + throw new IllegalArgumentException("AmazonS3Fixture "); + } + final Properties properties = new Properties(); + try (InputStream is = Files.newInputStream(PathUtils.get(args[1]))) { + properties.load(is); } - final AmazonS3Fixture fixture = new AmazonS3Fixture(args[0]); + final AmazonS3Fixture fixture = new AmazonS3Fixture(args[0], properties); fixture.listen(); } @@ -398,8 +403,9 @@ private PathTrie defaultHandlers(final Map bucke return handlers; } - private static String envVar(String varName) { - return Objects.requireNonNull(System.getenv(varName), "env variable '" + varName + "' is missing"); + private static String prop(Properties properties, String propertyName) { + return Objects.requireNonNull(properties.getProperty(propertyName), + "property '" + propertyName + "' is missing"); } /** @@ -417,17 +423,20 @@ static class Bucket { /** Blobs contained in the bucket **/ final Map objects; - Bucket(final String envPrefix) { - this(envPrefix, true); + Bucket(final Properties properties, Map buckets, final String prefix) { + this(properties, buckets, prefix, true); } - Bucket(final String envPrefix, final boolean tokenRequired) { - this.name = envVar(envPrefix + "_BUCKET_NAME"); - this.key = envVar(envPrefix + "_KEY"); - this.token = tokenRequired ? System.getenv(envPrefix + "_TOKEN") != null - ? envVar(envPrefix + "_TOKEN") : envVar(envPrefix + "_SESSION_TOKEN") + Bucket(final Properties properties, Map buckets, final String prefix, final boolean tokenRequired) { + this.name = prop(properties, prefix + "_bucket_name"); + this.key = prop(properties, prefix + "_key"); + this.token = tokenRequired + ? prop(properties, prefix + (properties.getProperty(prefix + "_token") != null ? "_token" : "_session_token")) : null; this.objects = ConcurrentCollections.newConcurrentMap(); + if (buckets.put(name, this) != null) { + throw new IllegalArgumentException("bucket " + name + " is already registered"); + } } } From a4b36139fd544f5dab0cff8b731af942eb935596 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Thu, 12 Jul 2018 20:29:28 +0200 Subject: [PATCH 09/12] fix s3FixtureProperties generation --- plugins/repository-s3/build.gradle | 36 ++++++++++++++++++------------ 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 8b76a67777d35..2449b9d4003ea 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -294,24 +294,32 @@ if (useFixture && minioDistribution) { project.check.dependsOn(integTestMinio) } -/** A task to start the AmazonS3Fixture which emulates an S3 service **/ -task s3Fixture(type: AntFixture) { - dependsOn testClasses +File parentFixtures = new File(project.buildDir, "fixtures") +File s3FixtureFile = new File(parentFixtures, 's3Fixture.properties') +task s3FixtureProperties { + outputs.file(s3FixtureFile) def s3FixtureOptions = [ - "s3Fixture.permanent_bucket_name": s3PermanentBucket, - "s3Fixture.permanent_key": s3PermanentAccessKey, - "s3Fixture.temporary_bucket_name": s3TemporaryBucket, - "s3Fixture.temporary_key": s3TemporaryAccessKey, - "s3Fixture.temporary_session_token": s3TemporarySessionToken, - "s3Fixture.ec2_bucket_name": s3EC2Bucket, - "s3Fixture.ec2_key": s3EC2AccessKey, - "s3Fixture.ec2_token": s3EC2Token + "s3Fixture.permanent_bucket_name" : s3PermanentBucket, + "s3Fixture.permanent_key" : s3PermanentAccessKey, + "s3Fixture.temporary_bucket_name" : s3TemporaryBucket, + "s3Fixture.temporary_key" : s3TemporaryAccessKey, + "s3Fixture.temporary_session_token": s3TemporarySessionToken, + "s3Fixture.ec2_bucket_name" : s3EC2Bucket, + "s3Fixture.ec2_key" : s3EC2AccessKey, + "s3Fixture.ec2_token" : s3EC2Token ] - File parentFixtures = new File(project.buildDir, "fixtures") - File s3FixtureFile = new File(parentFixtures, 's3Fixture.properties') - s3FixtureFile.setText(s3FixtureOptions.collect{ k, v -> "$k = $v"}.join("\n"), 'UTF-8') + doLast { + file(s3FixtureFile).text = s3FixtureOptions.collect { k, v -> "$k = $v" }.join("\n") + } +} + +/** A task to start the AmazonS3Fixture which emulates an S3 service **/ +task s3Fixture(type: AntFixture) { + dependsOn testClasses + dependsOn s3FixtureProperties + inputs.file(s3FixtureFile) env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }" executable = new File(project.runtimeJavaHome, 'bin/java') From 9c4c3e7d905168855968c65d1403e87bfec886f1 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Fri, 13 Jul 2018 09:47:42 +0200 Subject: [PATCH 10/12] make fixture random again --- plugins/repository-s3/build.gradle | 13 ++---- .../repositories/s3/AmazonS3Fixture.java | 41 +++++++++++++++---- 2 files changed, 36 insertions(+), 18 deletions(-) diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 2449b9d4003ea..225d523817e7d 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -91,14 +91,12 @@ String s3TemporaryBasePath = System.getenv("amazon_s3_base_path_temporary") String s3EC2Bucket = System.getenv("amazon_s3_bucket_ec2") String s3EC2BasePath = System.getenv("amazon_s3_base_path_ec2") -String s3EC2AccessKey = System.getenv("amazon_s3_access_key_ec2") -String s3EC2Token = System.getenv("amazon_s3_token_ec2") // If all these variables are missing then we are testing against the internal fixture instead, which has the following // credentials hard-coded in. if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3PermanentBasePath - && !s3EC2Bucket && !s3EC2BasePath && !s3EC2AccessKey && !s3EC2Token) { + && !s3EC2Bucket && !s3EC2BasePath) { s3PermanentAccessKey = 's3_integration_test_permanent_access_key' s3PermanentSecretKey = 's3_integration_test_permanent_secret_key' s3PermanentBucket = 'permanent-bucket-test' @@ -106,13 +104,11 @@ if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3P s3EC2Bucket = 'ec2-bucket-test' s3EC2BasePath = 'integration_test' - s3EC2AccessKey = 'ec2_accessKey' - s3EC2Token = 'ec2_tkn' useFixture = true } else if (!s3PermanentAccessKey || !s3PermanentSecretKey || !s3PermanentBucket || !s3PermanentBasePath - || !s3EC2Bucket || !s3EC2BasePath || !s3EC2AccessKey || !s3EC2Token) { + || !s3EC2Bucket || !s3EC2BasePath) { throw new IllegalArgumentException("not all options specified to run against external S3 service") } @@ -300,14 +296,13 @@ File s3FixtureFile = new File(parentFixtures, 's3Fixture.properties') task s3FixtureProperties { outputs.file(s3FixtureFile) def s3FixtureOptions = [ + "tests.seed" : project.testSeed, "s3Fixture.permanent_bucket_name" : s3PermanentBucket, "s3Fixture.permanent_key" : s3PermanentAccessKey, "s3Fixture.temporary_bucket_name" : s3TemporaryBucket, "s3Fixture.temporary_key" : s3TemporaryAccessKey, "s3Fixture.temporary_session_token": s3TemporarySessionToken, - "s3Fixture.ec2_bucket_name" : s3EC2Bucket, - "s3Fixture.ec2_key" : s3EC2AccessKey, - "s3Fixture.ec2_token" : s3EC2Token + "s3Fixture.ec2_bucket_name" : s3EC2Bucket ] doLast { diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java index be1f3cbf67a37..3c3bea5ef38b6 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java @@ -34,7 +34,6 @@ import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.RestUtils; -import org.elasticsearch.test.fixture.AbstractHttpFixture; import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; @@ -48,11 +47,13 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Properties; +import java.util.Random; import java.util.concurrent.TimeUnit; +import static com.carrotsearch.randomizedtesting.generators.RandomStrings.randomAsciiAlphanumOfLength; import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Objects.requireNonNull; /** * {@link AmazonS3Fixture} emulates an AWS S3 service @@ -80,9 +81,10 @@ public class AmazonS3Fixture extends AbstractHttpFixture { */ private AmazonS3Fixture(final String workingDir, Properties properties) { super(workingDir); + Random random = new Random(Long.parseUnsignedLong(requireNonNull(properties.getProperty("tests.seed")), 16)); this.permanentBucket = new Bucket(properties, buckets, "s3Fixture.permanent", false); this.temporaryBucket = new Bucket(properties, buckets, "s3Fixture.temporary"); - this.ec2Bucket = new Bucket(properties, buckets, "s3Fixture.ec2"); + this.ec2Bucket = new Bucket(properties, buckets, "s3Fixture.ec2", random); this.handlers = defaultHandlers(buckets); } @@ -404,7 +406,7 @@ private PathTrie defaultHandlers(final Map bucke } private static String prop(Properties properties, String propertyName) { - return Objects.requireNonNull(properties.getProperty(propertyName), + return requireNonNull(properties.getProperty(propertyName), "property '" + propertyName + "' is missing"); } @@ -427,12 +429,33 @@ static class Bucket { this(properties, buckets, prefix, true); } - Bucket(final Properties properties, Map buckets, final String prefix, final boolean tokenRequired) { + Bucket(final Properties properties, + final Map buckets, + final String prefix, + final boolean tokenRequired) { + this(properties, buckets, prefix, tokenRequired, null); + } + + Bucket(final Properties properties, + final Map buckets, + final String prefix, + final Random random) { + this(properties, buckets, prefix, false, random); + } + + private Bucket(final Properties properties, + final Map buckets, + final String prefix, + final boolean tokenRequired, + final Random random) { this.name = prop(properties, prefix + "_bucket_name"); - this.key = prop(properties, prefix + "_key"); - this.token = tokenRequired - ? prop(properties, prefix + (properties.getProperty(prefix + "_token") != null ? "_token" : "_session_token")) - : null; + if (random != null) { + this.key = randomAsciiAlphanumOfLength(random, 10); + this.token = randomAsciiAlphanumOfLength(random, 10); + } else { + this.key = prop(properties, prefix + "_key"); + this.token = tokenRequired ? prop(properties, prefix + "_session_token") : null; + } this.objects = ConcurrentCollections.newConcurrentMap(); if (buckets.put(name, this) != null) { throw new IllegalArgumentException("bucket " + name + " is already registered"); From 5430473e8439500f186d3df3576602eaef284da6 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Mon, 16 Jul 2018 14:05:20 +0200 Subject: [PATCH 11/12] some code improvements --- .../repositories/s3/AmazonS3Fixture.java | 53 ++++++------------- 1 file changed, 17 insertions(+), 36 deletions(-) diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java index 3c3bea5ef38b6..6898826924d83 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java @@ -52,6 +52,7 @@ import java.util.concurrent.TimeUnit; import static com.carrotsearch.randomizedtesting.generators.RandomStrings.randomAsciiAlphanumOfLength; +import static com.carrotsearch.randomizedtesting.generators.RandomStrings.randomAsciiAlphanumOfLengthBetween; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Objects.requireNonNull; @@ -66,27 +67,28 @@ public class AmazonS3Fixture extends AbstractHttpFixture { private static final String EC2_PROFILE = "ec2Profile"; + private final Properties properties; + private final Random random; + /** List of the buckets stored on this test server **/ private final Map buckets = ConcurrentCollections.newConcurrentMap(); /** Request handlers for the requests made by the S3 client **/ private final PathTrie handlers; - private final Bucket permanentBucket; - private final Bucket temporaryBucket; - private final Bucket ec2Bucket; - /** * Creates a {@link AmazonS3Fixture} */ private AmazonS3Fixture(final String workingDir, Properties properties) { super(workingDir); - Random random = new Random(Long.parseUnsignedLong(requireNonNull(properties.getProperty("tests.seed")), 16)); - this.permanentBucket = new Bucket(properties, buckets, "s3Fixture.permanent", false); - this.temporaryBucket = new Bucket(properties, buckets, "s3Fixture.temporary"); - this.ec2Bucket = new Bucket(properties, buckets, "s3Fixture.ec2", random); + this.properties = properties; + this.random = new Random(Long.parseUnsignedLong(requireNonNull(properties.getProperty("tests.seed")), 16)); + + new Bucket("s3Fixture.permanent", false, null); + new Bucket("s3Fixture.temporary", true, null); + final Bucket ec2Bucket = new Bucket("s3Fixture.ec2", false, random); - this.handlers = defaultHandlers(buckets); + this.handlers = defaultHandlers(buckets, ec2Bucket.name); } private static String nonAuthPath(Request request) { @@ -176,7 +178,7 @@ public static void main(final String[] args) throws Exception { } /** Builds the default request handlers **/ - private PathTrie defaultHandlers(final Map buckets) { + private PathTrie defaultHandlers(final Map buckets, final String ec2BucketName) { final PathTrie handlers = new PathTrie<>(RestUtils.REST_DECODER); // HEAD Object @@ -372,8 +374,8 @@ private PathTrie defaultHandlers(final Map bucke final String response = "{" + "\"AccessKeyId\": \"" + key + "\"," + "\"Expiration\": \"" + DateUtils.formatISO8601Date(expiration) + "\"," - + "\"RoleArn\": \"" + profileName + "_ROLE" + "\"," - + "\"SecretAccessKey\": \"" + profileName + "_SCR_KEY" + "\"," + + "\"RoleArn\": \"" + randomAsciiAlphanumOfLengthBetween(random, 1, 20) + "\"," + + "\"SecretAccessKey\": \"" + randomAsciiAlphanumOfLengthBetween(random, 1, 20) + "\"," + "\"Token\": \"" + token + "\"" + "}"; @@ -391,6 +393,7 @@ private PathTrie defaultHandlers(final Map bucke return new Response(RestStatus.OK.getStatus(), headers, response.getBytes(UTF_8)); }); + final Bucket ec2Bucket = buckets.get(ec2BucketName); // GET // // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html @@ -413,7 +416,7 @@ private static String prop(Properties properties, String propertyName) { /** * Represents a S3 bucket. */ - static class Bucket { + class Bucket { /** Bucket name **/ final String name; @@ -425,29 +428,7 @@ static class Bucket { /** Blobs contained in the bucket **/ final Map objects; - Bucket(final Properties properties, Map buckets, final String prefix) { - this(properties, buckets, prefix, true); - } - - Bucket(final Properties properties, - final Map buckets, - final String prefix, - final boolean tokenRequired) { - this(properties, buckets, prefix, tokenRequired, null); - } - - Bucket(final Properties properties, - final Map buckets, - final String prefix, - final Random random) { - this(properties, buckets, prefix, false, random); - } - - private Bucket(final Properties properties, - final Map buckets, - final String prefix, - final boolean tokenRequired, - final Random random) { + private Bucket(final String prefix, final boolean tokenRequired, final Random random) { this.name = prop(properties, prefix + "_bucket_name"); if (random != null) { this.key = randomAsciiAlphanumOfLength(random, 10); From 161deeef54c20a09dc317bb27ba77d1b37b1e779 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Tue, 17 Jul 2018 16:36:36 +0200 Subject: [PATCH 12/12] (hopefully) final polish of AmazonS3Fixture --- .../repositories/s3/AmazonS3Fixture.java | 105 +++++++++--------- 1 file changed, 53 insertions(+), 52 deletions(-) diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java index 6898826924d83..ce6c472314999 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java @@ -42,7 +42,6 @@ import java.io.InputStreamReader; import java.nio.file.Files; import java.util.ArrayList; -import java.util.Collection; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -84,11 +83,12 @@ private AmazonS3Fixture(final String workingDir, Properties properties) { this.properties = properties; this.random = new Random(Long.parseUnsignedLong(requireNonNull(properties.getProperty("tests.seed")), 16)); - new Bucket("s3Fixture.permanent", false, null); - new Bucket("s3Fixture.temporary", true, null); - final Bucket ec2Bucket = new Bucket("s3Fixture.ec2", false, random); + new Bucket("s3Fixture.permanent", false); + new Bucket("s3Fixture.temporary", true); + final Bucket ec2Bucket = new Bucket("s3Fixture.ec2", + randomAsciiAlphanumOfLength(random, 10), randomAsciiAlphanumOfLength(random, 10)); - this.handlers = defaultHandlers(buckets, ec2Bucket.name); + this.handlers = defaultHandlers(buckets, ec2Bucket); } private static String nonAuthPath(Request request) { @@ -118,46 +118,19 @@ protected Response handle(final Request request) throws IOException { final String authorizedPath = authPath(request); final RequestHandler handler = handlers.retrieve(authorizedPath, request.getParameters()); if (handler != null) { - final String authorization = request.getHeader("Authorization"); - if (authorization == null) { + final String bucketName = request.getParam("bucket"); + if (bucketName == null) { return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad access key", ""); } - final Collection values = buckets.values(); - String permittedBucket = null; - for (final Bucket bucket : values) { - if (authorization.contains(bucket.key)) { - final String sessionToken = request.getHeader("x-amz-security-token"); - if (bucket.token == null) { - if (sessionToken != null) { - return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Unexpected session token", ""); - } - } else { - if (sessionToken == null) { - return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "No session token", ""); - } - if (sessionToken.equals(bucket.token) == false) { - return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad session token", ""); - } - } - permittedBucket = bucket.name; - break; - } + final Bucket bucket = buckets.get(bucketName); + if (bucket == null) { + return newBucketNotFoundError(request.getId(), bucketName); } - - if (permittedBucket == null) { - return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad access key", ""); + final Response authResponse = authenticateBucket(request, bucket); + if (authResponse != null) { + return authResponse; } - final String bucket = request.getParam("bucket"); - if (bucket != null && permittedBucket.equals(bucket) == false) { - // allow a null bucket to support the multi-object-delete API which - // passes the bucket name in the host header instead of the URL. - if (buckets.containsKey(bucket)) { - return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad bucket", ""); - } else { - return newBucketNotFoundError(request.getId(), bucket); - } - } return handler.handle(request); } else { @@ -165,6 +138,29 @@ protected Response handle(final Request request) throws IOException { } } + private Response authenticateBucket(Request request, Bucket bucket) { + final String authorization = request.getHeader("Authorization"); + if (authorization == null) { + return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad access key", ""); + } + if (authorization.contains(bucket.key)) { + final String sessionToken = request.getHeader("x-amz-security-token"); + if (bucket.token == null) { + if (sessionToken != null) { + return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Unexpected session token", ""); + } + } else { + if (sessionToken == null) { + return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "No session token", ""); + } + if (sessionToken.equals(bucket.token) == false) { + return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad session token", ""); + } + } + } + return null; + } + public static void main(final String[] args) throws Exception { if (args == null || args.length != 2) { throw new IllegalArgumentException("AmazonS3Fixture "); @@ -178,7 +174,7 @@ public static void main(final String[] args) throws Exception { } /** Builds the default request handlers **/ - private PathTrie defaultHandlers(final Map buckets, final String ec2BucketName) { + private PathTrie defaultHandlers(final Map buckets, final Bucket ec2Bucket) { final PathTrie handlers = new PathTrie<>(RestUtils.REST_DECODER); // HEAD Object @@ -325,7 +321,7 @@ private PathTrie defaultHandlers(final Map bucke // Delete Multiple Objects // // https://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html - handlers.insert(authPath(HttpPost.METHOD_NAME, "/"), (request) -> { + handlers.insert(nonAuthPath(HttpPost.METHOD_NAME, "/"), (request) -> { final List deletes = new ArrayList<>(); final List errors = new ArrayList<>(); @@ -348,7 +344,12 @@ private PathTrie defaultHandlers(final Map bucke boolean found = false; for (Bucket bucket : buckets.values()) { - if (bucket.objects.remove(objectName) != null) { + if (bucket.objects.containsKey(objectName)) { + final Response authResponse = authenticateBucket(request, bucket); + if (authResponse != null) { + return authResponse; + } + bucket.objects.remove(objectName); found = true; } } @@ -393,7 +394,6 @@ private PathTrie defaultHandlers(final Map bucke return new Response(RestStatus.OK.getStatus(), headers, response.getBytes(UTF_8)); }); - final Bucket ec2Bucket = buckets.get(ec2BucketName); // GET // // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html @@ -428,15 +428,16 @@ class Bucket { /** Blobs contained in the bucket **/ final Map objects; - private Bucket(final String prefix, final boolean tokenRequired, final Random random) { + private Bucket(final String prefix, final boolean tokenRequired) { + this(prefix, prop(properties, prefix + "_key"), + tokenRequired ? prop(properties, prefix + "_session_token") : null); + } + + private Bucket(final String prefix, final String key, final String token) { this.name = prop(properties, prefix + "_bucket_name"); - if (random != null) { - this.key = randomAsciiAlphanumOfLength(random, 10); - this.token = randomAsciiAlphanumOfLength(random, 10); - } else { - this.key = prop(properties, prefix + "_key"); - this.token = tokenRequired ? prop(properties, prefix + "_session_token") : null; - } + this.key = key; + this.token = token; + this.objects = ConcurrentCollections.newConcurrentMap(); if (buckets.put(name, this) != null) { throw new IllegalArgumentException("bucket " + name + " is already registered");