Add EC2 credential test for repository-s3

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy

@@ -137,7 +137,7 @@ class ClusterConfiguration {
         this.project = project
     }
 
-    Map<String, String> systemProperties = new HashMap<>()
+    Map<String, Object> systemProperties = new HashMap<>()
 
     Map<String, Object> settings = new HashMap<>()
 
@@ -157,7 +157,7 @@ class ClusterConfiguration {
     List<Object> dependencies = new ArrayList<>()
 
     @Input
-    void systemProperty(String property, String value) {
+    void systemProperty(String property, Object value) {
         systemProperties.put(property, value)
     }
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -202,6 +202,7 @@ class ClusterFormationTasks {
         setup = configureCreateKeystoreTask(taskName(prefix, node, 'createKeystore'), project, setup, node)
         setup = configureAddKeystoreSettingTasks(prefix, project, setup, node)
         setup = configureAddKeystoreFileTasks(prefix, project, setup, node)
+        setup = configureESJavaOpts(prefix, project, setup, node)
 
         if (node.config.plugins.isEmpty() == false) {
             if (node.nodeVersion == VersionProperties.elasticsearch) {
@@ -412,6 +413,31 @@ class ClusterFormationTasks {
         return parentTask
     }
 
+    /** Configure ES JAVA OPTS - adds system properties, assertion flags, remote debug etc */
+    static Task configureESJavaOpts(String parent, Project project, Task setup, NodeInfo node) {
+        return project.tasks.create(name: taskName(parent, node, 'configureESJavaOpts'), type: DefaultTask, dependsOn: setup) {
+            doLast {
+                String collectedSystemProperties = node.config.systemProperties.collect { key, value -> "-D${key}=${value}" }.join(" ")
+                List<String> esJavaOpts = [collectedSystemProperties]
+                esJavaOpts.add(node.config.jvmArgs)
+                if (Boolean.parseBoolean(System.getProperty('tests.asserts', 'true'))) {
+                    // put the enable assertions options before other options to allow
+                    // flexibility to disable assertions for specific packages or classes
+                    // in the cluster-specific options
+                    esJavaOpts.add("-ea")
+                    esJavaOpts.add("-esa")
+                }
+                // we must add debug options inside the closure so the config is read at execution time, as
+                // gradle task options are not processed until the end of the configuration phase
+                if (node.config.debug) {
+                    println 'Running elasticsearch in debug mode, suspending until connected on port 8000'
+                    esJavaOpts.add('-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=8000')
+                }
+                node.env['ES_JAVA_OPTS'] = esJavaOpts.join(" ")
+            }
+        }
+    }
+
     static Task configureExtraConfigFilesTask(String name, Project project, Task setup, NodeInfo node) {
         if (node.config.extraConfigFiles.isEmpty()) {
             return setup
@@ -624,13 +650,6 @@ class ClusterFormationTasks {
                 node.writeWrapperScript()
             }
 
-            // we must add debug options inside the closure so the config is read at execution time, as
-            // gradle task options are not processed until the end of the configuration phase
-            if (node.config.debug) {
-                println 'Running elasticsearch in debug mode, suspending until connected on port 8000'
-                node.env['ES_JAVA_OPTS'] = '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=8000'
-            }
-
             node.getCommandString().eachLine { line -> logger.info(line) }
 
             if (logger.isInfoEnabled() || node.config.daemonize == false) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy

@@ -180,15 +180,7 @@ class NodeInfo {
         }
 
         args.addAll("-E", "node.portsfile=true")
-        String collectedSystemProperties = config.systemProperties.collect { key, value -> "-D${key}=${value}" }.join(" ")
-        String esJavaOpts = config.jvmArgs.isEmpty() ? collectedSystemProperties : collectedSystemProperties + " " + config.jvmArgs
-        if (Boolean.parseBoolean(System.getProperty('tests.asserts', 'true'))) {
-            // put the enable assertions options before other options to allow
-            // flexibility to disable assertions for specific packages or classes
-            // in the cluster-specific options
-            esJavaOpts = String.join(" ", "-ea", "-esa", esJavaOpts)
-        }
-        env = ['ES_JAVA_OPTS': esJavaOpts]
+        env = [:]
         for (Map.Entry<String, String> property : System.properties.entrySet()) {
             if (property.key.startsWith('tests.es.')) {
                 args.add("-E")

plugins/repository-s3/build.gradle

@@ -288,7 +288,9 @@ Map<String, Object> expansions = [
   'permanent_bucket': s3PermanentBucket,
   'permanent_base_path': s3PermanentBasePath,
   'temporary_bucket': s3TemporaryBucket,
-  'temporary_base_path': s3TemporaryBasePath
+  'temporary_base_path': s3TemporaryBasePath,
+  'ec2_bucket': s3TemporaryBucket,
+  'ec2_base_path': s3TemporaryBasePath
 ]
 
 processTestResources {
@@ -309,6 +311,10 @@ integTestCluster {
     /* Use a closure on the string to delay evaluation until tests are executed */
     setting 's3.client.integration_test_permanent.endpoint', "http://${-> s3Fixture.addressAndPort}"
     setting 's3.client.integration_test_temporary.endpoint', "http://${-> s3Fixture.addressAndPort}"
+    setting 's3.client.integration_test_ec2.endpoint', "http://${-> s3Fixture.addressAndPort}"
+
+    // to redirect InstanceProfileCredentialsProvider to custom auth point
+    systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", "http://${-> s3Fixture.addressAndPort}"
   } else {
     println "Using an external service to test the repository-s3 plugin"
   }

plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java

@@ -37,6 +37,8 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
 
@@ -70,17 +72,24 @@ private AmazonS3Fixture(final String workingDir, final String permanentBucketNam
 
     @Override
     protected Response handle(final Request request) throws IOException {
-        final RequestHandler handler = handlers.retrieve(request.getMethod() + " " + request.getPath(), request.getParameters());
+        final String nonAuthorizedPath = "* " + request.getMethod() + " " + request.getPath();
+        final RequestHandler nonAuthorizedHandler = handlers.retrieve(nonAuthorizedPath, request.getParameters());
+        if (nonAuthorizedHandler != null) {
+            return nonAuthorizedHandler.handle(request);
+        }
+
+        final String authorizedPath = "A " + request.getMethod() + " " + request.getPath();
+        final RequestHandler handler = handlers.retrieve(authorizedPath, request.getParameters());
         if (handler != null) {
             final String authorization = request.getHeader("Authorization");
             final String permittedBucket;
-            if (authorization.contains("s3_integration_test_permanent_access_key")) {
+            if (authorization != null && authorization.contains("s3_integration_test_permanent_access_key")) {
                 final String sessionToken = request.getHeader("x-amz-security-token");
                 if (sessionToken != null) {
                     return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Unexpected session token", "");
                 }
                 permittedBucket = permanentBucketName;
-            } else if (authorization.contains("s3_integration_test_temporary_access_key")) {
+            } else if (authorization != null && authorization.contains("s3_integration_test_temporary_access_key")) {
                 final String sessionToken = request.getHeader("x-amz-security-token");
                 if (sessionToken == null) {
                     return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "No session token", "");
@@ -89,6 +98,15 @@ protected Response handle(final Request request) throws IOException {
                     return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad session token", "");
                 }
                 permittedBucket = temporaryBucketName;
+            } else if (authorization != null && authorization.contains("securitycredentials42_KEYID")) {
+                final String sessionToken = request.getHeader("x-amz-security-token");
+                if (sessionToken == null) {
+                    return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "No session token", "");
+                }
+                if (sessionToken.equals("securitycredentials42_TKN") == false) {
+                    return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad session token", "");
+                }
+                permittedBucket = temporaryBucketName;
             } else {
                 return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad access key", "");
             }
@@ -129,7 +147,7 @@ private static PathTrie<RequestHandler> defaultHandlers(final Map<String, Bucket
         // HEAD Object
         //
         // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectHEAD.html
-        objectsPaths("HEAD /{bucket}").forEach(path ->
+        objectsPaths("A HEAD /{bucket}").forEach(path ->
             handlers.insert(path, (request) -> {
                 final String bucketName = request.getParam("bucket");
 
@@ -151,7 +169,7 @@ private static PathTrie<RequestHandler> defaultHandlers(final Map<String, Bucket
         // PUT Object
         //
         // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html
-        objectsPaths("PUT /{bucket}").forEach(path ->
+        objectsPaths("A PUT /{bucket}").forEach(path ->
             handlers.insert(path, (request) -> {
                 final String destBucketName = request.getParam("bucket");
 
@@ -201,7 +219,7 @@ private static PathTrie<RequestHandler> defaultHandlers(final Map<String, Bucket
         // DELETE Object
         //
         // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectDELETE.html
-        objectsPaths("DELETE /{bucket}").forEach(path ->
+        objectsPaths("A DELETE /{bucket}").forEach(path ->
             handlers.insert(path, (request) -> {
                 final String bucketName = request.getParam("bucket");
 
@@ -219,7 +237,7 @@ private static PathTrie<RequestHandler> defaultHandlers(final Map<String, Bucket
         // GET Object
         //
         // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html
-        objectsPaths("GET /{bucket}").forEach(path ->
+        objectsPaths("A GET /{bucket}").forEach(path ->
             handlers.insert(path, (request) -> {
                 final String bucketName = request.getParam("bucket");
 
@@ -240,7 +258,7 @@ private static PathTrie<RequestHandler> defaultHandlers(final Map<String, Bucket
         // HEAD Bucket
         //
         // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketHEAD.html
-        handlers.insert("HEAD /{bucket}", (request) -> {
+        handlers.insert("A HEAD /{bucket}", (request) -> {
             String bucket = request.getParam("bucket");
             if (Strings.hasText(bucket) && buckets.containsKey(bucket)) {
                 return new Response(RestStatus.OK.getStatus(), TEXT_PLAIN_CONTENT_TYPE, EMPTY_BYTE);
@@ -252,7 +270,7 @@ private static PathTrie<RequestHandler> defaultHandlers(final Map<String, Bucket
         // GET Bucket (List Objects) Version 1
         //
         // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGET.html
-        handlers.insert("GET /{bucket}/", (request) -> {
+        handlers.insert("A GET /{bucket}/", (request) -> {
             final String bucketName = request.getParam("bucket");
 
             final Bucket bucket = buckets.get(bucketName);
@@ -270,7 +288,7 @@ private static PathTrie<RequestHandler> defaultHandlers(final Map<String, Bucket
         // Delete Multiple Objects
         //
         // https://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
-        handlers.insert("POST /", (request) -> {
+        handlers.insert("A POST /", (request) -> {
             final List<String> deletes = new ArrayList<>();
             final List<String> errors = new ArrayList<>();
 
@@ -312,6 +330,40 @@ private static PathTrie<RequestHandler> defaultHandlers(final Map<String, Bucket
             return newInternalError(request.getId(), "Something is wrong with this POST multiple deletes request");
         });
 
+        // non-authorized requests
+
+        Function<String, Response> credentialResponseFunction = prefix -> {
+            final Date expiration = new Date(new Date().getTime() + TimeUnit.DAYS.toMillis(1));
+            final String response = "{"
+                 + "\"AccessKeyId\": \"" + prefix + "_KEYID" + "\","
+                 + "\"Expiration\": \"" + DateUtils.formatISO8601Date(expiration) + "\","
+                 + "\"RoleArn\": \"" + prefix + "_ROLE" + "\","
+                 + "\"SecretAccessKey\": \"" + prefix + "_SCR_KEY" + "\","
+                 + "\"Token\": \"" + prefix + "_TKN" + "\""
+                 + "}";
+
+            final Map<String, String> headers = new HashMap<>(contentType("application/json"));
+            return new Response(RestStatus.OK.getStatus(), headers, response.getBytes(UTF_8));
+        };
+
+        // GET
+        //
+        // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html
+        handlers.insert("* GET /latest/meta-data/iam/security-credentials/", (request) -> {
+            final String response = "securitycredentials42";
+
+            final Map<String, String> headers = new HashMap<>(contentType("text/plain"));
+            return new Response(RestStatus.OK.getStatus(), headers, response.getBytes(UTF_8));
+        });
+
+        // GET
+        //
+        // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html
+        handlers.insert("* GET /latest/meta-data/iam/security-credentials/{credentials}", (request) -> {
+            final String credentials = request.getParam("credentials");
+            return credentialResponseFunction.apply(credentials);
+        });
+
         return handlers;
     }