[GCE Discovery] Automatically set project-id and zone (#33721)

Fetch default values for project-id and zone from metadata server

Closes #13618

docs/plugins/discovery-gce.asciidoc

@@ -26,12 +26,17 @@ The following gce settings (prefixed with `cloud.gce`) are supported:
 
  `project_id`::
 
-     Your Google project id (mandatory).
+     Your Google project id.
+     By default the project id will be derived from the instance metadata.
+
+     Note: Deriving the project id from system properties or environment variables
+     (`GOOGLE_CLOUD_PROJECT` or `GCLOUD_PROJECT`) is not supported.
 
  `zone`::
 
-     helps to retrieve instances running in a given zone (mandatory). It should be one of the
-     https://developers.google.com/compute/docs/zones#available[GCE supported zones].
+     helps to retrieve instances running in a given zone.
+     It should be one of the https://developers.google.com/compute/docs/zones#available[GCE supported zones].
+     By default the zone will be derived from the instance metadata.
      See also <<discovery-gce-usage-zones>>.
 
  `retry`::

plugins/discovery-gce/build.gradle

@@ -26,6 +26,11 @@ dependencyLicenses {
   mapping from: /google-.*/, to: 'google'
 }
 
+check {
+  // also execute the QA tests when testing the plugin
+  dependsOn 'qa:gce:check'
+}
+
 test {
   // this is needed for insecure plugins, remove if possible!
   systemProperty 'tests.artifact', project.name 

plugins/discovery-gce/qa/build.gradle

@@ -0,0 +1 @@
+group = "${group}.plugins.discovery-gce.qa"

plugins/discovery-gce/qa/gce/build.gradle

@@ -0,0 +1,80 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.elasticsearch.gradle.MavenFilteringHack
+import org.elasticsearch.gradle.test.AntFixture
+
+apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.rest-test'
+
+final int gceNumberOfNodes = 3
+File gceDiscoveryFile =  new File(project.buildDir, 'generated-resources/nodes.uri')
+
+dependencies {
+    testCompile project(path: ':plugins:discovery-gce', configuration: 'runtime')
+}
+
+/** A task to start the GCEFixture which emulates a GCE service **/
+task gceFixture(type: AntFixture) {
+    dependsOn compileTestJava
+    env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }"
+    executable = new File(project.runtimeJavaHome, 'bin/java')
+    args 'org.elasticsearch.cloud.gce.GCEFixture', baseDir, gceDiscoveryFile.getAbsolutePath()
+}
+
+Map<String, Object> expansions = [
+        'expected_nodes': gceNumberOfNodes
+]
+
+processTestResources {
+    inputs.properties(expansions)
+    MavenFilteringHack.filter(it, expansions)
+}
+
+integTestCluster {
+    dependsOn gceFixture
+    numNodes = gceNumberOfNodes
+    plugin ':plugins:discovery-gce'
+    setting 'discovery.zen.hosts_provider', 'gce'
+
+    // use gce fixture for Auth calls instead of http://metadata.google.internal
+    integTestCluster.environment 'GCE_METADATA_HOST', "http://${-> gceFixture.addressAndPort}"
+
+    // allows to configure hidden settings (`cloud.gce.host` and `cloud.gce.root_url`)
+    systemProperty 'es.allow_reroute_gce_settings', 'true'
+
+    // use gce fixture for metadata server calls instead of http://metadata.google.internal
+    setting 'cloud.gce.host', "http://${-> gceFixture.addressAndPort}"
+    // use gce fixture for API calls instead of https://www.googleapis.com
+    setting 'cloud.gce.root_url', "http://${-> gceFixture.addressAndPort}"
+
+    unicastTransportUri = { seedNode, node, ant -> return null }
+
+    waitCondition = { node, ant ->
+        gceDiscoveryFile.parentFile.mkdirs()
+        gceDiscoveryFile.setText(integTest.nodes.collect { n -> "${n.transportUri()}" }.join('\n'), 'UTF-8')
+
+        File tmpFile = new File(node.cwd, 'wait.success')
+        ant.get(src: "http://${node.httpUri()}/",
+                dest: tmpFile.toString(),
+                ignoreerrors: true,
+                retries: 10)
+        return tmpFile.exists()
+    }
+}

plugins/discovery-gce/qa/gce/src/test/java/org/elasticsearch/cloud/gce/GCEDiscoveryClientYamlTestSuiteIT.java

@@ -0,0 +1,37 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cloud.gce;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+
+public class GCEDiscoveryClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
+
+    public GCEDiscoveryClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
+    }
+}

plugins/discovery-gce/qa/gce/src/test/java/org/elasticsearch/cloud/gce/GCEFixture.java

@@ -0,0 +1,214 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.cloud.gce;
+
+import org.apache.http.client.methods.HttpGet;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.collect.MapBuilder;
+import org.elasticsearch.common.path.PathTrie;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.rest.RestUtils;
+import org.elasticsearch.test.fixture.AbstractHttpFixture;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+import java.util.function.Function;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+
+/**
+ * {@link GCEFixture} is a fixture that emulates a GCE service.
+ */
+public class GCEFixture extends AbstractHttpFixture {
+
+    public static final String PROJECT_ID = "discovery-gce-test";
+    public static final String ZONE = "test-zone";
+    public static final String TOKEN = "1/fFAGRNJru1FTz70BzhT3Zg";
+    public static final String TOKEN_TYPE = "Bearer";
+
+    private final PathTrie<RequestHandler> handlers;
+
+    private final Path nodes;
+
+    private GCEFixture(final String workingDir, final String nodesUriPath) {
+        super(workingDir);
+        this.nodes = toPath(Objects.requireNonNull(nodesUriPath));
+        this.handlers = defaultHandlers();
+    }
+
+    public static void main(String[] args) throws Exception {
+        if (args == null || args.length != 2) {
+            throw new IllegalArgumentException("GCEFixture <working directory> <nodes transport uri file>");
+        }
+
+        final GCEFixture fixture = new GCEFixture(args[0], args[1]);
+        fixture.listen();
+    }
+
+    private static String nonAuthPath(Request request) {
+        return nonAuthPath(request.getMethod(), request.getPath());
+    }
+
+    private static String nonAuthPath(String method, String path) {
+        return "NONAUTH " + method + " " + path;
+    }
+
+    private static String authPath(Request request) {
+        return authPath(request.getMethod(), request.getPath());
+    }
+
+    private static String authPath(String method, String path) {
+        return "AUTH " + method + " " + path;
+    }
+
+    /** Builds the default request handlers **/
+    private PathTrie<RequestHandler> defaultHandlers() {
+        final PathTrie<RequestHandler> handlers = new PathTrie<>(RestUtils.REST_DECODER);
+
+        final Consumer<Map<String, String>> commonHeaderConsumer = headers -> headers.put("Metadata-Flavor", "Google");
+
+        final Function<String, Response> simpleValue = value -> {
+            final Map<String, String> headers = new HashMap<>(TEXT_PLAIN_CONTENT_TYPE);
+            commonHeaderConsumer.accept(headers);
+
+            final byte[] responseAsBytes = value.getBytes(StandardCharsets.UTF_8);
+            return new Response(RestStatus.OK.getStatus(), headers, responseAsBytes);
+        };
+
+        final Function<String, Response> jsonValue = value -> {
+            final Map<String, String> headers = new HashMap<>(JSON_CONTENT_TYPE);
+            commonHeaderConsumer.accept(headers);
+
+            final byte[] responseAsBytes = value.getBytes(StandardCharsets.UTF_8);
+            return new Response(RestStatus.OK.getStatus(), headers, responseAsBytes);
+        };
+
+        // https://cloud.google.com/compute/docs/storing-retrieving-metadata
+        handlers.insert(nonAuthPath(HttpGet.METHOD_NAME, "/computeMetadata/v1/project/project-id"),
+            request -> simpleValue.apply(PROJECT_ID));
+        handlers.insert(nonAuthPath(HttpGet.METHOD_NAME, "/computeMetadata/v1/project/attributes/google-compute-default-zone"),
+            request -> simpleValue.apply(ZONE));
+        // https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances
+        handlers.insert(nonAuthPath(HttpGet.METHOD_NAME, "/computeMetadata/v1/instance/service-accounts/default/token"),
+            request -> jsonValue.apply(Strings.toString(jsonBuilder()
+                .startObject()
+                    .field("access_token", TOKEN)
+                    .field("expires_in", TimeUnit.HOURS.toSeconds(1))
+                    .field("token_type", TOKEN_TYPE)
+                .endObject())));
+
+        // https://cloud.google.com/compute/docs/reference/rest/v1/instances
+        handlers.insert(authPath(HttpGet.METHOD_NAME, "/compute/v1/projects/{project}/zones/{zone}/instances"),
+            request -> {
+                final List items = new ArrayList();
+                int count = 0;
+                for (String address : Files.readAllLines(nodes)) {
+                    count++;
+                    items.add(MapBuilder.<String, Object>newMapBuilder()
+                        .put("id", Long.toString(9309873766405L + count))
+                        .put("description", "ES node" + count)
+                        .put("name", "test" + count)
+                        .put("kind", "compute#instance")
+                        .put("machineType", "n1-standard-1")
+                        .put("networkInterfaces",
+                            Collections.singletonList(MapBuilder.<String, Object>newMapBuilder()
+                            .put("accessConfigs", Collections.emptyList())
+                            .put("name", "nic0")
+                            .put("network", "default")
+                            .put("networkIP", address)
+                            .immutableMap()))
+                        .put("status", "RUNNING")
+                        .put("zone", ZONE)
+                        .immutableMap());
+                }
+
+                final String json = Strings.toString(jsonBuilder()
+                    .startObject()
+                    .field("id", "test-instances")
+                    .field("items", items)
+                    .endObject());
+
+                final byte[] responseAsBytes = json.getBytes(StandardCharsets.UTF_8);
+                final Map<String, String> headers = new HashMap<>(JSON_CONTENT_TYPE);
+                commonHeaderConsumer.accept(headers);
+                return new Response(RestStatus.OK.getStatus(), headers, responseAsBytes);
+        });
+        return handlers;
+    }
+
+    @Override
+    protected Response handle(final Request request) throws IOException {
+        final String nonAuthorizedPath = nonAuthPath(request);
+        final RequestHandler nonAuthorizedHandler = handlers.retrieve(nonAuthorizedPath, request.getParameters());
+        if (nonAuthorizedHandler != null) {
+            return nonAuthorizedHandler.handle(request);
+        }
+
+        final String authorizedPath = authPath(request);
+        final RequestHandler authorizedHandler = handlers.retrieve(authorizedPath, request.getParameters());
+        if (authorizedHandler != null) {
+            final String authorization = request.getHeader("Authorization");
+            if ((TOKEN_TYPE + " " + TOKEN).equals(authorization) == false) {
+                return newError(RestStatus.UNAUTHORIZED, "Authorization", "Login Required");
+            }
+            return authorizedHandler.handle(request);
+        }
+
+        return null;
+    }
+
+    private static Response newError(final RestStatus status, final String code, final String message) throws IOException {
+        final String response = Strings.toString(jsonBuilder()
+            .startObject()
+            .field("error", MapBuilder.<String, Object>newMapBuilder()
+                .put("errors", Collections.singletonList(
+                    MapBuilder.<String, Object>newMapBuilder()
+                    .put("domain", "global")
+                    .put("reason", "required")
+                    .put("message", message)
+                    .put("locationType", "header")
+                    .put("location", code)
+                    .immutableMap()
+                ))
+                .put("code", status.getStatus())
+                .put("message", message)
+                .immutableMap())
+            .endObject());
+
+        return new Response(status.getStatus(), JSON_CONTENT_TYPE, response.getBytes(UTF_8));
+    }
+
+    @SuppressForbidden(reason = "Paths#get is fine - we don't have environment here")
+    private static Path toPath(final String dir) {
+        return Paths.get(dir);
+    }
+}

plugins/discovery-gce/qa/gce/src/test/resources/rest-api-spec/test/discovery_gce/10_basic.yml

@@ -0,0 +1,15 @@
+# Integration tests for discovery-gce
+setup:
+ - do:
+     cluster.health:
+        wait_for_status: green
+        wait_for_nodes: ${expected_nodes}
+
+---
+"All nodes are correctly discovered":
+
+  - do:
+      nodes.info:
+        metric: [ transport ]
+
+  - match: { _nodes.total: ${expected_nodes} }

plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceInstancesService.java

@@ -75,4 +75,8 @@ public interface GceInstancesService extends Closeable {
      * @return a collection of running instances within the same GCE project
      */
     Collection<Instance> instances();
+
+    String projectId();
+
+    List<String> zones();
 }