diff --git a/docs/plugins/discovery-gce.asciidoc b/docs/plugins/discovery-gce.asciidoc index 33369eaba3c10..633b136bc7186 100644 --- a/docs/plugins/discovery-gce.asciidoc +++ b/docs/plugins/discovery-gce.asciidoc @@ -26,12 +26,17 @@ The following gce settings (prefixed with `cloud.gce`) are supported: `project_id`:: - Your Google project id (mandatory). + Your Google project id. + By default the project id will be derived from the instance metadata. + + Note: Deriving the project id from system properties or environment variables + (`GOOGLE_CLOUD_PROJECT` or `GCLOUD_PROJECT`) is not supported. `zone`:: - helps to retrieve instances running in a given zone (mandatory). It should be one of the - https://developers.google.com/compute/docs/zones#available[GCE supported zones]. + helps to retrieve instances running in a given zone. + It should be one of the https://developers.google.com/compute/docs/zones#available[GCE supported zones]. + By default the zone will be derived from the instance metadata. See also <>. `retry`:: diff --git a/plugins/discovery-gce/build.gradle b/plugins/discovery-gce/build.gradle index fa8005dfa4759..bccb496859ac8 100644 --- a/plugins/discovery-gce/build.gradle +++ b/plugins/discovery-gce/build.gradle @@ -26,6 +26,11 @@ dependencyLicenses { mapping from: /google-.*/, to: 'google' } +check { + // also execute the QA tests when testing the plugin + dependsOn 'qa:gce:check' +} + test { // this is needed for insecure plugins, remove if possible! systemProperty 'tests.artifact', project.name diff --git a/plugins/discovery-gce/qa/build.gradle b/plugins/discovery-gce/qa/build.gradle new file mode 100644 index 0000000000000..f6a418ae4bd3c --- /dev/null +++ b/plugins/discovery-gce/qa/build.gradle @@ -0,0 +1 @@ +group = "${group}.plugins.discovery-gce.qa" diff --git a/plugins/discovery-gce/qa/gce/build.gradle b/plugins/discovery-gce/qa/gce/build.gradle new file mode 100644 index 0000000000000..9b496207c5619 --- /dev/null +++ b/plugins/discovery-gce/qa/gce/build.gradle @@ -0,0 +1,80 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.elasticsearch.gradle.MavenFilteringHack +import org.elasticsearch.gradle.test.AntFixture + +apply plugin: 'elasticsearch.standalone-rest-test' +apply plugin: 'elasticsearch.rest-test' + +final int gceNumberOfNodes = 3 +File gceDiscoveryFile = new File(project.buildDir, 'generated-resources/nodes.uri') + +dependencies { + testCompile project(path: ':plugins:discovery-gce', configuration: 'runtime') +} + +/** A task to start the GCEFixture which emulates a GCE service **/ +task gceFixture(type: AntFixture) { + dependsOn compileTestJava + env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }" + executable = new File(project.runtimeJavaHome, 'bin/java') + args 'org.elasticsearch.cloud.gce.GCEFixture', baseDir, gceDiscoveryFile.getAbsolutePath() +} + +Map expansions = [ + 'expected_nodes': gceNumberOfNodes +] + +processTestResources { + inputs.properties(expansions) + MavenFilteringHack.filter(it, expansions) +} + +integTestCluster { + dependsOn gceFixture + numNodes = gceNumberOfNodes + plugin ':plugins:discovery-gce' + setting 'discovery.zen.hosts_provider', 'gce' + + // use gce fixture for Auth calls instead of http://metadata.google.internal + integTestCluster.environment 'GCE_METADATA_HOST', "http://${-> gceFixture.addressAndPort}" + + // allows to configure hidden settings (`cloud.gce.host` and `cloud.gce.root_url`) + systemProperty 'es.allow_reroute_gce_settings', 'true' + + // use gce fixture for metadata server calls instead of http://metadata.google.internal + setting 'cloud.gce.host', "http://${-> gceFixture.addressAndPort}" + // use gce fixture for API calls instead of https://www.googleapis.com + setting 'cloud.gce.root_url', "http://${-> gceFixture.addressAndPort}" + + unicastTransportUri = { seedNode, node, ant -> return null } + + waitCondition = { node, ant -> + gceDiscoveryFile.parentFile.mkdirs() + gceDiscoveryFile.setText(integTest.nodes.collect { n -> "${n.transportUri()}" }.join('\n'), 'UTF-8') + + File tmpFile = new File(node.cwd, 'wait.success') + ant.get(src: "http://${node.httpUri()}/", + dest: tmpFile.toString(), + ignoreerrors: true, + retries: 10) + return tmpFile.exists() + } +} diff --git a/plugins/discovery-gce/qa/gce/src/test/java/org/elasticsearch/cloud/gce/GCEDiscoveryClientYamlTestSuiteIT.java b/plugins/discovery-gce/qa/gce/src/test/java/org/elasticsearch/cloud/gce/GCEDiscoveryClientYamlTestSuiteIT.java new file mode 100644 index 0000000000000..6c1ca9c72d3f8 --- /dev/null +++ b/plugins/discovery-gce/qa/gce/src/test/java/org/elasticsearch/cloud/gce/GCEDiscoveryClientYamlTestSuiteIT.java @@ -0,0 +1,37 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cloud.gce; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; + +public class GCEDiscoveryClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + + public GCEDiscoveryClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return ESClientYamlSuiteTestCase.createParameters(); + } +} diff --git a/plugins/discovery-gce/qa/gce/src/test/java/org/elasticsearch/cloud/gce/GCEFixture.java b/plugins/discovery-gce/qa/gce/src/test/java/org/elasticsearch/cloud/gce/GCEFixture.java new file mode 100644 index 0000000000000..f52e613ee29c0 --- /dev/null +++ b/plugins/discovery-gce/qa/gce/src/test/java/org/elasticsearch/cloud/gce/GCEFixture.java @@ -0,0 +1,214 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.cloud.gce; + +import org.apache.http.client.methods.HttpGet; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.collect.MapBuilder; +import org.elasticsearch.common.path.PathTrie; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.rest.RestUtils; +import org.elasticsearch.test.fixture.AbstractHttpFixture; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import java.util.function.Function; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; + +/** + * {@link GCEFixture} is a fixture that emulates a GCE service. + */ +public class GCEFixture extends AbstractHttpFixture { + + public static final String PROJECT_ID = "discovery-gce-test"; + public static final String ZONE = "test-zone"; + public static final String TOKEN = "1/fFAGRNJru1FTz70BzhT3Zg"; + public static final String TOKEN_TYPE = "Bearer"; + + private final PathTrie handlers; + + private final Path nodes; + + private GCEFixture(final String workingDir, final String nodesUriPath) { + super(workingDir); + this.nodes = toPath(Objects.requireNonNull(nodesUriPath)); + this.handlers = defaultHandlers(); + } + + public static void main(String[] args) throws Exception { + if (args == null || args.length != 2) { + throw new IllegalArgumentException("GCEFixture "); + } + + final GCEFixture fixture = new GCEFixture(args[0], args[1]); + fixture.listen(); + } + + private static String nonAuthPath(Request request) { + return nonAuthPath(request.getMethod(), request.getPath()); + } + + private static String nonAuthPath(String method, String path) { + return "NONAUTH " + method + " " + path; + } + + private static String authPath(Request request) { + return authPath(request.getMethod(), request.getPath()); + } + + private static String authPath(String method, String path) { + return "AUTH " + method + " " + path; + } + + /** Builds the default request handlers **/ + private PathTrie defaultHandlers() { + final PathTrie handlers = new PathTrie<>(RestUtils.REST_DECODER); + + final Consumer> commonHeaderConsumer = headers -> headers.put("Metadata-Flavor", "Google"); + + final Function simpleValue = value -> { + final Map headers = new HashMap<>(TEXT_PLAIN_CONTENT_TYPE); + commonHeaderConsumer.accept(headers); + + final byte[] responseAsBytes = value.getBytes(StandardCharsets.UTF_8); + return new Response(RestStatus.OK.getStatus(), headers, responseAsBytes); + }; + + final Function jsonValue = value -> { + final Map headers = new HashMap<>(JSON_CONTENT_TYPE); + commonHeaderConsumer.accept(headers); + + final byte[] responseAsBytes = value.getBytes(StandardCharsets.UTF_8); + return new Response(RestStatus.OK.getStatus(), headers, responseAsBytes); + }; + + // https://cloud.google.com/compute/docs/storing-retrieving-metadata + handlers.insert(nonAuthPath(HttpGet.METHOD_NAME, "/computeMetadata/v1/project/project-id"), + request -> simpleValue.apply(PROJECT_ID)); + handlers.insert(nonAuthPath(HttpGet.METHOD_NAME, "/computeMetadata/v1/project/attributes/google-compute-default-zone"), + request -> simpleValue.apply(ZONE)); + // https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances + handlers.insert(nonAuthPath(HttpGet.METHOD_NAME, "/computeMetadata/v1/instance/service-accounts/default/token"), + request -> jsonValue.apply(Strings.toString(jsonBuilder() + .startObject() + .field("access_token", TOKEN) + .field("expires_in", TimeUnit.HOURS.toSeconds(1)) + .field("token_type", TOKEN_TYPE) + .endObject()))); + + // https://cloud.google.com/compute/docs/reference/rest/v1/instances + handlers.insert(authPath(HttpGet.METHOD_NAME, "/compute/v1/projects/{project}/zones/{zone}/instances"), + request -> { + final List items = new ArrayList(); + int count = 0; + for (String address : Files.readAllLines(nodes)) { + count++; + items.add(MapBuilder.newMapBuilder() + .put("id", Long.toString(9309873766405L + count)) + .put("description", "ES node" + count) + .put("name", "test" + count) + .put("kind", "compute#instance") + .put("machineType", "n1-standard-1") + .put("networkInterfaces", + Collections.singletonList(MapBuilder.newMapBuilder() + .put("accessConfigs", Collections.emptyList()) + .put("name", "nic0") + .put("network", "default") + .put("networkIP", address) + .immutableMap())) + .put("status", "RUNNING") + .put("zone", ZONE) + .immutableMap()); + } + + final String json = Strings.toString(jsonBuilder() + .startObject() + .field("id", "test-instances") + .field("items", items) + .endObject()); + + final byte[] responseAsBytes = json.getBytes(StandardCharsets.UTF_8); + final Map headers = new HashMap<>(JSON_CONTENT_TYPE); + commonHeaderConsumer.accept(headers); + return new Response(RestStatus.OK.getStatus(), headers, responseAsBytes); + }); + return handlers; + } + + @Override + protected Response handle(final Request request) throws IOException { + final String nonAuthorizedPath = nonAuthPath(request); + final RequestHandler nonAuthorizedHandler = handlers.retrieve(nonAuthorizedPath, request.getParameters()); + if (nonAuthorizedHandler != null) { + return nonAuthorizedHandler.handle(request); + } + + final String authorizedPath = authPath(request); + final RequestHandler authorizedHandler = handlers.retrieve(authorizedPath, request.getParameters()); + if (authorizedHandler != null) { + final String authorization = request.getHeader("Authorization"); + if ((TOKEN_TYPE + " " + TOKEN).equals(authorization) == false) { + return newError(RestStatus.UNAUTHORIZED, "Authorization", "Login Required"); + } + return authorizedHandler.handle(request); + } + + return null; + } + + private static Response newError(final RestStatus status, final String code, final String message) throws IOException { + final String response = Strings.toString(jsonBuilder() + .startObject() + .field("error", MapBuilder.newMapBuilder() + .put("errors", Collections.singletonList( + MapBuilder.newMapBuilder() + .put("domain", "global") + .put("reason", "required") + .put("message", message) + .put("locationType", "header") + .put("location", code) + .immutableMap() + )) + .put("code", status.getStatus()) + .put("message", message) + .immutableMap()) + .endObject()); + + return new Response(status.getStatus(), JSON_CONTENT_TYPE, response.getBytes(UTF_8)); + } + + @SuppressForbidden(reason = "Paths#get is fine - we don't have environment here") + private static Path toPath(final String dir) { + return Paths.get(dir); + } +} diff --git a/plugins/discovery-gce/qa/gce/src/test/resources/rest-api-spec/test/discovery_gce/10_basic.yml b/plugins/discovery-gce/qa/gce/src/test/resources/rest-api-spec/test/discovery_gce/10_basic.yml new file mode 100644 index 0000000000000..562d69a7a386c --- /dev/null +++ b/plugins/discovery-gce/qa/gce/src/test/resources/rest-api-spec/test/discovery_gce/10_basic.yml @@ -0,0 +1,15 @@ +# Integration tests for discovery-gce +setup: + - do: + cluster.health: + wait_for_status: green + wait_for_nodes: ${expected_nodes} + +--- +"All nodes are correctly discovered": + + - do: + nodes.info: + metric: [ transport ] + + - match: { _nodes.total: ${expected_nodes} } diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceInstancesService.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceInstancesService.java index d05f15344bc83..fc78a8a091b18 100644 --- a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceInstancesService.java +++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceInstancesService.java @@ -75,4 +75,8 @@ public interface GceInstancesService extends Closeable { * @return a collection of running instances within the same GCE project */ Collection instances(); + + String projectId(); + + List zones(); } diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceInstancesServiceImpl.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceInstancesServiceImpl.java index ed0bf07d75c7b..aab6e0c74ecdb 100644 --- a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceInstancesServiceImpl.java +++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceInstancesServiceImpl.java @@ -29,6 +29,11 @@ import com.google.api.client.googleapis.compute.ComputeCredential; import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.http.GenericUrl; +import com.google.api.client.http.HttpHeaders; +import com.google.api.client.http.HttpRequest; +import com.google.api.client.http.HttpRequestFactory; +import com.google.api.client.http.HttpResponse; import com.google.api.client.http.HttpTransport; import com.google.api.client.http.javanet.NetHttpTransport; import com.google.api.client.json.JsonFactory; @@ -103,9 +108,58 @@ public Collection instances() { public GceInstancesServiceImpl(Settings settings) { super(settings); - this.project = PROJECT_SETTING.get(settings); - this.zones = ZONE_SETTING.get(settings); this.validateCerts = GCE_VALIDATE_CERTIFICATES.get(settings); + this.project = resolveProject(); + this.zones = resolveZones(); + } + + private String resolveProject() { + if (PROJECT_SETTING.exists(settings)) { + return PROJECT_SETTING.get(settings); + } + + try { + // this code is based on a private GCE method: {@link com.google.cloud.ServiceOptions#getAppEngineProjectIdFromMetadataServer()} + return getAppEngineValueFromMetadataServer("/computeMetadata/v1/project/project-id"); + } catch (Exception e) { + logger.warn("unable to resolve project from metadata server for GCE discovery service", e); + } + return null; + } + + private List resolveZones() { + if (ZONE_SETTING.exists(settings)) { + return ZONE_SETTING.get(settings); + } + + try { + final String defaultZone = + getAppEngineValueFromMetadataServer("/computeMetadata/v1/project/attributes/google-compute-default-zone"); + return Collections.singletonList(defaultZone); + } catch (Exception e) { + logger.warn("unable to resolve default zone from metadata server for GCE discovery service", e); + } + return null; + } + + String getAppEngineValueFromMetadataServer(String serviceURL) throws GeneralSecurityException, IOException { + String metadata = GceMetadataService.GCE_HOST.get(settings); + GenericUrl url = Access.doPrivileged(() -> new GenericUrl(metadata + serviceURL)); + + HttpTransport httpTransport = getGceHttpTransport(); + HttpRequestFactory requestFactory = httpTransport.createRequestFactory(); + HttpRequest request = requestFactory.buildGetRequest(url) + .setConnectTimeout(500) + .setReadTimeout(500) + .setHeaders(new HttpHeaders().set("Metadata-Flavor", "Google")); + HttpResponse response = Access.doPrivilegedIOException(() -> request.execute()); + return headerContainsMetadataFlavor(response) ? response.parseAsString() : null; + } + + private static boolean headerContainsMetadataFlavor(HttpResponse response) { + // com.google.cloud.ServiceOptions#headerContainsMetadataFlavor(HttpResponse)} + String metadataFlavorValue = response.getHeaders().getFirstHeaderStringValue("Metadata-Flavor"); + return "Google".equals(metadataFlavorValue); } protected synchronized HttpTransport getGceHttpTransport() throws GeneralSecurityException, IOException { @@ -180,6 +234,16 @@ public synchronized Compute client() { return this.client; } + @Override + public String projectId() { + return project; + } + + @Override + public List zones() { + return zones; + } + @Override public void close() throws IOException { if (gceHttpTransport != null) { diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java index 778c38697c5ec..36f8aa36b34d0 100644 --- a/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java +++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java @@ -79,8 +79,8 @@ public GceUnicastHostsProvider(Settings settings, GceInstancesService gceInstanc this.networkService = networkService; this.refreshInterval = GceInstancesService.REFRESH_SETTING.get(settings); - this.project = GceInstancesService.PROJECT_SETTING.get(settings); - this.zones = GceInstancesService.ZONE_SETTING.get(settings); + this.project = gceInstancesService.projectId(); + this.zones = gceInstancesService.zones(); this.tags = TAGS_SETTING.get(settings); if (logger.isDebugEnabled()) { diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java index 183dfeda88477..418e3fffe2463 100644 --- a/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java +++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java @@ -22,6 +22,7 @@ import com.google.api.client.http.HttpHeaders; import com.google.api.client.util.ClassInfo; import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.Booleans; import org.elasticsearch.core.internal.io.IOUtils; import org.apache.lucene.util.SetOnce; import org.elasticsearch.cloud.gce.GceInstancesService; @@ -41,6 +42,7 @@ import java.io.Closeable; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -49,8 +51,12 @@ public class GceDiscoveryPlugin extends Plugin implements DiscoveryPlugin, Closeable { + /** Determines whether settings those reroutes GCE call should be allowed (for testing purposes only). */ + private static final boolean ALLOW_REROUTE_GCE_SETTINGS = + Booleans.parseBoolean(System.getProperty("es.allow_reroute_gce_settings", "false")); + public static final String GCE = "gce"; - private final Settings settings; + protected final Settings settings; private static final Logger logger = Loggers.getLogger(GceDiscoveryPlugin.class); // stashed when created in order to properly close private final SetOnce gceInstancesService = new SetOnce<>(); @@ -94,14 +100,22 @@ public NetworkService.CustomNameResolver getCustomNameResolver(Settings settings @Override public List> getSettings() { - return Arrays.asList( - // Register GCE settings - GceInstancesService.PROJECT_SETTING, - GceInstancesService.ZONE_SETTING, - GceUnicastHostsProvider.TAGS_SETTING, - GceInstancesService.REFRESH_SETTING, - GceInstancesService.RETRY_SETTING, - GceInstancesService.MAX_WAIT_SETTING); + List> settings = new ArrayList<>( + Arrays.asList( + // Register GCE settings + GceInstancesService.PROJECT_SETTING, + GceInstancesService.ZONE_SETTING, + GceUnicastHostsProvider.TAGS_SETTING, + GceInstancesService.REFRESH_SETTING, + GceInstancesService.RETRY_SETTING, + GceInstancesService.MAX_WAIT_SETTING) + ); + + if (ALLOW_REROUTE_GCE_SETTINGS) { + settings.add(GceMetadataService.GCE_HOST); + settings.add(GceInstancesServiceImpl.GCE_ROOT_URL); + } + return Collections.unmodifiableList(settings); } diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/cloud/gce/GceInstancesServiceImplTests.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/cloud/gce/GceInstancesServiceImplTests.java new file mode 100644 index 0000000000000..efb9b6c03d875 --- /dev/null +++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/cloud/gce/GceInstancesServiceImplTests.java @@ -0,0 +1,72 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.cloud.gce; + +import com.google.api.client.http.HttpTransport; +import com.google.api.client.http.LowLevelHttpRequest; +import com.google.api.client.http.LowLevelHttpResponse; +import com.google.api.client.json.Json; +import com.google.api.client.testing.http.MockHttpTransport; +import com.google.api.client.testing.http.MockLowLevelHttpRequest; +import com.google.api.client.testing.http.MockLowLevelHttpResponse; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESTestCase; + +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.core.Is.is; + +public class GceInstancesServiceImplTests extends ESTestCase { + + public void testHeaderContainsMetadataFlavor() throws Exception { + final AtomicBoolean addMetdataFlavor = new AtomicBoolean(); + final MockHttpTransport transport = new MockHttpTransport() { + @Override + public LowLevelHttpRequest buildRequest(String method, final String url) { + return new MockLowLevelHttpRequest() { + @Override + public LowLevelHttpResponse execute() { + MockLowLevelHttpResponse response = new MockLowLevelHttpResponse(); + response.setStatusCode(200); + response.setContentType(Json.MEDIA_TYPE); + response.setContent("value"); + if (addMetdataFlavor.get()) { + response.addHeader("Metadata-Flavor", "Google"); + } + return response; + } + }; + } + }; + + final GceInstancesServiceImpl service = new GceInstancesServiceImpl(Settings.EMPTY) { + @Override + protected synchronized HttpTransport getGceHttpTransport() { + return transport; + } + }; + + final String serviceURL = "/computeMetadata/v1/project/project-id"; + assertThat(service.getAppEngineValueFromMetadataServer(serviceURL), is(nullValue())); + + addMetdataFlavor.set(true); + assertThat(service.getAppEngineValueFromMetadataServer(serviceURL), is("value")); + } +} diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java index 3ceacdedcf7f2..cc5c400ce3b2c 100644 --- a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java +++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java @@ -170,6 +170,16 @@ public Collection instances() { }); } + @Override + public String projectId() { + return PROJECT_SETTING.get(settings); + } + + @Override + public List zones() { + return ZONE_SETTING.get(settings); + } + @Override public void close() throws IOException { } diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java index 816152186e761..629c2b3f52428 100644 --- a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java +++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.Version; import org.elasticsearch.cloud.gce.GceInstancesServiceImpl; +import org.elasticsearch.cloud.gce.GceMetadataService; import org.elasticsearch.common.network.NetworkService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; @@ -40,6 +41,7 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.not; /** * This test class uses a GCE HTTP Mock system which allows to simulate JSON Responses. @@ -211,7 +213,10 @@ public void testZeroNode43() { } public void testIllegalSettingsMissingAllRequired() { - Settings nodeSettings = Settings.EMPTY; + Settings nodeSettings = Settings.builder() + // to prevent being resolved using default GCE host + .put(GceMetadataService.GCE_HOST.getKey(), "http://internal") + .build(); mock = new GceInstancesServiceMock(nodeSettings); try { buildDynamicNodes(mock, nodeSettings); @@ -223,6 +228,8 @@ public void testIllegalSettingsMissingAllRequired() { public void testIllegalSettingsMissingProject() { Settings nodeSettings = Settings.builder() + // to prevent being resolved using default GCE host + .put(GceMetadataService.GCE_HOST.getKey(), "http://internal") .putList(GceInstancesServiceImpl.ZONE_SETTING.getKey(), "us-central1-a", "us-central1-b") .build(); mock = new GceInstancesServiceMock(nodeSettings); @@ -236,6 +243,8 @@ public void testIllegalSettingsMissingProject() { public void testIllegalSettingsMissingZone() { Settings nodeSettings = Settings.builder() + // to prevent being resolved using default GCE host + .put(GceMetadataService.GCE_HOST.getKey(), "http://internal") .put(GceInstancesServiceImpl.PROJECT_SETTING.getKey(), projectName) .build(); mock = new GceInstancesServiceMock(nodeSettings); @@ -261,4 +270,13 @@ public void testNoRegionReturnsEmptyList() { List dynamicHosts = buildDynamicNodes(mock, nodeSettings); assertThat(dynamicHosts, hasSize(1)); } + + public void testMetadataServerValues() { + Settings nodeSettings = Settings.EMPTY; + mock = new GceInstancesServiceMock(nodeSettings); + assertThat(mock.projectId(), not(projectName)); + + List dynamicHosts = buildDynamicNodes(mock, nodeSettings); + assertThat(dynamicHosts, hasSize(1)); + } } diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceInstancesServiceMock.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceInstancesServiceMock.java index d2612ca75ab74..0653391671bc8 100644 --- a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceInstancesServiceMock.java +++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceInstancesServiceMock.java @@ -32,11 +32,13 @@ public class GceInstancesServiceMock extends GceInstancesServiceImpl { public GceInstancesServiceMock(Settings settings) { super(settings); - this.mockHttpTransport = GceMockUtils.configureMock(); } @Override protected HttpTransport getGceHttpTransport() throws GeneralSecurityException, IOException { + if (this.mockHttpTransport == null) { + this.mockHttpTransport = GceMockUtils.configureMock(); + } return this.mockHttpTransport; } } diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceMockUtils.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceMockUtils.java index d0e33fcb67c8f..3a34e3629db80 100644 --- a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceMockUtils.java +++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceMockUtils.java @@ -39,7 +39,7 @@ public class GceMockUtils { protected static final Logger logger = Loggers.getLogger(GceMockUtils.class); - public static final String GCE_METADATA_URL = "http://metadata.google.internal/computeMetadata/v1/instance"; + public static final String GCE_METADATA_URL = "http://metadata.google.internal/computeMetadata/v1/"; protected static HttpTransport configureMock() { return new MockHttpTransport() { @@ -54,6 +54,7 @@ public LowLevelHttpResponse execute() throws IOException { if (url.startsWith(GCE_METADATA_URL)) { logger.info("--> Simulate GCE Auth/Metadata response for [{}]", url); response.setContent(readGoogleInternalJsonResponse(url)); + response.addHeader("Metadata-Flavor", "Google"); } else { logger.info("--> Simulate GCE API response for [{}]", url); response.setContent(readGoogleApiJsonResponse(url)); diff --git a/plugins/discovery-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/metadataserver/zones/europe-west1-b/instances b/plugins/discovery-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/metadataserver/zones/europe-west1-b/instances new file mode 100644 index 0000000000000..049e0e1e1b181 --- /dev/null +++ b/plugins/discovery-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/metadataserver/zones/europe-west1-b/instances @@ -0,0 +1,36 @@ +{ + "id": "dummy", + "items":[ + { + "description": "ES Node 1", + "id": "9309873766428965105", + "kind": "compute#instance", + "machineType": "n1-standard-1", + "name": "test1", + "networkInterfaces": [ + { + "accessConfigs": [ + { + "kind": "compute#accessConfig", + "name": "External NAT", + "natIP": "104.155.13.147", + "type": "ONE_TO_ONE_NAT" + } + ], + "name": "nic0", + "network": "default", + "networkIP": "10.240.79.59" + } + ], + "status": "RUNNING", + "tags": { + "fingerprint": "xA6QJb-rGtg=", + "items": [ + "elasticsearch", + "dev" + ] + }, + "zone": "europe-west1-b" + } + ] +} diff --git a/plugins/discovery-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/metadataserver/zones/us-central1-a/instances b/plugins/discovery-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/metadataserver/zones/us-central1-a/instances new file mode 100644 index 0000000000000..7e1e5d5d93a34 --- /dev/null +++ b/plugins/discovery-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/metadataserver/zones/us-central1-a/instances @@ -0,0 +1,36 @@ +{ + "id": "dummy", + "items":[ + { + "description": "ES Node 2", + "id": "9309873766428965105", + "kind": "compute#instance", + "machineType": "n1-standard-1", + "name": "test2", + "networkInterfaces": [ + { + "accessConfigs": [ + { + "kind": "compute#accessConfig", + "name": "External NAT", + "natIP": "104.155.13.147", + "type": "ONE_TO_ONE_NAT" + } + ], + "name": "nic0", + "network": "default", + "networkIP": "10.240.79.59" + } + ], + "status": "RUNNING", + "tags": { + "fingerprint": "xA6QJb-rGtg=", + "items": [ + "elasticsearch", + "dev" + ] + }, + "zone": "us-central1-a" + } + ] +} diff --git a/plugins/discovery-gce/src/test/resources/org/elasticsearch/discovery/gce/computeMetadata/v1/project/attributes/google-compute-default-zone b/plugins/discovery-gce/src/test/resources/org/elasticsearch/discovery/gce/computeMetadata/v1/project/attributes/google-compute-default-zone new file mode 100644 index 0000000000000..6cf886270bef1 --- /dev/null +++ b/plugins/discovery-gce/src/test/resources/org/elasticsearch/discovery/gce/computeMetadata/v1/project/attributes/google-compute-default-zone @@ -0,0 +1 @@ +europe-west1-b \ No newline at end of file diff --git a/plugins/discovery-gce/src/test/resources/org/elasticsearch/discovery/gce/computeMetadata/v1/project/project-id b/plugins/discovery-gce/src/test/resources/org/elasticsearch/discovery/gce/computeMetadata/v1/project/project-id new file mode 100644 index 0000000000000..25b8069381897 --- /dev/null +++ b/plugins/discovery-gce/src/test/resources/org/elasticsearch/discovery/gce/computeMetadata/v1/project/project-id @@ -0,0 +1 @@ +metadataserver \ No newline at end of file