From d7b31a8a355c713bbf9c5f5f5887d75e5452dd62 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Tue, 2 Jun 2020 09:40:37 +0200 Subject: [PATCH] Use 3rd party task to run integration tests on external service (#56587) --- plugins/repository-azure/build.gradle | 72 +++++-- plugins/repository-azure/qa/build.gradle | 1 - .../qa/microsoft-azure-storage/build.gradle | 94 --------- ...torageRepositoryClientYamlTestSuiteIT.java | 47 ----- .../AzureStorageCleanupThirdPartyTests.java | 12 ++ .../test/repository_azure/20_repository.yml} | 0 plugins/repository-gcs/build.gradle | 145 +++++++++++++- plugins/repository-gcs/qa/build.gradle | 1 - .../qa/google-cloud-storage/build.gradle | 177 ----------------- ...torageRepositoryClientYamlTestSuiteIT.java | 37 ---- .../GoogleCloudStorageThirdPartyTests.java | 0 .../test/repository_gcs/20_repository.yml} | 0 plugins/repository-s3/build.gradle | 181 ++++++++---------- .../qa/azure/build.gradle | 6 +- .../searchable-snapshots/qa/gcs/build.gradle | 9 +- .../searchable-snapshots/qa/s3/build.gradle | 8 +- 16 files changed, 308 insertions(+), 482 deletions(-) delete mode 100644 plugins/repository-azure/qa/build.gradle delete mode 100644 plugins/repository-azure/qa/microsoft-azure-storage/build.gradle delete mode 100644 plugins/repository-azure/qa/microsoft-azure-storage/src/test/java/org/elasticsearch/repositories/azure/AzureStorageRepositoryClientYamlTestSuiteIT.java rename plugins/repository-azure/{qa/microsoft-azure-storage/src/test/resources/rest-api-spec/test/repository_azure/10_repository.yml => src/test/resources/rest-api-spec/test/repository_azure/20_repository.yml} (100%) delete mode 100644 plugins/repository-gcs/qa/build.gradle delete mode 100644 plugins/repository-gcs/qa/google-cloud-storage/build.gradle delete mode 100644 plugins/repository-gcs/qa/google-cloud-storage/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepositoryClientYamlTestSuiteIT.java rename plugins/repository-gcs/{qa/google-cloud-storage => }/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageThirdPartyTests.java (100%) rename plugins/repository-gcs/{qa/google-cloud-storage/src/test/resources/rest-api-spec/test/repository_gcs/10_repository.yml => src/test/resources/rest-api-spec/test/repository_gcs/20_repository.yml} (100%) diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle index 4b9148b7cf75b..5a75d54112fba 100644 --- a/plugins/repository-azure/build.gradle +++ b/plugins/repository-azure/build.gradle @@ -1,5 +1,9 @@ +import org.elasticsearch.gradle.MavenFilteringHack import org.elasticsearch.gradle.info.BuildParams +import static org.elasticsearch.gradle.PropertyNormalization.DEFAULT +import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE + /* * Licensed to Elasticsearch under one or more contributor * license agreements. See the NOTICE file distributed with @@ -34,7 +38,7 @@ dependencies { restResources { restApi { - includeCore '_common', 'cluster', 'nodes' + includeCore '_common', 'cluster', 'nodes', 'snapshot', 'bulk', 'count', 'indices' } } @@ -69,16 +73,13 @@ thirdPartyAudit { ) } -check { - // also execute the QA tests when testing the plugin - dependsOn 'qa:microsoft-azure-storage:check' -} +boolean useFixture = false -testClusters { - integTest { - keystore 'azure.client.integration_test.account', 'azure_account' - keystore 'azure.client.integration_test.key', 'azure_key' - } +def azureAddress = { + assert useFixture: 'closure should not be used without a fixture' + int ephemeralPort = project(':test:fixtures:azure-fixture').postProcessFixture.ext."test.fixtures.azure-fixture.tcp.8091" + assert ephemeralPort > 0 + return 'ignored;DefaultEndpointsProtocol=http;BlobEndpoint=http://127.0.0.1:' + ephemeralPort + '/' } String azureAccount = System.getenv("azure_storage_account") @@ -87,19 +88,60 @@ String azureContainer = System.getenv("azure_storage_container") String azureBasePath = System.getenv("azure_storage_base_path") String azureSasToken = System.getenv("azure_storage_sas_token") +if (!azureAccount && !azureKey && !azureContainer && !azureBasePath && !azureSasToken) { + azureAccount = 'azure_integration_test_account' + azureKey = 'YXp1cmVfaW50ZWdyYXRpb25fdGVzdF9rZXk=' // The key is "azure_integration_test_key" encoded using base64 + azureContainer = 'container' + azureBasePath = '' + azureSasToken = '' + useFixture = true + + apply plugin: 'elasticsearch.test.fixtures' + testFixtures.useFixture ':test:fixtures:azure-fixture', 'azure-fixture' +} + +Map expansions = [ + 'container': azureContainer, + 'base_path': azureBasePath + "_integration_tests" +] + +processTestResources { + inputs.properties(expansions) + MavenFilteringHack.filter(it, expansions) +} + test { + // this is tested explicitly in a separate test task exclude '**/AzureStorageCleanupThirdPartyTests.class' } -task thirdPartyTest(type: Test) { +testClusters { + integTest { + keystore 'azure.client.integration_test.account', azureAccount + if (azureKey != null && azureKey.isEmpty() == false) { + keystore 'azure.client.integration_test.key', azureKey + } + if (azureSasToken != null && azureSasToken.isEmpty() == false) { + keystore 'azure.client.integration_test.sas_token', azureSasToken + } + if (useFixture) { + setting 'azure.client.integration_test.endpoint_suffix', azureAddress + String firstPartOfSeed = BuildParams.testSeed.tokenize(':').get(0) + setting 'thread_pool.repository_azure.max', (Math.abs(Long.parseUnsignedLong(firstPartOfSeed, 16) % 10) + 1).toString(), System.getProperty('ignore.tests.seed') == null ? DEFAULT : IGNORE_VALUE + } + } +} + +task azureThirdPartyTest(type: Test) { + dependsOn tasks.integTest include '**/AzureStorageCleanupThirdPartyTests.class' systemProperty 'test.azure.account', azureAccount ? azureAccount : "" systemProperty 'test.azure.key', azureKey ? azureKey : "" systemProperty 'test.azure.sas_token', azureSasToken ? azureSasToken : "" systemProperty 'test.azure.container', azureContainer ? azureContainer : "" systemProperty 'test.azure.base', (azureBasePath ? azureBasePath : "") + "_third_party_tests_" + BuildParams.testSeed + if (useFixture) { + nonInputProperties.systemProperty 'test.azure.endpoint_suffix', "${-> azureAddress.call() }" + } } - -if (azureAccount || azureKey || azureContainer || azureBasePath || azureSasToken) { - check.dependsOn(thirdPartyTest) -} +check.dependsOn(azureThirdPartyTest) diff --git a/plugins/repository-azure/qa/build.gradle b/plugins/repository-azure/qa/build.gradle deleted file mode 100644 index 5c25485a8f596..0000000000000 --- a/plugins/repository-azure/qa/build.gradle +++ /dev/null @@ -1 +0,0 @@ -group = "${group}.plugins.repository-azure.qa" diff --git a/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle b/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle deleted file mode 100644 index 3c2146095e651..0000000000000 --- a/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -import org.elasticsearch.gradle.MavenFilteringHack -import org.elasticsearch.gradle.info.BuildParams -import org.elasticsearch.gradle.test.AntFixture - -import static org.elasticsearch.gradle.PropertyNormalization.DEFAULT -import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE - -apply plugin: 'elasticsearch.standalone-rest-test' -apply plugin: 'elasticsearch.rest-test' -apply plugin: 'elasticsearch.test.fixtures' - -restResources { - restApi { - includeCore '_common', 'snapshot', 'bulk', 'count', 'indices' - } -} - -testFixtures.useFixture ":test:fixtures:azure-fixture", "azure-fixture" - -boolean useFixture = false - -String azureAccount = System.getenv("azure_storage_account") -String azureKey = System.getenv("azure_storage_key") -String azureContainer = System.getenv("azure_storage_container") -String azureBasePath = System.getenv("azure_storage_base_path") -String azureSasToken = System.getenv("azure_storage_sas_token") - -if (!azureAccount && !azureKey && !azureContainer && !azureBasePath && !azureSasToken) { - azureAccount = 'azure_integration_test_account' - azureKey = 'YXp1cmVfaW50ZWdyYXRpb25fdGVzdF9rZXk=' // The key is "azure_integration_test_key" encoded using base64 - azureContainer = 'container' - azureBasePath = '' - azureSasToken = '' - useFixture = true -} - -Map expansions = [ - 'container': azureContainer, - 'base_path': azureBasePath + "_integration_tests" -] - -processTestResources { - inputs.properties(expansions) - MavenFilteringHack.filter(it, expansions) -} - -integTest { - dependsOn project(':plugins:repository-azure').bundlePlugin -} - -testClusters.integTest { - plugin project(':plugins:repository-azure').bundlePlugin.archiveFile - keystore 'azure.client.integration_test.account', azureAccount - if (azureKey != null && azureKey.isEmpty() == false) { - keystore 'azure.client.integration_test.key', azureKey - } - if (azureSasToken != null && azureSasToken.isEmpty() == false) { - keystore 'azure.client.integration_test.sas_token', azureSasToken - } - - if (useFixture) { - def azureAddress = { - int ephemeralPort = project(':test:fixtures:azure-fixture').postProcessFixture.ext."test.fixtures.azure-fixture.tcp.8091" - assert ephemeralPort > 0 - 'http://127.0.0.1:' + ephemeralPort - } - // Use a closure on the string to delay evaluation until tests are executed. The endpoint_suffix is used - // in a hacky way to change the protocol and endpoint. We must fix that. - setting 'azure.client.integration_test.endpoint_suffix', - { "ignored;DefaultEndpointsProtocol=http;BlobEndpoint=${-> azureAddress()}" }, IGNORE_VALUE - String firstPartOfSeed = BuildParams.testSeed.tokenize(':').get(0) - setting 'thread_pool.repository_azure.max', (Math.abs(Long.parseUnsignedLong(firstPartOfSeed, 16) % 10) + 1).toString(), System.getProperty('ignore.tests.seed') == null ? DEFAULT : IGNORE_VALUE - } -} diff --git a/plugins/repository-azure/qa/microsoft-azure-storage/src/test/java/org/elasticsearch/repositories/azure/AzureStorageRepositoryClientYamlTestSuiteIT.java b/plugins/repository-azure/qa/microsoft-azure-storage/src/test/java/org/elasticsearch/repositories/azure/AzureStorageRepositoryClientYamlTestSuiteIT.java deleted file mode 100644 index 9822da98fde70..0000000000000 --- a/plugins/repository-azure/qa/microsoft-azure-storage/src/test/java/org/elasticsearch/repositories/azure/AzureStorageRepositoryClientYamlTestSuiteIT.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.repositories.azure; - -import com.carrotsearch.randomizedtesting.annotations.Name; -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.test.rest.ESRestTestCase; -import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; -import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; - -public class AzureStorageRepositoryClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { - - public AzureStorageRepositoryClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { - super(testCandidate); - } - - @ParametersFactory - public static Iterable parameters() throws Exception { - return ESClientYamlSuiteTestCase.createParameters(); - } - - @Override - protected Settings restClientSettings() { - // Give more time to repository-azure to complete the snapshot operations - return Settings.builder().put(super.restClientSettings()) - .put(ESRestTestCase.CLIENT_SOCKET_TIMEOUT, "60s") - .build(); - } -} diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java index f791550adeb99..985fae740702e 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java @@ -50,6 +50,18 @@ protected Collection> getPlugins() { return pluginList(AzureRepositoryPlugin.class); } + @Override + protected Settings nodeSettings() { + final String endpoint = System.getProperty("test.azure.endpoint_suffix"); + if (Strings.hasText(endpoint)) { + return Settings.builder() + .put(super.nodeSettings()) + .put("azure.client.default.endpoint_suffix", endpoint) + .build(); + } + return super.nodeSettings(); + } + @Override protected SecureSettings credentials() { assertThat(System.getProperty("test.azure.account"), not(blankOrNullString())); diff --git a/plugins/repository-azure/qa/microsoft-azure-storage/src/test/resources/rest-api-spec/test/repository_azure/10_repository.yml b/plugins/repository-azure/src/test/resources/rest-api-spec/test/repository_azure/20_repository.yml similarity index 100% rename from plugins/repository-azure/qa/microsoft-azure-storage/src/test/resources/rest-api-spec/test/repository_azure/10_repository.yml rename to plugins/repository-azure/src/test/resources/rest-api-spec/test/repository_azure/20_repository.yml diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle index 52377735048d2..93fa4fa1dbe25 100644 --- a/plugins/repository-gcs/build.gradle +++ b/plugins/repository-gcs/build.gradle @@ -1,3 +1,15 @@ +import java.nio.file.Files +import java.security.KeyPair +import java.security.KeyPairGenerator +import org.elasticsearch.gradle.MavenFilteringHack +import org.elasticsearch.gradle.info.BuildParams +import org.elasticsearch.gradle.test.RestIntegTestTask + +import java.nio.file.Files +import java.security.KeyPair +import java.security.KeyPairGenerator + +import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE /* * Licensed to Elasticsearch under one or more contributor * license agreements. See the NOTICE file distributed with @@ -56,7 +68,7 @@ dependencies { restResources { restApi { - includeCore '_common', 'cluster', 'nodes' + includeCore '_common', 'cluster', 'nodes', 'snapshot','indices', 'index', 'bulk', 'count' } } @@ -182,7 +194,132 @@ thirdPartyAudit { ) } -check { - // also execute the QA tests when testing the plugin - dependsOn 'qa:google-cloud-storage:check' +boolean useFixture = false + +def fixtureAddress = { fixture -> + assert useFixture: 'closure should not be used without a fixture' + int ephemeralPort = project(':test:fixtures:gcs-fixture').postProcessFixture.ext."test.fixtures.${fixture}.tcp.80" + assert ephemeralPort > 0 + 'http://127.0.0.1:' + ephemeralPort +} + +String gcsServiceAccount = System.getenv("google_storage_service_account") +String gcsBucket = System.getenv("google_storage_bucket") +String gcsBasePath = System.getenv("google_storage_base_path") +File serviceAccountFile = null + +if (!gcsServiceAccount && !gcsBucket && !gcsBasePath) { + serviceAccountFile = new File(project.buildDir, 'generated-resources/service_account_test.json') + gcsBucket = 'bucket' + gcsBasePath = 'integration_test' + useFixture = true + + apply plugin: 'elasticsearch.test.fixtures' + testFixtures.useFixture(':test:fixtures:gcs-fixture', 'gcs-fixture') + testFixtures.useFixture(':test:fixtures:gcs-fixture', 'gcs-fixture-third-party') + +} else if (!gcsServiceAccount || !gcsBucket || !gcsBasePath) { + throw new IllegalArgumentException("not all options specified to run tests against external GCS service are present") +} else { + serviceAccountFile = new File(gcsServiceAccount) +} + +def encodedCredentials = { + Base64.encoder.encodeToString(Files.readAllBytes(serviceAccountFile.toPath())) +} + +/** A service account file that points to the Google Cloud Storage service emulated by the fixture **/ +task createServiceAccountFile() { + doLast { + KeyPairGenerator keyPairGenerator = KeyPairGenerator.getInstance("RSA") + keyPairGenerator.initialize(1024) + KeyPair keyPair = keyPairGenerator.generateKeyPair() + String encodedKey = Base64.getEncoder().encodeToString(keyPair.private.getEncoded()) + + serviceAccountFile.parentFile.mkdirs() + serviceAccountFile.setText("{\n" + + ' "type": "service_account",\n' + + ' "project_id": "integration_test",\n' + + ' "private_key_id": "' + UUID.randomUUID().toString() + '",\n' + + ' "private_key": "-----BEGIN PRIVATE KEY-----\\n' + encodedKey + '\\n-----END PRIVATE KEY-----\\n",\n' + + ' "client_email": "integration_test@appspot.gserviceaccount.com",\n' + + ' "client_id": "123456789101112130594"\n' + + '}', 'UTF-8') + } +} + +Map expansions = [ + 'bucket' : gcsBucket, + 'base_path': gcsBasePath + "_integration_tests" +] + +processTestResources { + inputs.properties(expansions) + MavenFilteringHack.filter(it, expansions) +} + +test { + // this is tested explicitly in a separate test task + exclude '**/GoogleCloudStorageThirdPartyTests.class' +} + +final Closure testClustersConfiguration = { + keystore 'gcs.client.integration_test.credentials_file', serviceAccountFile, IGNORE_VALUE + + if (useFixture) { + /* Use a closure on the string to delay evaluation until tests are executed */ + setting 'gcs.client.integration_test.endpoint', { "${-> fixtureAddress('gcs-fixture')}" }, IGNORE_VALUE + setting 'gcs.client.integration_test.token_uri', { "${-> fixtureAddress('gcs-fixture')}/o/oauth2/token" }, IGNORE_VALUE + } else { + println "Using an external service to test the repository-gcs plugin" + } +} + +integTest { + if (useFixture) { + dependsOn createServiceAccountFile + } +} +check.dependsOn integTest + +testClusters { + integTest testClustersConfiguration +} + +/* + * We only use a small amount of data in these tests, which means that the resumable upload path is not tested. We add + * an additional test that forces the large blob threshold to be small to exercise the resumable upload path. + */ +task largeBlobIntegTest(type: RestIntegTestTask) { + mustRunAfter integTest + dependsOn project(':plugins:repository-gcs').bundlePlugin + if (useFixture) { + dependsOn createServiceAccountFile + } +} +check.dependsOn largeBlobIntegTest + +testClusters.largeBlobIntegTest testClustersConfiguration +testClusters { + largeBlobIntegTest { + plugin project(':plugins:repository-gcs').bundlePlugin.archiveFile + + // force large blob uploads by setting the threshold small, forcing this code path to be tested + systemProperty 'es.repository_gcs.large_blob_threshold_byte_size', '256' + } +} + +task gcsThirdPartyTest(type: Test) { + dependsOn integTest,largeBlobIntegTest + include '**/GoogleCloudStorageThirdPartyTests.class' + systemProperty 'tests.security.manager', false + systemProperty 'test.google.bucket', gcsBucket + nonInputProperties.systemProperty 'test.google.base', gcsBasePath + "_third_party_tests_" + BuildParams.testSeed + nonInputProperties.systemProperty 'test.google.account', "${-> encodedCredentials.call()}" + if (useFixture) { + dependsOn createServiceAccountFile + nonInputProperties.systemProperty 'test.google.endpoint', "${-> fixtureAddress('gcs-fixture-third-party')}" + nonInputProperties.systemProperty 'test.google.tokenURI', "${-> fixtureAddress('gcs-fixture-third-party')}/o/oauth2/token" + } } +check.dependsOn(gcsThirdPartyTest) diff --git a/plugins/repository-gcs/qa/build.gradle b/plugins/repository-gcs/qa/build.gradle deleted file mode 100644 index d10d9050dfecf..0000000000000 --- a/plugins/repository-gcs/qa/build.gradle +++ /dev/null @@ -1 +0,0 @@ -group = "${group}.plugins.repository-gcs.qa" diff --git a/plugins/repository-gcs/qa/google-cloud-storage/build.gradle b/plugins/repository-gcs/qa/google-cloud-storage/build.gradle deleted file mode 100644 index 6995b9bec45c9..0000000000000 --- a/plugins/repository-gcs/qa/google-cloud-storage/build.gradle +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -import org.elasticsearch.gradle.MavenFilteringHack -import org.elasticsearch.gradle.info.BuildParams -import org.elasticsearch.gradle.test.RestIntegTestTask - -import java.nio.file.Files -import java.security.KeyPair -import java.security.KeyPairGenerator - -import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE - -apply plugin: 'elasticsearch.standalone-rest-test' -apply plugin: 'elasticsearch.rest-test' -apply plugin: 'elasticsearch.test.fixtures' - -// TODO think about flattening qa:google-cloud-storage project into parent -dependencies { - testCompile project(path: ':plugins:repository-gcs') -} - -restResources { - restApi { - includeCore '_common', 'snapshot','indices', 'index', 'bulk', 'count' - } -} - -testFixtures.useFixture(':test:fixtures:gcs-fixture', 'gcs-fixture') -testFixtures.useFixture(':test:fixtures:gcs-fixture', 'gcs-fixture-third-party') -boolean useFixture = false - -String gcsServiceAccount = System.getenv("google_storage_service_account") -String gcsBucket = System.getenv("google_storage_bucket") -String gcsBasePath = System.getenv("google_storage_base_path") - -File serviceAccountFile = null -if (!gcsServiceAccount && !gcsBucket && !gcsBasePath) { - serviceAccountFile = new File(project.buildDir, 'generated-resources/service_account_test.json') - gcsBucket = 'bucket' - gcsBasePath = 'integration_test' - useFixture = true -} else if (!gcsServiceAccount || !gcsBucket || !gcsBasePath) { - throw new IllegalArgumentException("not all options specified to run tests against external GCS service are present") -} else { - serviceAccountFile = new File(gcsServiceAccount) -} - -def encodedCredentials = { - Base64.encoder.encodeToString(Files.readAllBytes(serviceAccountFile.toPath())) -} - -def fixtureAddress = { fixture -> - assert useFixture: 'closure should not be used without a fixture' - int ephemeralPort = project(':test:fixtures:gcs-fixture').postProcessFixture.ext."test.fixtures.${fixture}.tcp.80" - assert ephemeralPort > 0 - 'http://127.0.0.1:' + ephemeralPort -} - -/** A service account file that points to the Google Cloud Storage service emulated by the fixture **/ -task createServiceAccountFile() { - doLast { - KeyPairGenerator keyPairGenerator = KeyPairGenerator.getInstance("RSA") - keyPairGenerator.initialize(1024) - KeyPair keyPair = keyPairGenerator.generateKeyPair() - String encodedKey = Base64.getEncoder().encodeToString(keyPair.private.getEncoded()) - - serviceAccountFile.parentFile.mkdirs() - serviceAccountFile.setText("{\n" + - ' "type": "service_account",\n' + - ' "project_id": "integration_test",\n' + - ' "private_key_id": "' + UUID.randomUUID().toString() + '",\n' + - ' "private_key": "-----BEGIN PRIVATE KEY-----\\n' + encodedKey + '\\n-----END PRIVATE KEY-----\\n",\n' + - ' "client_email": "integration_test@appspot.gserviceaccount.com",\n' + - ' "client_id": "123456789101112130594"\n' + - '}', 'UTF-8') - } -} - -task thirdPartyTest(type: Test) { - if (useFixture) { - thirdPartyTest.dependsOn createServiceAccountFile - nonInputProperties.systemProperty 'test.google.endpoint', "${-> fixtureAddress('gcs-fixture-third-party')}" - nonInputProperties.systemProperty 'test.google.tokenURI', "${-> fixtureAddress('gcs-fixture-third-party')}/o/oauth2/token" - - gradle.taskGraph.whenReady { - if (it.hasTask(gcsThirdPartyTests)) { - throw new IllegalStateException("Tried to run third party tests but not all of the necessary environment variables " + - "'google_storage_service_account', 'google_storage_bucket', 'google_storage_base_path' are set.") - } - } - } - - include '**/GoogleCloudStorageThirdPartyTests.class' - systemProperty 'tests.security.manager', false - systemProperty 'test.google.bucket', gcsBucket - nonInputProperties.systemProperty 'test.google.base', gcsBasePath + "_third_party_tests_" + BuildParams.testSeed - nonInputProperties.systemProperty 'test.google.account', "${-> encodedCredentials.call()}" -} - -task gcsThirdPartyTests { - dependsOn check -} - -check.dependsOn thirdPartyTest - -integTest.mustRunAfter(thirdPartyTest) - -/* - * We only use a small amount of data in these tests, which means that the resumable upload path is not tested. We add - * an additional test that forces the large blob threshold to be small to exercise the resumable upload path. - */ -task largeBlobIntegTest(type: RestIntegTestTask) { - mustRunAfter(thirdPartyTest, integTest) -} - -check.dependsOn integTest -check.dependsOn largeBlobIntegTest - -Map expansions = [ - 'bucket' : gcsBucket, - 'base_path': gcsBasePath + "_integration_tests" -] - -processTestResources { - inputs.properties(expansions) - MavenFilteringHack.filter(it, expansions) -} - -final Closure integTestConfiguration = { - dependsOn project(':plugins:repository-gcs').bundlePlugin -} - -integTest integTestConfiguration - -largeBlobIntegTest integTestConfiguration - -final Closure testClustersConfiguration = { - plugin project(':plugins:repository-gcs').bundlePlugin.archiveFile - - keystore 'gcs.client.integration_test.credentials_file', serviceAccountFile, IGNORE_VALUE - - if (useFixture) { - tasks.integTest.dependsOn createServiceAccountFile - tasks.largeBlobIntegTest.dependsOn createServiceAccountFile - /* Use a closure on the string to delay evaluation until tests are executed */ - setting 'gcs.client.integration_test.endpoint', { "${-> fixtureAddress('gcs-fixture')}" }, IGNORE_VALUE - setting 'gcs.client.integration_test.token_uri', { "${-> fixtureAddress('gcs-fixture')}/o/oauth2/token" }, IGNORE_VALUE - } else { - println "Using an external service to test the repository-gcs plugin" - } -} - -testClusters.integTest testClustersConfiguration - -testClusters.largeBlobIntegTest testClustersConfiguration -testClusters.largeBlobIntegTest { - // force large blob uploads by setting the threshold small, forcing this code path to be tested - systemProperty 'es.repository_gcs.large_blob_threshold_byte_size', '256' -} diff --git a/plugins/repository-gcs/qa/google-cloud-storage/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepositoryClientYamlTestSuiteIT.java b/plugins/repository-gcs/qa/google-cloud-storage/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepositoryClientYamlTestSuiteIT.java deleted file mode 100644 index 8d85b38919380..0000000000000 --- a/plugins/repository-gcs/qa/google-cloud-storage/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepositoryClientYamlTestSuiteIT.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.repositories.gcs; - -import com.carrotsearch.randomizedtesting.annotations.Name; -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; -import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; - -public class GoogleCloudStorageRepositoryClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { - - public GoogleCloudStorageRepositoryClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { - super(testCandidate); - } - - @ParametersFactory - public static Iterable parameters() throws Exception { - return createParameters(); - } -} diff --git a/plugins/repository-gcs/qa/google-cloud-storage/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageThirdPartyTests.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageThirdPartyTests.java similarity index 100% rename from plugins/repository-gcs/qa/google-cloud-storage/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageThirdPartyTests.java rename to plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageThirdPartyTests.java diff --git a/plugins/repository-gcs/qa/google-cloud-storage/src/test/resources/rest-api-spec/test/repository_gcs/10_repository.yml b/plugins/repository-gcs/src/test/resources/rest-api-spec/test/repository_gcs/20_repository.yml similarity index 100% rename from plugins/repository-gcs/qa/google-cloud-storage/src/test/resources/rest-api-spec/test/repository_gcs/10_repository.yml rename to plugins/repository-gcs/src/test/resources/rest-api-spec/test/repository_gcs/20_repository.yml diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index f8bcf2e0f54db..b316486bbdbf3 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -22,8 +22,6 @@ import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE * specific language governing permissions and limitations * under the License. */ -apply plugin: 'elasticsearch.test.fixtures' - esplugin { description 'The S3 repository plugin adds S3 repositories' classname 'org.elasticsearch.repositories.s3.S3RepositoryPlugin' @@ -68,13 +66,15 @@ bundlePlugin { } } -test { - // this is tested explicitly in separate test tasks - exclude '**/S3RepositoryThirdPartyTests.class' -} - boolean useFixture = false +def fixtureAddress = { fixture, name, port -> + assert useFixture: 'closure should not be used without a fixture' + int ephemeralPort = project(":test:fixtures:${fixture}").postProcessFixture.ext."test.fixtures.${name}.tcp.${port}" + assert ephemeralPort > 0 + 'http://127.0.0.1:' + ephemeralPort +} + // We test against two repositories, one which uses the usual two-part "permanent" credentials and // the other which uses three-part "temporary" or "session" credentials. @@ -106,6 +106,7 @@ if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3P s3PermanentBucket = 'bucket' s3PermanentBasePath = 'base_path' + apply plugin: 'elasticsearch.test.fixtures' useFixture = true } else if (!s3PermanentAccessKey || !s3PermanentSecretKey || !s3PermanentBucket || !s3PermanentBasePath) { @@ -132,72 +133,6 @@ if (!s3EC2Bucket && !s3EC2BasePath && !s3ECSBucket && !s3ECSBasePath) { throw new IllegalArgumentException("not all options specified to run EC2/ECS tests are present") } -task thirdPartyTest(type: Test) { - include '**/S3RepositoryThirdPartyTests.class' - systemProperty 'test.s3.account', s3PermanentAccessKey - systemProperty 'test.s3.key', s3PermanentSecretKey - systemProperty 'test.s3.bucket', s3PermanentBucket - nonInputProperties.systemProperty 'test.s3.base', s3PermanentBasePath + "_third_party_tests_" + BuildParams.testSeed -} - -if (useFixture) { - testFixtures.useFixture(':test:fixtures:minio-fixture', 'minio-fixture') - - def minioAddress = { - int minioPort = project(':test:fixtures:minio-fixture').postProcessFixture.ext."test.fixtures.minio-fixture.tcp.9000" - assert minioPort > 0 - 'http://127.0.0.1:' + minioPort - } - - normalization { - runtimeClasspath { - // ignore generated address file for the purposes of build avoidance - ignore 's3Fixture.address' - } - } - - thirdPartyTest { - dependsOn tasks.bundlePlugin - nonInputProperties.systemProperty 'test.s3.endpoint', "${ -> minioAddress.call() }" - } - - task integTestMinio(type: RestIntegTestTask) { - description = "Runs REST tests using the Minio repository." - dependsOn tasks.bundlePlugin - runner { - // Minio only supports a single access key, see https://github.com/minio/minio/pull/5968 - systemProperty 'tests.rest.blacklist', [ - 'repository_s3/30_repository_temporary_credentials/*', - 'repository_s3/40_repository_ec2_credentials/*', - 'repository_s3/50_repository_ecs_credentials/*' - ].join(",") - } - } - check.dependsOn(integTestMinio) - - testClusters.integTestMinio { - keystore 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey - keystore 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey - setting 's3.client.integration_test_permanent.endpoint', minioAddress, IGNORE_VALUE - plugin tasks.bundlePlugin.archiveFile - } - - integTest.runner { - systemProperty 'tests.rest.blacklist', 'repository_s3/50_repository_ecs_credentials/*' - } -} else { - integTest.runner { - systemProperty 'tests.rest.blacklist', - [ - 'repository_s3/30_repository_temporary_credentials/*', - 'repository_s3/40_repository_ec2_credentials/*', - 'repository_s3/50_repository_ecs_credentials/*' - ].join(",") - } -} - -check.dependsOn(thirdPartyTest) - processTestResources { Map expansions = [ 'permanent_bucket' : s3PermanentBucket, @@ -214,18 +149,25 @@ processTestResources { MavenFilteringHack.filter(it, expansions) } -[ - 's3-fixture', - 's3-fixture-with-session-token', - 's3-fixture-with-ec2', - 's3-fixture-with-ecs', -].forEach { fixture -> testFixtures.useFixture(':test:fixtures:s3-fixture', fixture) } +test { + // this is tested explicitly in a separate test task + exclude '**/S3RepositoryThirdPartyTests.class' +} -def fixtureAddress = { fixture -> - assert useFixture: 'closure should not be used without a fixture' - int ephemeralPort = project(':test:fixtures:s3-fixture').postProcessFixture.ext."test.fixtures.${fixture}.tcp.80" - assert ephemeralPort > 0 - 'http://127.0.0.1:' + ephemeralPort +// IntegTest +integTest { + runner { + systemProperty 'tests.rest.blacklist', ( + useFixture ? + ['repository_s3/50_repository_ecs_credentials/*'] + : + [ + 'repository_s3/30_repository_temporary_credentials/*', + 'repository_s3/40_repository_ec2_credentials/*', + 'repository_s3/50_repository_ecs_credentials/*' + ] + ).join(",") + } } testClusters.integTest { @@ -237,22 +179,58 @@ testClusters.integTest { keystore 's3.client.integration_test_temporary.session_token', s3TemporarySessionToken if (useFixture) { - setting 's3.client.integration_test_permanent.endpoint', { "${-> fixtureAddress('s3-fixture')}" }, IGNORE_VALUE - setting 's3.client.integration_test_temporary.endpoint', { "${-> fixtureAddress('s3-fixture-with-session-token')}" }, IGNORE_VALUE - setting 's3.client.integration_test_ec2.endpoint', { "${-> fixtureAddress('s3-fixture-with-ec2')}" }, IGNORE_VALUE + testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture') + testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture-with-session-token') + testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture-with-ec2') + + normalization { + runtimeClasspath { + // ignore generated address file for the purposes of build avoidance + ignore 's3Fixture.address' + } + } + + setting 's3.client.integration_test_permanent.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture', '80')}" }, IGNORE_VALUE + setting 's3.client.integration_test_temporary.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-session-token', '80')}" }, IGNORE_VALUE + setting 's3.client.integration_test_ec2.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-ec2', '80')}" }, IGNORE_VALUE // to redirect InstanceProfileCredentialsProvider to custom auth point - systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", { "${-> fixtureAddress('s3-fixture-with-ec2')}" }, IGNORE_VALUE + systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-ec2', '80')}" }, IGNORE_VALUE } else { println "Using an external service to test the repository-s3 plugin" } } -task s3ThirdPartyTests { - dependsOn check +// MinIO +if (useFixture) { + testFixtures.useFixture(':test:fixtures:minio-fixture', 'minio-fixture') + + task integTestMinio(type: RestIntegTestTask) { + description = "Runs REST tests using the Minio repository." + dependsOn tasks.bundlePlugin + runner { + // Minio only supports a single access key, see https://github.com/minio/minio/pull/5968 + systemProperty 'tests.rest.blacklist', [ + 'repository_s3/30_repository_temporary_credentials/*', + 'repository_s3/40_repository_ec2_credentials/*', + 'repository_s3/50_repository_ecs_credentials/*' + ].join(",") + } + } + check.dependsOn(integTestMinio) + + testClusters.integTestMinio { + keystore 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey + keystore 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey + setting 's3.client.integration_test_permanent.endpoint', { "${-> fixtureAddress('minio-fixture', 'minio-fixture', '9000')}" }, IGNORE_VALUE + plugin tasks.bundlePlugin.archiveFile + } } +// ECS if (useFixture) { + testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture-with-ecs') + task integTestECS(type: RestIntegTestTask.class) { description = "Runs tests using the ECS repository." dependsOn('bundlePlugin') @@ -268,18 +246,27 @@ if (useFixture) { check.dependsOn(integTestECS) testClusters.integTestECS { - setting 's3.client.integration_test_ecs.endpoint', { "${-> fixtureAddress('s3-fixture-with-ecs')}" }, IGNORE_VALUE + setting 's3.client.integration_test_ecs.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-ecs', '80')}" }, IGNORE_VALUE plugin tasks.bundlePlugin.archiveFile - environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI', { "${-> fixtureAddress('s3-fixture-with-ecs')}/ecs_credentials_endpoint" }, IGNORE_VALUE + environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-ecs', '80')}/ecs_credentials_endpoint" }, IGNORE_VALUE } +} - gradle.taskGraph.whenReady { - if (it.hasTask(s3ThirdPartyTests)) { - throw new IllegalStateException("Tried to run third party tests but not all of the necessary environment variables 'amazon_s3_access_key', " + - "'amazon_s3_secret_key', 'amazon_s3_bucket', and 'amazon_s3_base_path' are set."); - } +// 3rd Party Tests +task s3ThirdPartyTest(type: Test) { + include '**/S3RepositoryThirdPartyTests.class' + systemProperty 'test.s3.account', s3PermanentAccessKey + systemProperty 'test.s3.key', s3PermanentSecretKey + systemProperty 'test.s3.bucket', s3PermanentBucket + nonInputProperties.systemProperty 'test.s3.base', s3PermanentBasePath + "_third_party_tests_" + BuildParams.testSeed + if (useFixture) { + dependsOn tasks.integTestMinio + nonInputProperties.systemProperty 'test.s3.endpoint', "${-> fixtureAddress('minio-fixture', 'minio-fixture', '9000') }" + } else { + dependsOn tasks.integTest } } +check.dependsOn(s3ThirdPartyTest) thirdPartyAudit.ignoreMissingClasses( // classes are missing diff --git a/x-pack/plugin/searchable-snapshots/qa/azure/build.gradle b/x-pack/plugin/searchable-snapshots/qa/azure/build.gradle index b544bf60cf4ff..616ef163d9203 100644 --- a/x-pack/plugin/searchable-snapshots/qa/azure/build.gradle +++ b/x-pack/plugin/searchable-snapshots/qa/azure/build.gradle @@ -63,7 +63,7 @@ integTest { dependsOn repositoryPlugin.bundlePlugin runner { systemProperty 'test.azure.container', azureContainer - systemProperty 'test.azure.base_path', azureBasePath + "/searchable_snapshots_tests" + systemProperty 'test.azure.base_path', azureBasePath + "_searchable_snapshots_tests" } } @@ -100,3 +100,7 @@ testClusters.integTest { } } +task azureThirdPartyTest { + dependsOn integTest +} + diff --git a/x-pack/plugin/searchable-snapshots/qa/gcs/build.gradle b/x-pack/plugin/searchable-snapshots/qa/gcs/build.gradle index 684a4faf1ff44..cd1e6a21966f5 100644 --- a/x-pack/plugin/searchable-snapshots/qa/gcs/build.gradle +++ b/x-pack/plugin/searchable-snapshots/qa/gcs/build.gradle @@ -61,10 +61,6 @@ if (!gcsServiceAccount && !gcsBucket && !gcsBasePath) { serviceAccountFile = new File(gcsServiceAccount) } -def encodedCredentials = { - Base64.encoder.encodeToString(Files.readAllBytes(serviceAccountFile.toPath())) -} - /** A service account file that points to the Google Cloud Storage service emulated by the fixture **/ task createServiceAccountFile() { doLast { @@ -111,7 +107,7 @@ integTest { dependsOn repositoryPlugin.bundlePlugin runner { systemProperty 'test.gcs.bucket', gcsBucket - systemProperty 'test.gcs.base_path', gcsBasePath + "/searchable_snapshots_tests" + systemProperty 'test.gcs.base_path', gcsBasePath + "_searchable_snapshots_tests" } } @@ -136,3 +132,6 @@ testClusters.integTest { setting 'xpack.license.self_generated.type', 'trial' } +task gcsThirdPartyTest { + dependsOn integTest +} diff --git a/x-pack/plugin/searchable-snapshots/qa/s3/build.gradle b/x-pack/plugin/searchable-snapshots/qa/s3/build.gradle index 76a615546ccf2..34e3625e501ed 100644 --- a/x-pack/plugin/searchable-snapshots/qa/s3/build.gradle +++ b/x-pack/plugin/searchable-snapshots/qa/s3/build.gradle @@ -29,7 +29,7 @@ if (!s3AccessKey && !s3SecretKey && !s3Bucket && !s3BasePath) { s3AccessKey = 'access_key' s3SecretKey = 'secret_key' s3Bucket = 'bucket' - s3BasePath = 'base_path' + s3BasePath = null useFixture = true } else if (!s3AccessKey || !s3SecretKey || !s3Bucket || !s3BasePath) { @@ -45,7 +45,7 @@ integTest { dependsOn repositoryPlugin.bundlePlugin runner { systemProperty 'test.s3.bucket', s3Bucket - systemProperty 'test.s3.base_path', s3BasePath + "/searchable_snapshots_tests" + systemProperty 'test.s3.base_path', s3BasePath ? s3BasePath + "_searchable_snapshots_tests" : 'base_path' } } @@ -68,7 +68,6 @@ testClusters.integTest { assert ephemeralPort > 0 '127.0.0.1:' + ephemeralPort } - setting 's3.client.searchable_snapshots.protocol', 'http' setting 's3.client.searchable_snapshots.endpoint', { "${-> fixtureAddress('s3-fixture-other')}" }, IGNORE_VALUE @@ -77,3 +76,6 @@ testClusters.integTest { } } +task s3ThirdPartyTest { + dependsOn integTest +}