From daf3d1a7ba065e358fbd9539bd53639b1bdad473 Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Mon, 6 Apr 2020 19:07:40 -0500 Subject: [PATCH 1/5] initial commit --- .../gradle/plugin/PluginBuildPlugin.groovy | 70 +++++++++---------- .../testclusters/ElasticsearchCluster.java | 16 +++++ .../testclusters/ElasticsearchNode.java | 47 ++++++++++--- .../TestClusterConfiguration.java | 8 +++ docs/build.gradle | 2 +- modules/ingest-common/build.gradle | 2 +- modules/lang-painless/build.gradle | 4 +- modules/rank-eval/build.gradle | 4 +- modules/reindex/build.gradle | 8 +-- .../discovery-ec2/qa/amazon-ec2/build.gradle | 2 +- plugins/discovery-gce/qa/gce/build.gradle | 2 +- .../qa/microsoft-azure-storage/build.gradle | 2 +- .../qa/google-cloud-storage/build.gradle | 2 +- plugins/repository-hdfs/build.gradle | 2 +- plugins/repository-s3/build.gradle | 4 +- qa/smoke-test-plugins/build.gradle | 2 +- .../qa/azure/build.gradle | 2 +- .../searchable-snapshots/qa/s3/build.gradle | 2 +- x-pack/qa/smoke-test-plugins-ssl/build.gradle | 2 +- x-pack/qa/smoke-test-plugins/build.gradle | 2 +- 20 files changed, 115 insertions(+), 70 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy index 1c78b12ffe4ac..ca602e0fcf554 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy @@ -57,37 +57,34 @@ class PluginBuildPlugin implements Plugin { PluginPropertiesExtension extension = project.extensions.create(PLUGIN_EXTENSION_NAME, PluginPropertiesExtension, project) configureDependencies(project) - // this afterEvaluate must happen before the afterEvaluate added by integTest creation, - // so that the file name resolution for installing the plugin will be setup - project.afterEvaluate { - boolean isXPackModule = project.path.startsWith(':x-pack:plugin') - boolean isModule = project.path.startsWith(':modules:') || isXPackModule - PluginPropertiesExtension extension1 = project.getExtensions().getByType(PluginPropertiesExtension.class) - String name = extension1.name - project.archivesBaseName = name - project.description = extension1.description - configurePublishing(project, extension1) + boolean isXPackModule = project.path.startsWith(':x-pack:plugin') + boolean isModule = project.path.startsWith(':modules:') || isXPackModule - project.tasks.integTest.dependsOn(project.tasks.bundlePlugin) - if (isModule) { + createIntegTestTask(project) + createBundleTasks(project, extension) + project.tasks.integTest.dependsOn(project.tasks.bundlePlugin) + if (isModule) { + project.testClusters.integTest.module(project.tasks.bundlePlugin.archiveFile) + } else { + project.testClusters.integTest.plugin(project.tasks.bundlePlugin.archiveFile) + } + + project.extensions.getByType(PluginPropertiesExtension).extendedPlugins.each { pluginName -> + // Auto add dependent modules to the test cluster + if (project.findProject(":modules:${pluginName}") != null) { + project.integTest.dependsOn(project.project(":modules:${pluginName}").tasks.bundlePlugin) project.testClusters.integTest.module( - project.file(project.tasks.bundlePlugin.archiveFile) - ) - } else { - project.testClusters.integTest.plugin( - project.file(project.tasks.bundlePlugin.archiveFile) + project.project(":modules:${pluginName}").tasks.bundlePlugin.archiveFile ) } + } - project.extensions.getByType(PluginPropertiesExtension).extendedPlugins.each { pluginName -> - // Auto add dependent modules to the test cluster - if (project.findProject(":modules:${pluginName}") != null) { - project.integTest.dependsOn(project.project(":modules:${pluginName}").tasks.bundlePlugin) - project.testClusters.integTest.module( - project.file(project.project(":modules:${pluginName}").tasks.bundlePlugin.archiveFile) - ) - } - } + project.afterEvaluate { + PluginPropertiesExtension extension1 = project.getExtensions().getByType(PluginPropertiesExtension.class) + configurePublishing(project, extension1) + String name = extension1.name + project.archivesBaseName = name + project.description = extension1.description if (extension1.name == null) { throw new InvalidUserDataException('name is a required setting for esplugin') @@ -100,15 +97,15 @@ class PluginBuildPlugin implements Plugin { } Copy buildProperties = project.tasks.getByName('pluginProperties') Map properties = [ - 'name' : extension1.name, - 'description' : extension1.description, - 'version' : extension1.version, - 'elasticsearchVersion': Version.fromString(VersionProperties.elasticsearch).toString(), - 'javaVersion' : project.targetCompatibility as String, - 'classname' : extension1.classname, - 'extendedPlugins' : extension1.extendedPlugins.join(','), - 'hasNativeController' : extension1.hasNativeController, - 'requiresKeystore' : extension1.requiresKeystore + 'name' : extension1.name, + 'description' : extension1.description, + 'version' : extension1.version, + 'elasticsearchVersion': Version.fromString(VersionProperties.elasticsearch).toString(), + 'javaVersion' : project.targetCompatibility as String, + 'classname' : extension1.classname, + 'extendedPlugins' : extension1.extendedPlugins.join(','), + 'hasNativeController' : extension1.hasNativeController, + 'requiresKeystore' : extension1.requiresKeystore ] buildProperties.expand(properties) buildProperties.inputs.properties(properties) @@ -116,6 +113,7 @@ class PluginBuildPlugin implements Plugin { addNoticeGeneration(project, extension1) } } + project.tasks.named('testingConventions').configure { naming.clear() naming { @@ -129,8 +127,6 @@ class PluginBuildPlugin implements Plugin { } } } - createIntegTestTask(project) - createBundleTasks(project, extension) project.configurations.getByName('default') .extendsFrom(project.configurations.getByName('runtimeClasspath')) // allow running ES with this plugin in the foreground of a build diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java index 0c700e51972d8..5ce85802ba996 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java @@ -25,8 +25,10 @@ import org.gradle.api.Named; import org.gradle.api.NamedDomainObjectContainer; import org.gradle.api.Project; +import org.gradle.api.file.RegularFile; import org.gradle.api.logging.Logger; import org.gradle.api.logging.Logging; +import org.gradle.api.provider.Provider; import org.gradle.api.tasks.Internal; import org.gradle.api.tasks.Nested; @@ -141,11 +143,25 @@ public void plugin(File plugin) { nodes.all(each -> each.plugin(plugin)); } + @Override + public void plugin(Supplier plugin) { + nodes.all(each -> each.plugin(plugin)); + } + @Override + public void plugin(Provider plugin) { + nodes.all(each -> each.plugin(plugin)); + } + @Override public void module(File module) { nodes.all(each -> each.module(module)); } + @Override + public void module(Provider module) { + nodes.all(each -> each.module(module)); + } + @Override public void keystore(String key, String value) { nodes.all(each -> each.keystore(key, value)); diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java index 81dbf95fae145..fe0a69e51597d 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java @@ -35,8 +35,12 @@ import org.gradle.api.Named; import org.gradle.api.NamedDomainObjectContainer; import org.gradle.api.Project; +import org.gradle.api.file.RegularFile; +import org.gradle.api.file.RegularFileProperty; import org.gradle.api.logging.Logger; import org.gradle.api.logging.Logging; +import org.gradle.api.provider.Provider; +import org.gradle.api.provider.ProviderFactory; import org.gradle.api.tasks.Classpath; import org.gradle.api.tasks.Input; import org.gradle.api.tasks.InputFile; @@ -120,8 +124,8 @@ public class ElasticsearchNode implements TestClusterConfiguration { private final Path workingDir; private final LinkedHashMap> waitConditions = new LinkedHashMap<>(); - private final List plugins = new ArrayList<>(); - private final List modules = new ArrayList<>(); + private final List> plugins = new ArrayList<>(); + private final List> modules = new ArrayList<>(); private final LazyPropertyMap settings = new LazyPropertyMap<>("Settings", this); private final LazyPropertyMap keystoreSettings = new LazyPropertyMap<>("Keystore", this); private final LazyPropertyMap keystoreFiles = new LazyPropertyMap<>("Keystore files", this, FileEntry::new); @@ -258,7 +262,7 @@ private void setDistributionType(ElasticsearchDistribution distribution, TestDis } @Override - public void plugin(URI plugin) { + public void plugin(Supplier plugin) { requireNonNull(plugin, "Plugin name can't be null"); checkFrozen(); if (plugins.contains(plugin)) { @@ -267,13 +271,30 @@ public void plugin(URI plugin) { this.plugins.add(plugin); } + @Override + public void plugin(Provider plugin) { + this.plugin(() -> plugin.get().getAsFile().toURI()); + } + + @Override + public void plugin(URI plugin) { + this.plugin(() -> plugin); + } + @Override public void plugin(File plugin) { - plugin(plugin.toURI()); + this.plugin(() -> plugin.toURI()); } @Override public void module(File module) { + RegularFileProperty file = project.getObjects().fileProperty(); + file.fileValue(module); + this.module(file); + } + + @Override + public void module(Provider module) { this.modules.add(module); } @@ -416,7 +437,8 @@ public synchronized void start() { if (plugins.isEmpty() == false) { if (getVersion().onOrAfter("7.6.0")) { logToProcessStdout("installing " + plugins.size() + " plugins in a single transaction"); - final String[] arguments = Stream.concat(Stream.of("install", "--batch"), plugins.stream().map(URI::toString)) + final String[] arguments = Stream.concat(Stream.of("install", "--batch"), + plugins.stream().map(Supplier::get).map(URI::toString)) .toArray(String[]::new); runElasticsearchBinScript("elasticsearch-plugin", arguments); } else { @@ -552,16 +574,17 @@ private void copyExtraJars() { private void installModules() { if (testDistribution == TestDistribution.INTEG_TEST) { logToProcessStdout("Installing " + modules.size() + "modules"); - for (File module : modules) { + for (Provider module : modules) { + File fileModule = module.get().getAsFile(); Path destination = getDistroDir().resolve("modules") - .resolve(module.getName().replace(".zip", "").replace("-" + getVersion(), "").replace("-SNAPSHOT", "")); + .resolve(fileModule.getName().replace(".zip", "").replace("-" + getVersion(), "").replace("-SNAPSHOT", "")); // only install modules that are not already bundled with the integ-test distribution if (Files.exists(destination) == false) { project.copy(spec -> { - if (module.getName().toLowerCase().endsWith(".zip")) { + if (fileModule.getName().toLowerCase().endsWith(".zip")) { spec.from(project.zipTree(module)); - } else if (module.isDirectory()) { + } else if (fileModule.isDirectory()) { spec.from(module); } else { throw new IllegalArgumentException("Not a valid module " + module + " for " + this); @@ -1105,7 +1128,8 @@ private Path getExtractedDistributionDir() { } private List getInstalledFileSet(Action filter) { - return Stream.concat(plugins.stream().filter(uri -> uri.getScheme().equalsIgnoreCase("file")).map(File::new), modules.stream()) + return Stream.concat(plugins.stream().map(Supplier::get).filter(uri -> uri.getScheme().equalsIgnoreCase("file")).map(File::new), + modules.stream().map(p -> p.get().getAsFile())) .filter(File::exists) // TODO: We may be able to simplify this with Gradle 5.6 // https://docs.gradle.org/nightly/release-notes.html#improved-handling-of-zip-archives-on-classpaths @@ -1117,7 +1141,8 @@ private List getInstalledFileSet(Action filter) @Input public Set getRemotePlugins() { - Set file = plugins.stream().filter(uri -> uri.getScheme().equalsIgnoreCase("file") == false).collect(Collectors.toSet()); + Set file = plugins.stream().map(Supplier::get).filter(uri -> uri.getScheme().equalsIgnoreCase("file") == false) + .collect(Collectors.toSet()); return file; } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java index dd3efb1ac1089..371a4939f2e0b 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java @@ -20,7 +20,9 @@ import org.elasticsearch.gradle.FileSupplier; import org.elasticsearch.gradle.PropertyNormalization; +import org.gradle.api.file.RegularFile; import org.gradle.api.logging.Logging; +import org.gradle.api.provider.Provider; import org.slf4j.Logger; import java.io.File; @@ -46,8 +48,14 @@ public interface TestClusterConfiguration { void plugin(File plugin); + void plugin(Supplier plugin); + + void plugin(Provider plugin); + void module(File module); + void module(Provider module); + void keystore(String key, String value); void keystore(String key, Supplier valueSupplier); diff --git a/docs/build.gradle b/docs/build.gradle index 5efceb14459ee..ea351fa848fff 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -96,7 +96,7 @@ project.rootProject.subprojects.findAll { it.parent.path == ':plugins' }.each { // FIXME subproj.afterEvaluate { // need to wait until the project has been configured testClusters.integTest { - plugin file(subproj.bundlePlugin.archiveFile) + plugin subproj.bundlePlugin.archiveFile } tasks.integTest.dependsOn subproj.bundlePlugin } diff --git a/modules/ingest-common/build.gradle b/modules/ingest-common/build.gradle index 2ff50f1543ed9..605096e26cd02 100644 --- a/modules/ingest-common/build.gradle +++ b/modules/ingest-common/build.gradle @@ -38,5 +38,5 @@ restResources { testClusters.integTest { // Needed in order to test ingest pipeline templating: // (this is because the integTest node is not using default distribution, but only the minimal number of required modules) - module file(project(':modules:lang-mustache').tasks.bundlePlugin.archiveFile) + module project(':modules:lang-mustache').tasks.bundlePlugin.archiveFile } diff --git a/modules/lang-painless/build.gradle b/modules/lang-painless/build.gradle index 68a990a3dda9a..04533194d7cb5 100644 --- a/modules/lang-painless/build.gradle +++ b/modules/lang-painless/build.gradle @@ -24,7 +24,7 @@ esplugin { } testClusters.integTest { - module file(project(':modules:mapper-extras').tasks.bundlePlugin.archiveFile) + module project(':modules:mapper-extras').tasks.bundlePlugin.archiveFile systemProperty 'es.scripting.update.ctx_in_params', 'false' // TODO: remove this once cname is prepended to transport.publish_address by default in 8.0 systemProperty 'es.transport.cname_in_publish_address', 'true' @@ -46,7 +46,7 @@ dependencyLicenses { restResources { restApi { - includeCore '_common', 'cluster', 'nodes', 'indices', 'index', 'search', 'get', 'bulk', 'update', + includeCore '_common', 'cluster', 'nodes', 'indices', 'index', 'search', 'get', 'bulk', 'update', 'scripts_painless_execute', 'put_script', 'delete_script' } } diff --git a/modules/rank-eval/build.gradle b/modules/rank-eval/build.gradle index 1adcd3016e50d..6961f3e5ba8f9 100644 --- a/modules/rank-eval/build.gradle +++ b/modules/rank-eval/build.gradle @@ -24,11 +24,11 @@ esplugin { restResources { restApi { - includeCore '_common', 'indices', 'index', 'rank_eval' + includeCore '_common', 'indices', 'index', 'rank_eval' } } testClusters.integTest { // Modules who's integration is explicitly tested in integration tests - module file(project(':modules:lang-mustache').tasks.bundlePlugin.archiveFile) + module project(':modules:lang-mustache').tasks.bundlePlugin.archiveFile } diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle index a6b60166c343f..5588af0f44b71 100644 --- a/modules/reindex/build.gradle +++ b/modules/reindex/build.gradle @@ -31,8 +31,8 @@ esplugin { testClusters.integTest { // Modules who's integration is explicitly tested in integration tests - module file(project(':modules:parent-join').tasks.bundlePlugin.archiveFile) - module file(project(':modules:lang-painless').tasks.bundlePlugin.archiveFile) + module project(':modules:parent-join').tasks.bundlePlugin.archiveFile + module project(':modules:lang-painless').tasks.bundlePlugin.archiveFile // Whitelist reindexing from the local node so we can test reindex-from-remote. setting 'reindex.remote.whitelist', '127.0.0.1:*' } @@ -57,8 +57,8 @@ dependencies { restResources { restApi { - includeCore '_common', 'cluster', 'nodes', 'indices', 'index', 'get', 'search', 'mget', 'count', - 'update_by_query', 'delete_by_query', 'reindex_rethrottle', 'tasks', 'reindex', 'put_script' + includeCore '_common', 'cluster', 'nodes', 'indices', 'index', 'get', 'search', 'mget', 'count', + 'update_by_query', 'delete_by_query', 'reindex_rethrottle', 'tasks', 'reindex', 'put_script' } } diff --git a/plugins/discovery-ec2/qa/amazon-ec2/build.gradle b/plugins/discovery-ec2/qa/amazon-ec2/build.gradle index a10d9feccfdce..f2a940412d6d5 100644 --- a/plugins/discovery-ec2/qa/amazon-ec2/build.gradle +++ b/plugins/discovery-ec2/qa/amazon-ec2/build.gradle @@ -79,7 +79,7 @@ integTest.enabled = false testClusters."integTest${action}" { numberOfNodes = ec2NumberOfNodes - plugin file(project(':plugins:discovery-ec2').bundlePlugin.archiveFile) + plugin project(':plugins:discovery-ec2').bundlePlugin.archiveFile setting 'discovery.seed_providers', 'ec2' setting 'network.host', '_ec2_' diff --git a/plugins/discovery-gce/qa/gce/build.gradle b/plugins/discovery-gce/qa/gce/build.gradle index 7eb2fe13aae8f..515530873df9f 100644 --- a/plugins/discovery-gce/qa/gce/build.gradle +++ b/plugins/discovery-gce/qa/gce/build.gradle @@ -62,7 +62,7 @@ integTest { testClusters.integTest { numberOfNodes = gceNumberOfNodes - plugin file(project(':plugins:discovery-gce').bundlePlugin.archiveFile) + plugin project(':plugins:discovery-gce').bundlePlugin.archiveFile // use gce fixture for Auth calls instead of http://metadata.google.internal environment 'GCE_METADATA_HOST', { "http://${gceFixture.addressAndPort}" }, IGNORE_VALUE // allows to configure hidden settings (`cloud.gce.host` and `cloud.gce.root_url`) diff --git a/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle b/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle index b594ff3b172f2..3c2146095e651 100644 --- a/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle +++ b/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle @@ -69,7 +69,7 @@ integTest { } testClusters.integTest { - plugin file(project(':plugins:repository-azure').bundlePlugin.archiveFile) + plugin project(':plugins:repository-azure').bundlePlugin.archiveFile keystore 'azure.client.integration_test.account', azureAccount if (azureKey != null && azureKey.isEmpty() == false) { keystore 'azure.client.integration_test.key', azureKey diff --git a/plugins/repository-gcs/qa/google-cloud-storage/build.gradle b/plugins/repository-gcs/qa/google-cloud-storage/build.gradle index 10105d2f2348c..d70c1921908f5 100644 --- a/plugins/repository-gcs/qa/google-cloud-storage/build.gradle +++ b/plugins/repository-gcs/qa/google-cloud-storage/build.gradle @@ -135,7 +135,7 @@ integTest { } testClusters.integTest { - plugin file(project(':plugins:repository-gcs').bundlePlugin.archiveFile) + plugin project(':plugins:repository-gcs').bundlePlugin.archiveFile keystore 'gcs.client.integration_test.credentials_file', serviceAccountFile, IGNORE_VALUE diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index 0963fccde9ef3..d0935cfb6cc82 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -198,7 +198,7 @@ for (String integTestTaskName : ['integTestHa', 'integTestSecure', 'integTestSec } testClusters."${integTestTaskName}" { - plugin(file(bundlePlugin.archiveFile)) + plugin(bundlePlugin.archiveFile) if (integTestTaskName.contains("Secure")) { systemProperty "java.security.krb5.conf", krb5conf extraConfigFile( diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 11c1bfb584357..f8bcf2e0f54db 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -179,7 +179,7 @@ if (useFixture) { keystore 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey keystore 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey setting 's3.client.integration_test_permanent.endpoint', minioAddress, IGNORE_VALUE - plugin file(tasks.bundlePlugin.archiveFile) + plugin tasks.bundlePlugin.archiveFile } integTest.runner { @@ -269,7 +269,7 @@ if (useFixture) { testClusters.integTestECS { setting 's3.client.integration_test_ecs.endpoint', { "${-> fixtureAddress('s3-fixture-with-ecs')}" }, IGNORE_VALUE - plugin file(tasks.bundlePlugin.archiveFile) + plugin tasks.bundlePlugin.archiveFile environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI', { "${-> fixtureAddress('s3-fixture-with-ecs')}/ecs_credentials_endpoint" }, IGNORE_VALUE } diff --git a/qa/smoke-test-plugins/build.gradle b/qa/smoke-test-plugins/build.gradle index fef52b777b9a8..4485c0f30940f 100644 --- a/qa/smoke-test-plugins/build.gradle +++ b/qa/smoke-test-plugins/build.gradle @@ -32,7 +32,7 @@ testClusters.integTest { // Do not attempt to install ingest-attachment in FIPS 140 as it is not supported (it depends on non-FIPS BouncyCastle) return } - plugin file(pluginProject.tasks.bundlePlugin.archiveFile) + plugin pluginProject.tasks.bundlePlugin.archiveFile tasks.integTest.dependsOn pluginProject.tasks.bundlePlugin pluginsCount += 1 } diff --git a/x-pack/plugin/searchable-snapshots/qa/azure/build.gradle b/x-pack/plugin/searchable-snapshots/qa/azure/build.gradle index bd5eaa8336daf..fa89352321b5c 100644 --- a/x-pack/plugin/searchable-snapshots/qa/azure/build.gradle +++ b/x-pack/plugin/searchable-snapshots/qa/azure/build.gradle @@ -71,7 +71,7 @@ integTest { testClusters.integTest { testDistribution = 'DEFAULT' - plugin file(repositoryPlugin.bundlePlugin.archiveFile) + plugin repositoryPlugin.bundlePlugin.archiveFile keystore 'azure.client.searchable_snapshots.account', azureAccount keystore 'azure.client.searchable_snapshots.key', azureKey diff --git a/x-pack/plugin/searchable-snapshots/qa/s3/build.gradle b/x-pack/plugin/searchable-snapshots/qa/s3/build.gradle index 029aa8837d022..76a615546ccf2 100644 --- a/x-pack/plugin/searchable-snapshots/qa/s3/build.gradle +++ b/x-pack/plugin/searchable-snapshots/qa/s3/build.gradle @@ -51,7 +51,7 @@ integTest { testClusters.integTest { testDistribution = 'DEFAULT' - plugin file(repositoryPlugin.bundlePlugin.archiveFile) + plugin repositoryPlugin.bundlePlugin.archiveFile if (BuildParams.isSnapshotBuild() == false) { systemProperty 'es.searchable_snapshots_feature_enabled', 'true' diff --git a/x-pack/qa/smoke-test-plugins-ssl/build.gradle b/x-pack/qa/smoke-test-plugins-ssl/build.gradle index e41de2ca82171..35ff78375ec74 100644 --- a/x-pack/qa/smoke-test-plugins-ssl/build.gradle +++ b/x-pack/qa/smoke-test-plugins-ssl/build.gradle @@ -73,7 +73,7 @@ testClusters.integTest { // Do not attempt to install ingest-attachment in FIPS 140 as it is not supported (it depends on non-FIPS BouncyCastle) return } - plugin file(pluginProject.tasks.bundlePlugin.archiveFile) + plugin pluginProject.tasks.bundlePlugin.archiveFile tasks.integTest.dependsOn pluginProject.tasks.bundlePlugin pluginsCount += 1 } diff --git a/x-pack/qa/smoke-test-plugins/build.gradle b/x-pack/qa/smoke-test-plugins/build.gradle index 3a14103b333b3..f884378c762f0 100644 --- a/x-pack/qa/smoke-test-plugins/build.gradle +++ b/x-pack/qa/smoke-test-plugins/build.gradle @@ -20,7 +20,7 @@ testClusters.integTest { // Do not attempt to install ingest-attachment in FIPS 140 as it is not supported (it depends on non-FIPS BouncyCastle) return } - plugin file(pluginProject.tasks.bundlePlugin.archiveFile) + plugin pluginProject.tasks.bundlePlugin.archiveFile tasks.integTest.dependsOn pluginProject.tasks.bundlePlugin pluginsCount += 1 } From 95cff57223b256ed01ecc4e55757c782d4f6aaaf Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Mon, 6 Apr 2020 19:37:09 -0500 Subject: [PATCH 2/5] clean up --- .../testclusters/ElasticsearchCluster.java | 5 +++-- .../gradle/testclusters/ElasticsearchNode.java | 18 +++++++++++------- .../testclusters/TestClusterConfiguration.java | 2 +- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java index 5ce85802ba996..d30005c982fd1 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java @@ -144,11 +144,12 @@ public void plugin(File plugin) { } @Override - public void plugin(Supplier plugin) { + public void plugin(Supplier plugin) { nodes.all(each -> each.plugin(plugin)); } + @Override - public void plugin(Provider plugin) { + public void plugin(Provider plugin) { nodes.all(each -> each.plugin(plugin)); } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java index fe0a69e51597d..c746989fbe49f 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java @@ -40,7 +40,6 @@ import org.gradle.api.logging.Logger; import org.gradle.api.logging.Logging; import org.gradle.api.provider.Provider; -import org.gradle.api.provider.ProviderFactory; import org.gradle.api.tasks.Classpath; import org.gradle.api.tasks.Input; import org.gradle.api.tasks.InputFile; @@ -437,9 +436,10 @@ public synchronized void start() { if (plugins.isEmpty() == false) { if (getVersion().onOrAfter("7.6.0")) { logToProcessStdout("installing " + plugins.size() + " plugins in a single transaction"); - final String[] arguments = Stream.concat(Stream.of("install", "--batch"), - plugins.stream().map(Supplier::get).map(URI::toString)) - .toArray(String[]::new); + final String[] arguments = Stream.concat( + Stream.of("install", "--batch"), + plugins.stream().map(Supplier::get).map(URI::toString) + ).toArray(String[]::new); runElasticsearchBinScript("elasticsearch-plugin", arguments); } else { logToProcessStdout("installing " + plugins.size() + " plugins sequentially"); @@ -1128,8 +1128,10 @@ private Path getExtractedDistributionDir() { } private List getInstalledFileSet(Action filter) { - return Stream.concat(plugins.stream().map(Supplier::get).filter(uri -> uri.getScheme().equalsIgnoreCase("file")).map(File::new), - modules.stream().map(p -> p.get().getAsFile())) + return Stream.concat( + plugins.stream().map(Supplier::get).filter(uri -> uri.getScheme().equalsIgnoreCase("file")).map(File::new), + modules.stream().map(p -> p.get().getAsFile()) + ) .filter(File::exists) // TODO: We may be able to simplify this with Gradle 5.6 // https://docs.gradle.org/nightly/release-notes.html#improved-handling-of-zip-archives-on-classpaths @@ -1141,7 +1143,9 @@ private List getInstalledFileSet(Action filter) @Input public Set getRemotePlugins() { - Set file = plugins.stream().map(Supplier::get).filter(uri -> uri.getScheme().equalsIgnoreCase("file") == false) + Set file = plugins.stream() + .map(Supplier::get) + .filter(uri -> uri.getScheme().equalsIgnoreCase("file") == false) .collect(Collectors.toSet()); return file; } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java index 371a4939f2e0b..02155377fb0f2 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java @@ -50,7 +50,7 @@ public interface TestClusterConfiguration { void plugin(Supplier plugin); - void plugin(Provider plugin); + void plugin(Provider plugin); void module(File module); From b3068d11c6975222ee200c2857c0a4ab96c9b126 Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Mon, 6 Apr 2020 20:14:01 -0500 Subject: [PATCH 3/5] move plugin dependency back to afterEvaluate --- .../gradle/plugin/PluginBuildPlugin.groovy | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy index ca602e0fcf554..e0af55820f432 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy @@ -69,17 +69,16 @@ class PluginBuildPlugin implements Plugin { project.testClusters.integTest.plugin(project.tasks.bundlePlugin.archiveFile) } - project.extensions.getByType(PluginPropertiesExtension).extendedPlugins.each { pluginName -> - // Auto add dependent modules to the test cluster - if (project.findProject(":modules:${pluginName}") != null) { - project.integTest.dependsOn(project.project(":modules:${pluginName}").tasks.bundlePlugin) - project.testClusters.integTest.module( - project.project(":modules:${pluginName}").tasks.bundlePlugin.archiveFile - ) - } - } - project.afterEvaluate { + project.extensions.getByType(PluginPropertiesExtension).extendedPlugins.each { pluginName -> + // Auto add dependent modules to the test cluster + if (project.findProject(":modules:${pluginName}") != null) { + project.integTest.dependsOn(project.project(":modules:${pluginName}").tasks.bundlePlugin) + project.testClusters.integTest.module( + project.project(":modules:${pluginName}").tasks.bundlePlugin.archiveFile + ) + } + } PluginPropertiesExtension extension1 = project.getExtensions().getByType(PluginPropertiesExtension.class) configurePublishing(project, extension1) String name = extension1.name From 73427ab3692e08fd17b78d5a5c53490f2dc7b8cc Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Wed, 8 Apr 2020 10:54:16 -0500 Subject: [PATCH 4/5] review changes --- .../testclusters/ElasticsearchCluster.java | 7 +--- .../testclusters/ElasticsearchNode.java | 40 +++++++++---------- .../TestClusterConfiguration.java | 4 +- 3 files changed, 21 insertions(+), 30 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java index d30005c982fd1..65f8b71b7ef79 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java @@ -144,12 +144,7 @@ public void plugin(File plugin) { } @Override - public void plugin(Supplier plugin) { - nodes.all(each -> each.plugin(plugin)); - } - - @Override - public void plugin(Provider plugin) { + public void plugin(Provider plugin) { nodes.all(each -> each.plugin(plugin)); } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java index c746989fbe49f..e2fd8586b2775 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java @@ -39,6 +39,7 @@ import org.gradle.api.file.RegularFileProperty; import org.gradle.api.logging.Logger; import org.gradle.api.logging.Logging; +import org.gradle.api.provider.Property; import org.gradle.api.provider.Provider; import org.gradle.api.tasks.Classpath; import org.gradle.api.tasks.Input; @@ -123,8 +124,8 @@ public class ElasticsearchNode implements TestClusterConfiguration { private final Path workingDir; private final LinkedHashMap> waitConditions = new LinkedHashMap<>(); - private final List> plugins = new ArrayList<>(); - private final List> modules = new ArrayList<>(); + private final List> plugins = new ArrayList<>(); + private final List> modules = new ArrayList<>(); private final LazyPropertyMap settings = new LazyPropertyMap<>("Settings", this); private final LazyPropertyMap keystoreSettings = new LazyPropertyMap<>("Keystore", this); private final LazyPropertyMap keystoreFiles = new LazyPropertyMap<>("Keystore files", this, FileEntry::new); @@ -261,7 +262,7 @@ private void setDistributionType(ElasticsearchDistribution distribution, TestDis } @Override - public void plugin(Supplier plugin) { + public void plugin(Provider plugin) { requireNonNull(plugin, "Plugin name can't be null"); checkFrozen(); if (plugins.contains(plugin)) { @@ -270,19 +271,18 @@ public void plugin(Supplier plugin) { this.plugins.add(plugin); } - @Override - public void plugin(Provider plugin) { - this.plugin(() -> plugin.get().getAsFile().toURI()); - } - @Override public void plugin(URI plugin) { - this.plugin(() -> plugin); + Property uri = project.getObjects().property(URI.class); + uri.set(plugin); + this.plugin(uri); } @Override public void plugin(File plugin) { - this.plugin(() -> plugin.toURI()); + Property uri = project.getObjects().property(URI.class); + uri.set(plugin.toURI()); + this.plugin(uri); } @Override @@ -294,7 +294,7 @@ public void module(File module) { @Override public void module(Provider module) { - this.modules.add(module); + this.modules.add(module.map(RegularFile::getAsFile)); } @Override @@ -438,7 +438,7 @@ public synchronized void start() { logToProcessStdout("installing " + plugins.size() + " plugins in a single transaction"); final String[] arguments = Stream.concat( Stream.of("install", "--batch"), - plugins.stream().map(Supplier::get).map(URI::toString) + plugins.stream().map(Provider::get).map(URI::toString) ).toArray(String[]::new); runElasticsearchBinScript("elasticsearch-plugin", arguments); } else { @@ -574,17 +574,15 @@ private void copyExtraJars() { private void installModules() { if (testDistribution == TestDistribution.INTEG_TEST) { logToProcessStdout("Installing " + modules.size() + "modules"); - for (Provider module : modules) { - File fileModule = module.get().getAsFile(); + for (Provider module : modules) { Path destination = getDistroDir().resolve("modules") - .resolve(fileModule.getName().replace(".zip", "").replace("-" + getVersion(), "").replace("-SNAPSHOT", "")); - + .resolve(module.get().getName().replace(".zip", "").replace("-" + getVersion(), "").replace("-SNAPSHOT", "")); // only install modules that are not already bundled with the integ-test distribution if (Files.exists(destination) == false) { project.copy(spec -> { - if (fileModule.getName().toLowerCase().endsWith(".zip")) { + if (module.get().getName().toLowerCase().endsWith(".zip")) { spec.from(project.zipTree(module)); - } else if (fileModule.isDirectory()) { + } else if (module.get().isDirectory()) { spec.from(module); } else { throw new IllegalArgumentException("Not a valid module " + module + " for " + this); @@ -1129,8 +1127,8 @@ private Path getExtractedDistributionDir() { private List getInstalledFileSet(Action filter) { return Stream.concat( - plugins.stream().map(Supplier::get).filter(uri -> uri.getScheme().equalsIgnoreCase("file")).map(File::new), - modules.stream().map(p -> p.get().getAsFile()) + plugins.stream().map(Provider::get).filter(uri -> uri.getScheme().equalsIgnoreCase("file")).map(File::new), + modules.stream().map(p -> p.get()) ) .filter(File::exists) // TODO: We may be able to simplify this with Gradle 5.6 @@ -1144,7 +1142,7 @@ private List getInstalledFileSet(Action filter) @Input public Set getRemotePlugins() { Set file = plugins.stream() - .map(Supplier::get) + .map(Provider::get) .filter(uri -> uri.getScheme().equalsIgnoreCase("file") == false) .collect(Collectors.toSet()); return file; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java index 02155377fb0f2..c879e927dfb44 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java @@ -48,9 +48,7 @@ public interface TestClusterConfiguration { void plugin(File plugin); - void plugin(Supplier plugin); - - void plugin(Provider plugin); + void plugin(Provider plugin); void module(File module); From 909f0f3e5e9adf5a64b0528ac70b3d3da094cb21 Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Wed, 8 Apr 2020 13:36:08 -0500 Subject: [PATCH 5/5] add missing lazy File based plugin option --- .../gradle/testclusters/ElasticsearchCluster.java | 6 ++++++ .../gradle/testclusters/ElasticsearchNode.java | 5 +++++ .../gradle/testclusters/TestClusterConfiguration.java | 3 +++ 3 files changed, 14 insertions(+) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java index 65f8b71b7ef79..7e4cc43037569 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java @@ -26,6 +26,7 @@ import org.gradle.api.NamedDomainObjectContainer; import org.gradle.api.Project; import org.gradle.api.file.RegularFile; +import org.gradle.api.file.RegularFileProperty; import org.gradle.api.logging.Logger; import org.gradle.api.logging.Logging; import org.gradle.api.provider.Provider; @@ -148,6 +149,11 @@ public void plugin(Provider plugin) { nodes.all(each -> each.plugin(plugin)); } + @Override + public void plugin(RegularFileProperty plugin) { + nodes.all(each -> each.plugin(plugin)); + } + @Override public void module(File module) { nodes.all(each -> each.module(module)); diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java index e2fd8586b2775..a29a9ce213504 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java @@ -261,6 +261,11 @@ private void setDistributionType(ElasticsearchDistribution distribution, TestDis } } + @Override + public void plugin(RegularFileProperty plugin) { + this.plugins.add(plugin.map(p -> p.getAsFile().toURI())); + } + @Override public void plugin(Provider plugin) { requireNonNull(plugin, "Plugin name can't be null"); diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java index c879e927dfb44..76389c14157cc 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java @@ -21,6 +21,7 @@ import org.elasticsearch.gradle.FileSupplier; import org.elasticsearch.gradle.PropertyNormalization; import org.gradle.api.file.RegularFile; +import org.gradle.api.file.RegularFileProperty; import org.gradle.api.logging.Logging; import org.gradle.api.provider.Provider; import org.slf4j.Logger; @@ -50,6 +51,8 @@ public interface TestClusterConfiguration { void plugin(Provider plugin); + void plugin(RegularFileProperty plugin); + void module(File module); void module(Provider module);