diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy index c738cc3665bc5..ac7d392fde8e7 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy @@ -61,37 +61,33 @@ class PluginBuildPlugin implements Plugin { PluginPropertiesExtension extension = project.extensions.create(PLUGIN_EXTENSION_NAME, PluginPropertiesExtension, project) configureDependencies(project) - // this afterEvaluate must happen before the afterEvaluate added by integTest creation, - // so that the file name resolution for installing the plugin will be setup - project.afterEvaluate { - boolean isXPackModule = project.path.startsWith(':x-pack:plugin') - boolean isModule = project.path.startsWith(':modules:') || isXPackModule - PluginPropertiesExtension extension1 = project.getExtensions().getByType(PluginPropertiesExtension.class) - String name = extension1.name - project.archivesBaseName = name - project.description = extension1.description - configurePublishing(project, extension1) + boolean isXPackModule = project.path.startsWith(':x-pack:plugin') + boolean isModule = project.path.startsWith(':modules:') || isXPackModule - project.tasks.integTest.dependsOn(project.tasks.bundlePlugin) - if (isModule) { - project.testClusters.integTest.module( - project.file(project.tasks.bundlePlugin.archiveFile) - ) - } else { - project.testClusters.integTest.plugin( - project.file(project.tasks.bundlePlugin.archiveFile) - ) - } + createIntegTestTask(project) + createBundleTasks(project, extension) + project.tasks.integTest.dependsOn(project.tasks.bundlePlugin) + if (isModule) { + project.testClusters.integTest.module(project.tasks.bundlePlugin.archiveFile) + } else { + project.testClusters.integTest.plugin(project.tasks.bundlePlugin.archiveFile) + } + project.afterEvaluate { project.extensions.getByType(PluginPropertiesExtension).extendedPlugins.each { pluginName -> // Auto add dependent modules to the test cluster if (project.findProject(":modules:${pluginName}") != null) { project.integTest.dependsOn(project.project(":modules:${pluginName}").tasks.bundlePlugin) project.testClusters.integTest.module( - project.file(project.project(":modules:${pluginName}").tasks.bundlePlugin.archiveFile) + project.project(":modules:${pluginName}").tasks.bundlePlugin.archiveFile ) } } + PluginPropertiesExtension extension1 = project.getExtensions().getByType(PluginPropertiesExtension.class) + configurePublishing(project, extension1) + String name = extension1.name + project.archivesBaseName = name + project.description = extension1.description if (extension1.name == null) { throw new InvalidUserDataException('name is a required setting for esplugin') @@ -104,15 +100,15 @@ class PluginBuildPlugin implements Plugin { } Map properties = [ - 'name' : extension1.name, - 'description' : extension1.description, - 'version' : extension1.version, - 'elasticsearchVersion': Version.fromString(VersionProperties.elasticsearch).toString(), - 'javaVersion' : project.targetCompatibility as String, - 'classname' : extension1.classname, - 'extendedPlugins' : extension1.extendedPlugins.join(','), - 'hasNativeController' : extension1.hasNativeController, - 'requiresKeystore' : extension1.requiresKeystore + 'name' : extension1.name, + 'description' : extension1.description, + 'version' : extension1.version, + 'elasticsearchVersion': Version.fromString(VersionProperties.elasticsearch).toString(), + 'javaVersion' : project.targetCompatibility as String, + 'classname' : extension1.classname, + 'extendedPlugins' : extension1.extendedPlugins.join(','), + 'hasNativeController' : extension1.hasNativeController, + 'requiresKeystore' : extension1.requiresKeystore ] project.tasks.named('pluginProperties').configure { expand(properties) @@ -122,6 +118,7 @@ class PluginBuildPlugin implements Plugin { addNoticeGeneration(project, extension1) } } + project.tasks.named('testingConventions').configure { naming.clear() naming { @@ -135,9 +132,8 @@ class PluginBuildPlugin implements Plugin { } } } - createIntegTestTask(project) - createBundleTasks(project, extension) - project.configurations.getByName('default').extendsFrom(project.configurations.getByName('runtime')) + project.configurations.getByName('default') + .extendsFrom(project.configurations.getByName('runtimeClasspath')) // allow running ES with this plugin in the foreground of a build project.tasks.register('run', RunTask) { dependsOn(project.tasks.bundlePlugin) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java index 71c9aa0b62771..00ef6ea57e074 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java @@ -26,8 +26,11 @@ import org.gradle.api.Named; import org.gradle.api.NamedDomainObjectContainer; import org.gradle.api.Project; +import org.gradle.api.file.RegularFile; +import org.gradle.api.file.RegularFileProperty; import org.gradle.api.logging.Logger; import org.gradle.api.logging.Logging; +import org.gradle.api.provider.Provider; import org.gradle.api.tasks.Internal; import org.gradle.api.tasks.Nested; @@ -144,11 +147,26 @@ public void plugin(File plugin) { nodes.all(each -> each.plugin(plugin)); } + @Override + public void plugin(Provider plugin) { + nodes.all(each -> each.plugin(plugin)); + } + + @Override + public void plugin(RegularFileProperty plugin) { + nodes.all(each -> each.plugin(plugin)); + } + @Override public void module(File module) { nodes.all(each -> each.module(module)); } + @Override + public void module(Provider module) { + nodes.all(each -> each.module(module)); + } + @Override public void keystore(String key, String value) { nodes.all(each -> each.keystore(key, value)); diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java index 049fe7797e26f..2451d38cf0ada 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java @@ -36,8 +36,12 @@ import org.gradle.api.Named; import org.gradle.api.NamedDomainObjectContainer; import org.gradle.api.Project; +import org.gradle.api.file.RegularFile; +import org.gradle.api.file.RegularFileProperty; import org.gradle.api.logging.Logger; import org.gradle.api.logging.Logging; +import org.gradle.api.provider.Property; +import org.gradle.api.provider.Provider; import org.gradle.api.tasks.Classpath; import org.gradle.api.tasks.Input; import org.gradle.api.tasks.InputFile; @@ -122,8 +126,8 @@ public class ElasticsearchNode implements TestClusterConfiguration { private final Path workingDir; private final LinkedHashMap> waitConditions = new LinkedHashMap<>(); - private final List plugins = new ArrayList<>(); - private final List modules = new ArrayList<>(); + private final List> plugins = new ArrayList<>(); + private final List> modules = new ArrayList<>(); final LazyPropertyMap settings = new LazyPropertyMap<>("Settings", this); private final LazyPropertyMap keystoreSettings = new LazyPropertyMap<>("Keystore", this); private final LazyPropertyMap keystoreFiles = new LazyPropertyMap<>("Keystore files", this, FileEntry::new); @@ -264,7 +268,12 @@ private void setDistributionType(ElasticsearchDistribution distribution, TestDis } @Override - public void plugin(URI plugin) { + public void plugin(RegularFileProperty plugin) { + this.plugins.add(plugin.map(p -> p.getAsFile().toURI())); + } + + @Override + public void plugin(Provider plugin) { requireNonNull(plugin, "Plugin name can't be null"); checkFrozen(); if (plugins.contains(plugin)) { @@ -273,14 +282,30 @@ public void plugin(URI plugin) { this.plugins.add(plugin); } + @Override + public void plugin(URI plugin) { + Property uri = project.getObjects().property(URI.class); + uri.set(plugin); + this.plugin(uri); + } + @Override public void plugin(File plugin) { - plugin(plugin.toURI()); + Property uri = project.getObjects().property(URI.class); + uri.set(plugin.toURI()); + this.plugin(uri); } @Override public void module(File module) { - this.modules.add(module); + RegularFileProperty file = project.getObjects().fileProperty(); + file.fileValue(module); + this.module(file); + } + + @Override + public void module(Provider module) { + this.modules.add(module.map(RegularFile::getAsFile)); } @Override @@ -426,7 +451,7 @@ public synchronized void start() { final List pluginsToInstall = new ArrayList<>(); if (plugins.isEmpty() == false) { - pluginsToInstall.addAll(plugins.stream().map(URI::toString).collect(Collectors.toList())); + pluginsToInstall.addAll(plugins.stream().map(Provider::get).map(URI::toString).collect(Collectors.toList())); } if (getVersion().before("6.3.0") && testDistribution == TestDistribution.DEFAULT) { @@ -577,16 +602,15 @@ private void copyExtraJars() { private void installModules() { if (testDistribution == TestDistribution.INTEG_TEST) { logToProcessStdout("Installing " + modules.size() + "modules"); - for (File module : modules) { + for (Provider module : modules) { Path destination = getDistroDir().resolve("modules") - .resolve(module.getName().replace(".zip", "").replace("-" + getVersion(), "").replace("-SNAPSHOT", "")); - + .resolve(module.get().getName().replace(".zip", "").replace("-" + getVersion(), "").replace("-SNAPSHOT", "")); // only install modules that are not already bundled with the integ-test distribution if (Files.exists(destination) == false) { project.copy(spec -> { - if (module.getName().toLowerCase().endsWith(".zip")) { + if (module.get().getName().toLowerCase().endsWith(".zip")) { spec.from(project.zipTree(module)); - } else if (module.isDirectory()) { + } else if (module.get().isDirectory()) { spec.from(module); } else { throw new IllegalArgumentException("Not a valid module " + module + " for " + this); @@ -1155,7 +1179,10 @@ private Path getExtractedDistributionDir() { } private List getInstalledFileSet(Action filter) { - return Stream.concat(plugins.stream().filter(uri -> uri.getScheme().equalsIgnoreCase("file")).map(File::new), modules.stream()) + return Stream.concat( + plugins.stream().map(Provider::get).filter(uri -> uri.getScheme().equalsIgnoreCase("file")).map(File::new), + modules.stream().map(p -> p.get()) + ) .filter(File::exists) // TODO: We may be able to simplify this with Gradle 5.6 // https://docs.gradle.org/nightly/release-notes.html#improved-handling-of-zip-archives-on-classpaths @@ -1167,7 +1194,10 @@ private List getInstalledFileSet(Action filter) @Input public Set getRemotePlugins() { - Set file = plugins.stream().filter(uri -> uri.getScheme().equalsIgnoreCase("file") == false).collect(Collectors.toSet()); + Set file = plugins.stream() + .map(Provider::get) + .filter(uri -> uri.getScheme().equalsIgnoreCase("file") == false) + .collect(Collectors.toSet()); return file; } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java index 01acae2c050bd..3ffe63b0c6373 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java @@ -20,7 +20,10 @@ import org.elasticsearch.gradle.FileSupplier; import org.elasticsearch.gradle.PropertyNormalization; +import org.gradle.api.file.RegularFile; +import org.gradle.api.file.RegularFileProperty; import org.gradle.api.logging.Logging; +import org.gradle.api.provider.Provider; import org.slf4j.Logger; import java.io.File; @@ -45,8 +48,14 @@ public interface TestClusterConfiguration { void plugin(File plugin); + void plugin(Provider plugin); + + void plugin(RegularFileProperty plugin); + void module(File module); + void module(Provider module); + void keystore(String key, String value); void keystore(String key, Supplier valueSupplier); diff --git a/docs/build.gradle b/docs/build.gradle index 96d7f1613e024..4fec4749f3746 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -96,7 +96,7 @@ project.rootProject.subprojects.findAll { it.parent.path == ':plugins' }.each { // FIXME subproj.afterEvaluate { // need to wait until the project has been configured testClusters.integTest { - plugin file(subproj.bundlePlugin.archiveFile) + plugin subproj.bundlePlugin.archiveFile } tasks.integTest.dependsOn subproj.bundlePlugin } diff --git a/modules/ingest-common/build.gradle b/modules/ingest-common/build.gradle index 2ff50f1543ed9..605096e26cd02 100644 --- a/modules/ingest-common/build.gradle +++ b/modules/ingest-common/build.gradle @@ -38,5 +38,5 @@ restResources { testClusters.integTest { // Needed in order to test ingest pipeline templating: // (this is because the integTest node is not using default distribution, but only the minimal number of required modules) - module file(project(':modules:lang-mustache').tasks.bundlePlugin.archiveFile) + module project(':modules:lang-mustache').tasks.bundlePlugin.archiveFile } diff --git a/modules/lang-painless/build.gradle b/modules/lang-painless/build.gradle index 68a990a3dda9a..04533194d7cb5 100644 --- a/modules/lang-painless/build.gradle +++ b/modules/lang-painless/build.gradle @@ -24,7 +24,7 @@ esplugin { } testClusters.integTest { - module file(project(':modules:mapper-extras').tasks.bundlePlugin.archiveFile) + module project(':modules:mapper-extras').tasks.bundlePlugin.archiveFile systemProperty 'es.scripting.update.ctx_in_params', 'false' // TODO: remove this once cname is prepended to transport.publish_address by default in 8.0 systemProperty 'es.transport.cname_in_publish_address', 'true' @@ -46,7 +46,7 @@ dependencyLicenses { restResources { restApi { - includeCore '_common', 'cluster', 'nodes', 'indices', 'index', 'search', 'get', 'bulk', 'update', + includeCore '_common', 'cluster', 'nodes', 'indices', 'index', 'search', 'get', 'bulk', 'update', 'scripts_painless_execute', 'put_script', 'delete_script' } } diff --git a/modules/rank-eval/build.gradle b/modules/rank-eval/build.gradle index 5bc7506a594f1..582833dc1bd22 100644 --- a/modules/rank-eval/build.gradle +++ b/modules/rank-eval/build.gradle @@ -25,11 +25,11 @@ esplugin { restResources { restApi { - includeCore '_common', 'indices', 'index', 'rank_eval' + includeCore '_common', 'indices', 'index', 'rank_eval' } } testClusters.integTest { // Modules who's integration is explicitly tested in integration tests - module file(project(':modules:lang-mustache').tasks.bundlePlugin.archiveFile) + module project(':modules:lang-mustache').tasks.bundlePlugin.archiveFile } diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle index 42b47cf590a1e..617d5de034458 100644 --- a/modules/reindex/build.gradle +++ b/modules/reindex/build.gradle @@ -35,8 +35,8 @@ esplugin { testClusters.integTest { // Modules who's integration is explicitly tested in integration tests - module file(project(':modules:parent-join').tasks.bundlePlugin.archiveFile) - module file(project(':modules:lang-painless').tasks.bundlePlugin.archiveFile) + module project(':modules:parent-join').tasks.bundlePlugin.archiveFile + module project(':modules:lang-painless').tasks.bundlePlugin.archiveFile // Whitelist reindexing from the local node so we can test reindex-from-remote. setting 'reindex.remote.whitelist', '127.0.0.1:*' } @@ -61,8 +61,8 @@ dependencies { restResources { restApi { - includeCore '_common', 'cluster', 'nodes', 'indices', 'index', 'get', 'search', 'mget', 'count', - 'update_by_query', 'delete_by_query', 'reindex_rethrottle', 'tasks', 'reindex', 'put_script' + includeCore '_common', 'cluster', 'nodes', 'indices', 'index', 'get', 'search', 'mget', 'count', + 'update_by_query', 'delete_by_query', 'reindex_rethrottle', 'tasks', 'reindex', 'put_script' } } diff --git a/plugins/discovery-ec2/qa/amazon-ec2/build.gradle b/plugins/discovery-ec2/qa/amazon-ec2/build.gradle index a10d9feccfdce..f2a940412d6d5 100644 --- a/plugins/discovery-ec2/qa/amazon-ec2/build.gradle +++ b/plugins/discovery-ec2/qa/amazon-ec2/build.gradle @@ -79,7 +79,7 @@ integTest.enabled = false testClusters."integTest${action}" { numberOfNodes = ec2NumberOfNodes - plugin file(project(':plugins:discovery-ec2').bundlePlugin.archiveFile) + plugin project(':plugins:discovery-ec2').bundlePlugin.archiveFile setting 'discovery.seed_providers', 'ec2' setting 'network.host', '_ec2_' diff --git a/plugins/discovery-gce/qa/gce/build.gradle b/plugins/discovery-gce/qa/gce/build.gradle index 7eb2fe13aae8f..515530873df9f 100644 --- a/plugins/discovery-gce/qa/gce/build.gradle +++ b/plugins/discovery-gce/qa/gce/build.gradle @@ -62,7 +62,7 @@ integTest { testClusters.integTest { numberOfNodes = gceNumberOfNodes - plugin file(project(':plugins:discovery-gce').bundlePlugin.archiveFile) + plugin project(':plugins:discovery-gce').bundlePlugin.archiveFile // use gce fixture for Auth calls instead of http://metadata.google.internal environment 'GCE_METADATA_HOST', { "http://${gceFixture.addressAndPort}" }, IGNORE_VALUE // allows to configure hidden settings (`cloud.gce.host` and `cloud.gce.root_url`) diff --git a/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle b/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle index b594ff3b172f2..3c2146095e651 100644 --- a/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle +++ b/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle @@ -69,7 +69,7 @@ integTest { } testClusters.integTest { - plugin file(project(':plugins:repository-azure').bundlePlugin.archiveFile) + plugin project(':plugins:repository-azure').bundlePlugin.archiveFile keystore 'azure.client.integration_test.account', azureAccount if (azureKey != null && azureKey.isEmpty() == false) { keystore 'azure.client.integration_test.key', azureKey diff --git a/plugins/repository-gcs/qa/google-cloud-storage/build.gradle b/plugins/repository-gcs/qa/google-cloud-storage/build.gradle index 2cf4ae5456ee1..c90cac4d07fb1 100644 --- a/plugins/repository-gcs/qa/google-cloud-storage/build.gradle +++ b/plugins/repository-gcs/qa/google-cloud-storage/build.gradle @@ -135,7 +135,7 @@ integTest { } testClusters.integTest { - plugin file(project(':plugins:repository-gcs').bundlePlugin.archiveFile) + plugin project(':plugins:repository-gcs').bundlePlugin.archiveFile keystore 'gcs.client.integration_test.credentials_file', serviceAccountFile, IGNORE_VALUE diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index 489d969232d4e..149b3b0dbb267 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -201,7 +201,7 @@ for (String integTestTaskName : ['integTestHa', 'integTestSecure', 'integTestSec } testClusters."${integTestTaskName}" { - plugin(file(bundlePlugin.archiveFile)) + plugin(bundlePlugin.archiveFile) if (integTestTaskName.contains("Secure")) { systemProperty "java.security.krb5.conf", krb5conf extraConfigFile( diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index df8a690517235..3953c1c1ce90a 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -186,7 +186,7 @@ if (useFixture) { keystore 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey keystore 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey setting 's3.client.integration_test_permanent.endpoint', minioAddress, IGNORE_VALUE - plugin file(tasks.bundlePlugin.archiveFile) + plugin tasks.bundlePlugin.archiveFile } integTest.runner { @@ -271,7 +271,7 @@ if (useFixture) { testClusters.integTestECS { setting 's3.client.integration_test_ecs.endpoint', { "${-> fixtureAddress('s3-fixture-with-ecs')}" }, IGNORE_VALUE - plugin file(tasks.bundlePlugin.archiveFile) + plugin tasks.bundlePlugin.archiveFile environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI', { "${-> fixtureAddress('s3-fixture-with-ecs')}/ecs_credentials_endpoint" }, IGNORE_VALUE } diff --git a/qa/smoke-test-plugins/build.gradle b/qa/smoke-test-plugins/build.gradle index f5b441f07631a..03992b3c1dc9c 100644 --- a/qa/smoke-test-plugins/build.gradle +++ b/qa/smoke-test-plugins/build.gradle @@ -32,7 +32,7 @@ testClusters.integTest { //Do not attempt to install ingest-attachment in FIPS 140 as it is not supported (it depends on non-FIPS BouncyCastle return } - plugin file(pluginProject.tasks.bundlePlugin.archiveFile) + plugin pluginProject.tasks.bundlePlugin.archiveFile tasks.integTest.dependsOn pluginProject.tasks.bundlePlugin pluginsCount += 1 } diff --git a/x-pack/qa/smoke-test-plugins-ssl/build.gradle b/x-pack/qa/smoke-test-plugins-ssl/build.gradle index dc04f7bfa3258..1929b1f3eaf6f 100644 --- a/x-pack/qa/smoke-test-plugins-ssl/build.gradle +++ b/x-pack/qa/smoke-test-plugins-ssl/build.gradle @@ -67,7 +67,7 @@ testClusters.integTest { user username: "monitoring_agent", password: "x-pack-test-password", role: "remote_monitoring_agent" project(':plugins').getChildProjects().each { pluginName, pluginProject -> - plugin file(pluginProject.tasks.bundlePlugin.archiveFile) + plugin pluginProject.tasks.bundlePlugin.archiveFile tasks.integTest.dependsOn pluginProject.tasks.bundlePlugin pluginsCount += 1 } diff --git a/x-pack/qa/smoke-test-plugins/build.gradle b/x-pack/qa/smoke-test-plugins/build.gradle index ded92bea09e72..be17bfe3558c8 100644 --- a/x-pack/qa/smoke-test-plugins/build.gradle +++ b/x-pack/qa/smoke-test-plugins/build.gradle @@ -15,7 +15,7 @@ testClusters.integTest { setting 'xpack.license.self_generated.type', 'trial' user username: "test_user", password: "x-pack-test-password" project(':plugins').getChildProjects().each { pluginName, pluginProject -> - plugin file(pluginProject.tasks.bundlePlugin.archiveFile) + plugin pluginProject.tasks.bundlePlugin.archiveFile tasks.integTest.dependsOn pluginProject.tasks.bundlePlugin pluginsCount += 1 }