Smarter copying of the rest specs and tests

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -24,6 +24,7 @@ import org.elasticsearch.gradle.NoticeTask
 import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.info.BuildParams
+import org.elasticsearch.gradle.test.rest.CopyRestApiPlugin
 import org.elasticsearch.gradle.test.RestIntegTestTask
 import org.elasticsearch.gradle.testclusters.RunTask
 import org.elasticsearch.gradle.testclusters.TestClustersPlugin
@@ -51,6 +52,7 @@ class PluginBuildPlugin implements Plugin<Project> {
     void apply(Project project) {
         project.pluginManager.apply(BuildPlugin)
         project.pluginManager.apply(TestClustersPlugin)
+        project.pluginManager.apply(CopyRestApiPlugin)
 
         PluginPropertiesExtension extension = project.extensions.create(PLUGIN_EXTENSION_NAME, PluginPropertiesExtension, project)
         configureDependencies(project)

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

@@ -18,29 +18,19 @@
  */
 package org.elasticsearch.gradle.test
 
-import org.elasticsearch.gradle.VersionProperties
-import org.elasticsearch.gradle.info.BuildParams
 import org.elasticsearch.gradle.testclusters.ElasticsearchCluster
 import org.elasticsearch.gradle.testclusters.RestTestRunnerTask
-import org.elasticsearch.gradle.tool.Boilerplate
 import org.gradle.api.DefaultTask
 import org.gradle.api.Task
-import org.gradle.api.file.FileCopyDetails
-import org.gradle.api.tasks.Copy
-import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.testing.Test
-import org.gradle.plugins.ide.idea.IdeaPlugin
+
 /**
  * A wrapper task around setting up a cluster and running rest tests.
  */
 class RestIntegTestTask extends DefaultTask {
 
     protected Test runner
 
-    /** Flag indicating whether the rest tests in the rest spec should be run. */
-    @Input
-    Boolean includePackaged = false
-
     RestIntegTestTask() {
         runner = project.tasks.create("${name}Runner", RestTestRunnerTask.class)
         super.dependsOn(runner)
@@ -69,10 +59,6 @@ class RestIntegTestTask extends DefaultTask {
             runner.systemProperty('test.clustername', System.getProperty("tests.clustername"))
         }
 
-        // copy the rest spec/tests onto the test classpath
-        Copy copyRestSpec = createCopyRestSpecTask()
-        project.sourceSets.test.output.builtBy(copyRestSpec)
-
         // this must run after all projects have been configured, so we know any project
         // references can be accessed as a fully configured
         project.gradle.projectsEvaluated {
@@ -83,12 +69,6 @@ class RestIntegTestTask extends DefaultTask {
         }
     }
 
-    /** Sets the includePackaged property */
-    public void includePackaged(boolean include) {
-        includePackaged = include
-    }
-
-
     @Override
     public Task dependsOn(Object... dependencies) {
         runner.dependsOn(dependencies)
@@ -114,37 +94,4 @@ class RestIntegTestTask extends DefaultTask {
         project.tasks.getByName("${name}Runner").configure(configure)
     }
 
-    Copy createCopyRestSpecTask() {
-        Boilerplate.maybeCreate(project.configurations, 'restSpec') {
-            project.dependencies.add(
-                    'restSpec',
-                    BuildParams.internal ? project.project(':rest-api-spec') :
-                            "org.elasticsearch:rest-api-spec:${VersionProperties.elasticsearch}"
-            )
-        }
-
-        return Boilerplate.maybeCreate(project.tasks, 'copyRestSpec', Copy) { Copy copy ->
-            copy.dependsOn project.configurations.restSpec
-            copy.into(project.sourceSets.test.output.resourcesDir)
-            copy.from({ project.zipTree(project.configurations.restSpec.singleFile) }) {
-                includeEmptyDirs = false
-                include 'rest-api-spec/**'
-                filesMatching('rest-api-spec/test/**') { FileCopyDetails details ->
-                    if (includePackaged == false) {
-                        details.exclude()
-                    }
-                }
-            }
-
-            if (project.plugins.hasPlugin(IdeaPlugin)) {
-                project.idea {
-                    module {
-                        if (scopes.TEST != null) {
-                            scopes.TEST.plus.add(project.configurations.restSpec)
-                        }
-                    }
-                }
-            }
-        }
-    }
 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy

@@ -26,9 +26,9 @@ import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask
 import org.elasticsearch.gradle.info.BuildParams
 import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin
 import org.elasticsearch.gradle.precommit.PrecommitTasks
+import org.elasticsearch.gradle.test.rest.CopyRestApiPlugin
 import org.elasticsearch.gradle.testclusters.TestClustersPlugin
 import org.gradle.api.InvalidUserDataException
-import org.gradle.api.JavaVersion
 import org.gradle.api.Plugin
 import org.gradle.api.Project
 import org.gradle.api.artifacts.Configuration
@@ -42,6 +42,7 @@ import org.gradle.api.tasks.compile.JavaCompile
 import org.gradle.api.tasks.testing.Test
 import org.gradle.plugins.ide.eclipse.model.EclipseModel
 import org.gradle.plugins.ide.idea.model.IdeaModel
+
 /**
  * Configures the build to compile tests against Elasticsearch's test framework
  * and run REST tests. Use BuildPlugin if you want to build main code as well
@@ -74,6 +75,8 @@ class StandaloneRestTestPlugin implements Plugin<Project> {
         // only setup tests to build
         SourceSetContainer sourceSets = project.extensions.getByType(SourceSetContainer)
         SourceSet testSourceSet = sourceSets.create('test')
+        // need to apply plugin after test source sets are created
+        project.pluginManager.apply(CopyRestApiPlugin)
 
         project.tasks.withType(Test) { Test test ->
             test.testClassesDirs = testSourceSet.output.classesDirs

buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java

@@ -250,7 +250,7 @@ public void doCheck() throws IOException {
             Files.write(getSuccessMarker().toPath(), new byte[] {}, StandardOpenOption.CREATE);
         } else {
             getLogger().error(problems);
-            throw new IllegalStateException("Testing conventions are not honored");
+            throw new IllegalStateException(String.format("Testing conventions [%s] are not honored", problems));
         }
     }
 

buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiPlugin.java

@@ -0,0 +1,136 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.test.rest;
+
+import org.elasticsearch.gradle.VersionProperties;
+import org.elasticsearch.gradle.info.BuildParams;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.artifacts.Dependency;
+import org.gradle.api.provider.Provider;
+
+import java.util.Map;
+
+/**
+ * <p>
+ * Gradle plugin to help configure {@link CopyRestApiTask}'s and {@link CopyRestTestsTask} that copies the artifacts needed for the Rest API
+ * spec and YAML based rest tests.
+ * </p>
+ * <strong>Rest API specification:</strong> <br>
+ * When the {@link CopyRestApiPlugin} has been applied the {@link CopyRestApiTask} will automatically copy the core Rest API specification
+ * if there are any Rest YAML tests present in source, or copied from {@link CopyRestTestsTask} output. X-pack specs must be explicitly
+ * declared to be copied.
+ * <br>
+ * <i>For example:</i>
+ * <pre>
+ * restResources {
+ *   restApi {
+ *     includeXpack 'enrich'
+ *   }
+ * }
+ * </pre>
+ * Will copy the entire core Rest API specifications (assuming the project has tests) and any of the the X-pack specs starting with enrich*.
+ * It is recommended (but not required) to also explicitly declare which core specs your project depends on to help optimize the caching
+ * behavior.
+ * <i>For example:</i>
+ * <pre>
+ * restResources {
+ *   restApi {
+ *     includeCore 'index', 'cat'
+ *     includeXpack 'enrich'
+ *   }
+ * }
+ * </pre>
+ * <br>
+ * <strong>Rest YAML tests :</strong> <br>
+ * When the {@link CopyRestApiPlugin} has been applied the {@link CopyRestTestsTask} will copy the Rest YAML tests if explicitly
+ * configured with `includeCore` or `includeXpack` through the `restResources.restTests` extension.
+ * <i>For example:</i>
+ * <pre>
+ * restResources {
+ *  restApi {
+ *      includeXpack 'graph'
+ *   }
+ *   restTests {
+ *     includeXpack 'graph'
+ *   }
+ * }
+ * </pre>
+ * Will copy any of the the x-pack tests that start with graph, and will copy the X-pack graph specification, as well as the full core
+ * Rest API specification.
+ *
+ * @see CopyRestApiTask
+ * @see CopyRestTestsTask
+ */
+public class CopyRestApiPlugin implements Plugin<Project> {
+
+    private static final String EXTENSION_NAME = "restResources";
+
+    @Override
+    public void apply(Project project) {
+        RestResourcesExtension extension = project.getExtensions().create(EXTENSION_NAME, RestResourcesExtension.class);
+
+        Provider<CopyRestTestsTask> copyRestYamlTestTask = project.getTasks()
+            .register("copyYamlTestsTask", CopyRestTestsTask.class, task -> {
+                task.includeCore.set(extension.restTests.getIncludeCore());
+                task.includeXpack.set(extension.restTests.getIncludeXpack());
+                task.coreConfig = project.getConfigurations().create("restTest");
+                if (BuildParams.isInternal()) {
+                    Dependency dependency = project.getDependencies()
+                        .project(Map.of("path", ":rest-api-spec", "configuration", "restTests"));
+                    project.getDependencies().add(task.coreConfig.getName(), dependency);
+
+                    task.xpackConfig = project.getConfigurations().create("restXpackTest");
+                    dependency = project.getDependencies().project(Map.of("path", ":x-pack:plugin", "configuration", "restXpackTests"));
+                    project.getDependencies().add(task.xpackConfig.getName(), dependency);
+                    task.dependsOn(task.xpackConfig);
+                } else {
+                    Dependency dependency = project.getDependencies()
+                        .create("org.elasticsearch:rest-api-spec:" + VersionProperties.getElasticsearch());
+                    project.getDependencies().add(task.coreConfig.getName(), dependency);
+                }
+                task.dependsOn(task.coreConfig);
+            });
+
+        Provider<CopyRestApiTask> copyRestYamlSpecTask = project.getTasks()
+            .register("copyRestApiSpecsTask", CopyRestApiTask.class, task -> {
+                task.includeCore.set(extension.restApi.getIncludeCore());
+                task.includeXpack.set(extension.restApi.getIncludeXpack());
+                task.dependsOn(copyRestYamlTestTask);
+                task.coreConfig = project.getConfigurations().create("restSpec");
+                if (BuildParams.isInternal()) {
+                    Dependency dependency = project.getDependencies()
+                        .project(Map.of("path", ":rest-api-spec", "configuration", "restSpecs"));
+                    project.getDependencies().add(task.coreConfig.getName(), dependency);
+
+                    task.xpackConfig = project.getConfigurations().create("restXpackSpec");
+                    dependency = project.getDependencies().project(Map.of("path", ":x-pack:plugin", "configuration", "restXpackSpecs"));
+                    project.getDependencies().add(task.xpackConfig.getName(), dependency);
+                    task.dependsOn(task.xpackConfig);
+                } else {
+                    Dependency dependency = project.getDependencies()
+                        .create("org.elasticsearch:rest-api-spec:" + VersionProperties.getElasticsearch());
+                    project.getDependencies().add(task.coreConfig.getName(), dependency);
+                }
+                task.dependsOn(task.coreConfig);
+            });
+
+        project.getTasks().named("processTestResources").configure(t -> t.dependsOn(copyRestYamlSpecTask));
+    }
+}