Smarter copying of the rest specs and tests

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -24,6 +24,7 @@ import org.elasticsearch.gradle.NoticeTask
 import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.info.BuildParams
+import org.elasticsearch.gradle.test.rest.RestResourcesPlugin
 import org.elasticsearch.gradle.test.RestIntegTestTask
 import org.elasticsearch.gradle.testclusters.RunTask
 import org.elasticsearch.gradle.testclusters.TestClustersPlugin
@@ -51,6 +52,7 @@ class PluginBuildPlugin implements Plugin<Project> {
     void apply(Project project) {
         project.pluginManager.apply(BuildPlugin)
         project.pluginManager.apply(TestClustersPlugin)
+        project.pluginManager.apply(RestResourcesPlugin)
 
         PluginPropertiesExtension extension = project.extensions.create(PLUGIN_EXTENSION_NAME, PluginPropertiesExtension, project)
         configureDependencies(project)

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

@@ -18,29 +18,19 @@
  */
 package org.elasticsearch.gradle.test
 
-import org.elasticsearch.gradle.VersionProperties
-import org.elasticsearch.gradle.info.BuildParams
 import org.elasticsearch.gradle.testclusters.ElasticsearchCluster
 import org.elasticsearch.gradle.testclusters.RestTestRunnerTask
-import org.elasticsearch.gradle.tool.Boilerplate
 import org.gradle.api.DefaultTask
 import org.gradle.api.Task
-import org.gradle.api.file.FileCopyDetails
-import org.gradle.api.tasks.Copy
-import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.testing.Test
-import org.gradle.plugins.ide.idea.IdeaPlugin
+
 /**
  * A wrapper task around setting up a cluster and running rest tests.
  */
 class RestIntegTestTask extends DefaultTask {
 
     protected Test runner
 
-    /** Flag indicating whether the rest tests in the rest spec should be run. */
-    @Input
-    Boolean includePackaged = false
-
     RestIntegTestTask() {
         runner = project.tasks.create("${name}Runner", RestTestRunnerTask.class)
         super.dependsOn(runner)
@@ -69,10 +59,6 @@ class RestIntegTestTask extends DefaultTask {
             runner.systemProperty('test.clustername', System.getProperty("tests.clustername"))
         }
 
-        // copy the rest spec/tests onto the test classpath
-        Copy copyRestSpec = createCopyRestSpecTask()
-        project.sourceSets.test.output.builtBy(copyRestSpec)
-
         // this must run after all projects have been configured, so we know any project
         // references can be accessed as a fully configured
         project.gradle.projectsEvaluated {
@@ -83,12 +69,6 @@ class RestIntegTestTask extends DefaultTask {
         }
     }
 
-    /** Sets the includePackaged property */
-    public void includePackaged(boolean include) {
-        includePackaged = include
-    }
-
-
     @Override
     public Task dependsOn(Object... dependencies) {
         runner.dependsOn(dependencies)
@@ -114,37 +94,4 @@ class RestIntegTestTask extends DefaultTask {
         project.tasks.getByName("${name}Runner").configure(configure)
     }
 
-    Copy createCopyRestSpecTask() {
-        Boilerplate.maybeCreate(project.configurations, 'restSpec') {
-            project.dependencies.add(
-                    'restSpec',
-                    BuildParams.internal ? project.project(':rest-api-spec') :
-                            "org.elasticsearch:rest-api-spec:${VersionProperties.elasticsearch}"
-            )
-        }
-
-        return Boilerplate.maybeCreate(project.tasks, 'copyRestSpec', Copy) { Copy copy ->
-            copy.dependsOn project.configurations.restSpec
-            copy.into(project.sourceSets.test.output.resourcesDir)
-            copy.from({ project.zipTree(project.configurations.restSpec.singleFile) }) {
-                includeEmptyDirs = false
-                include 'rest-api-spec/**'
-                filesMatching('rest-api-spec/test/**') { FileCopyDetails details ->
-                    if (includePackaged == false) {
-                        details.exclude()
-                    }
-                }
-            }
-
-            if (project.plugins.hasPlugin(IdeaPlugin)) {
-                project.idea {
-                    module {
-                        if (scopes.TEST != null) {
-                            scopes.TEST.plus.add(project.configurations.restSpec)
-                        }
-                    }
-                }
-            }
-        }
-    }
 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy

@@ -26,9 +26,9 @@ import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask
 import org.elasticsearch.gradle.info.BuildParams
 import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin
 import org.elasticsearch.gradle.precommit.PrecommitTasks
+import org.elasticsearch.gradle.test.rest.RestResourcesPlugin
 import org.elasticsearch.gradle.testclusters.TestClustersPlugin
 import org.gradle.api.InvalidUserDataException
-import org.gradle.api.JavaVersion
 import org.gradle.api.Plugin
 import org.gradle.api.Project
 import org.gradle.api.artifacts.Configuration
@@ -42,6 +42,7 @@ import org.gradle.api.tasks.compile.JavaCompile
 import org.gradle.api.tasks.testing.Test
 import org.gradle.plugins.ide.eclipse.model.EclipseModel
 import org.gradle.plugins.ide.idea.model.IdeaModel
+
 /**
  * Configures the build to compile tests against Elasticsearch's test framework
  * and run REST tests. Use BuildPlugin if you want to build main code as well
@@ -74,6 +75,8 @@ class StandaloneRestTestPlugin implements Plugin<Project> {
         // only setup tests to build
         SourceSetContainer sourceSets = project.extensions.getByType(SourceSetContainer)
         SourceSet testSourceSet = sourceSets.create('test')
+        // need to apply plugin after test source sets are created
+        project.pluginManager.apply(RestResourcesPlugin)
 
         project.tasks.withType(Test) { Test test ->
             test.testClassesDirs = testSourceSet.output.classesDirs

buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java

@@ -250,7 +250,7 @@ public void doCheck() throws IOException {
             Files.write(getSuccessMarker().toPath(), new byte[] {}, StandardOpenOption.CREATE);
         } else {
             getLogger().error(problems);
-            throw new IllegalStateException("Testing conventions are not honored");
+            throw new IllegalStateException(String.format("Testing conventions [%s] are not honored", problems));
         }
     }
 

buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java

@@ -0,0 +1,195 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.test.rest;
+
+import org.elasticsearch.gradle.VersionProperties;
+import org.elasticsearch.gradle.info.BuildParams;
+import org.elasticsearch.gradle.tool.Boilerplate;
+import org.gradle.api.DefaultTask;
+import org.gradle.api.Project;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.file.ConfigurableFileCollection;
+import org.gradle.api.file.FileTree;
+import org.gradle.api.provider.ListProperty;
+import org.gradle.api.tasks.Input;
+import org.gradle.api.tasks.InputFiles;
+import org.gradle.api.tasks.OutputDirectory;
+import org.gradle.api.tasks.SkipWhenEmpty;
+import org.gradle.api.tasks.SourceSet;
+import org.gradle.api.tasks.TaskAction;
+import org.gradle.api.tasks.util.PatternFilterable;
+import org.gradle.api.tasks.util.PatternSet;
+import org.gradle.internal.Factory;
+
+import javax.inject.Inject;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Copies the files needed for the Rest YAML specs to the current projects test resources output directory.
+ * This is intended to be be used from {@link RestResourcesPlugin} since the plugin wires up the needed
+ * configurations and custom extensions.
+ * @see RestResourcesPlugin
+ */
+public class CopyRestApiTask extends DefaultTask {
+    private static final String COPY_TO = "rest-api-spec/api";
+    final ListProperty<String> includeCore = getProject().getObjects().listProperty(String.class);
+    final ListProperty<String> includeXpack = getProject().getObjects().listProperty(String.class);
+
+    Configuration coreConfig;
+    Configuration xpackConfig;
+
+    private final PatternFilterable corePatternSet;
+    private final PatternFilterable xpackPatternSet;
+
+    public CopyRestApiTask() {
+        corePatternSet = getPatternSetFactory().create();
+        xpackPatternSet = getPatternSetFactory().create();
+    }
+
+    @Inject
+    protected Factory<PatternSet> getPatternSetFactory() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Input
+    public ListProperty<String> getIncludeCore() {
+        return includeCore;
+    }
+
+    @Input
+    public ListProperty<String> getIncludeXpack() {
+        return includeXpack;
+    }
+
+    @SkipWhenEmpty
+    @InputFiles
+    public FileTree getInputDir() {
+        xpackPatternSet.setIncludes(includeXpack.get().stream().map(prefix -> prefix + "*/**").collect(Collectors.toList()));
+        ConfigurableFileCollection fileCollection = getProject().files(xpackConfig.getAsFileTree().matching(xpackPatternSet));
+        if (BuildParams.isInternal()) {
+            corePatternSet.setIncludes(includeCore.get().stream().map(prefix -> prefix + "*/**").collect(Collectors.toList()));
+            fileCollection.plus(coreConfig.getAsFileTree().matching(corePatternSet));
+        } else {
+            fileCollection.plus(coreConfig);
+        }
+        // if project has rest tests or the includes are explicitly configured execute the task, else NO-SOURCE due to the null input
+        return projectHasYamlRestTests() || includeCore.get().isEmpty() == false || includeXpack.get().isEmpty() == false
+            ? fileCollection.getAsFileTree()
+            : null;
+    }
+
+    @OutputDirectory
+    public File getOutputDir() {
+        return new File(getTestSourceSet().getOutput().getResourcesDir(), COPY_TO);
+    }
+
+    @TaskAction
+    void copy() {
+        Project project = getProject();
+        // always copy the core specs if the task executes
+        if (BuildParams.isInternal()) {
+            getLogger().debug("Rest specs for project [{}] will be copied to the test resources.", project.getPath());
+            project.copy(c -> {
+                c.from(coreConfig.getSingleFile());
+                c.into(getOutputDir());
+                c.include(corePatternSet.getIncludes());
+            });
+        } else {
+            getLogger().debug(
+                "Rest specs for project [{}] will be copied to the test resources from the published jar (version: [{}]).",
+                project.getPath(),
+                VersionProperties.getElasticsearch()
+            );
+            project.copy(c -> {
+                c.from(project.zipTree(coreConfig.getSingleFile()));
+                c.into(getTestSourceSet().getOutput().getResourcesDir()); // this ends up as the same dir as outputDir
+                c.include(includeCore.get().stream().map(prefix -> COPY_TO + "/" + prefix + "*/**").collect(Collectors.toList()));
+            });
+        }
+        // only copy x-pack specs if explicitly instructed
+        if (includeXpack.get().isEmpty() == false) {
+            getLogger().debug("X-pack rest specs for project [{}] will be copied to the test resources.", project.getPath());
+            project.copy(c -> {
+                c.from(xpackConfig.getSingleFile());
+                c.into(getOutputDir());
+                c.include(xpackPatternSet.getIncludes());
+            });
+        }
+    }
+
+    /**
+     * Returns true if any files with a .yml extension exist the test resources rest-api-spec/test directory (from source or output dir)
+     */
+    private boolean projectHasYamlRestTests() {
+        File testSourceResourceDir = getTestSourceResourceDir();
+        File testOutputResourceDir = getTestOutputResourceDir(); // check output for cases where tests are copied programmatically
+
+        if (testSourceResourceDir == null && testOutputResourceDir == null) {
+            return false;
+        }
+        try {
+            if (testSourceResourceDir != null) {
+                return new File(testSourceResourceDir, "rest-api-spec/test").exists() == false
+                    || Files.walk(testSourceResourceDir.toPath().resolve("rest-api-spec/test"))
+                        .anyMatch(p -> p.getFileName().toString().endsWith("yml"));
+            }
+            if (testOutputResourceDir != null) {
+                return new File(testOutputResourceDir, "rest-api-spec/test").exists() == false
+                    || Files.walk(testOutputResourceDir.toPath().resolve("rest-api-spec/test"))
+                        .anyMatch(p -> p.getFileName().toString().endsWith("yml"));
+            }
+        } catch (IOException e) {
+            throw new IllegalStateException(String.format("Error determining if this project [%s] has rest tests.", getProject()), e);
+        }
+        return false;
+    }
+
+    private File getTestSourceResourceDir() {
+        SourceSet testSources = getTestSourceSet();
+        if (testSources == null) {
+            return null;
+        }
+        Set<File> resourceDir = testSources.getResources()
+            .getSrcDirs()
+            .stream()
+            .filter(f -> f.isDirectory() && f.getParentFile().getName().equals("test") && f.getName().equals("resources"))
+            .collect(Collectors.toSet());
+        assert resourceDir.size() <= 1;
+        if (resourceDir.size() == 0) {
+            return null;
+        }
+        return resourceDir.iterator().next();
+    }
+
+    private File getTestOutputResourceDir() {
+        SourceSet testSources = getTestSourceSet();
+        if (testSources == null) {
+            return null;
+        }
+        return testSources.getOutput().getResourcesDir();
+    }
+
+    private SourceSet getTestSourceSet() {
+        return Boilerplate.getJavaSourceSets(getProject()).findByName("test");
+    }
+}