Introduce simple public yaml-rest-test plugin (7.x backport) (#77054)

* Introduce simple public yaml-rest-test plugin (#76554) This introduces a basic public yaml rest test plugin that is supposed to be used by external elasticsearch plugin authors. This is driven by #76215 - Rename yaml-rest-test to intern-yaml-rest-test - Use public yaml plugin in example plugins Co-authored-by: Mark Vieira <[email protected]> * Fix test assertion after output normalization Co-authored-by: Mark Vieira <[email protected]>
elastic · Sep 3, 2021 · 4039c35 · 4039c35
1 parent a98df4e
commit 4039c35
Show file tree

Hide file tree

Showing 66 changed files with 290 additions and 81 deletions.
diff --git a/build-tools-internal/build.gradle b/build-tools-internal/build.gradle
@@ -152,8 +152,8 @@ gradlePlugin {
       implementationClass = 'org.elasticsearch.gradle.internal.precommit.ValidateRestSpecPlugin'
     }
     yamlRestTest {
-      id = 'elasticsearch.yaml-rest-test'
-      implementationClass = 'org.elasticsearch.gradle.internal.test.rest.YamlRestTestPlugin'
+      id = 'elasticsearch.internal-yaml-rest-test'
+      implementationClass = 'org.elasticsearch.gradle.internal.test.rest.InternalYamlRestTestPlugin'
     }
   }
 }

diff --git a/...st/rest/YamlRestTestPluginFuncTest.groovy → ...InternalYamlRestTestPluginFuncTest.groovy b/...st/rest/YamlRestTestPluginFuncTest.groovy → ...InternalYamlRestTestPluginFuncTest.groovy
@@ -11,13 +11,13 @@ package org.elasticsearch.gradle.internal.test.rest
 import org.elasticsearch.gradle.fixtures.AbstractRestResourcesFuncTest
 import org.gradle.testkit.runner.TaskOutcome
 
-class YamlRestTestPluginFuncTest extends AbstractRestResourcesFuncTest {
+class InternalYamlRestTestPluginFuncTest extends AbstractRestResourcesFuncTest {
 
     def "yamlRestTest does nothing when there are no tests"() {
         given:
         buildFile << """
         plugins {
-          id 'elasticsearch.yaml-rest-test'
+          id 'elasticsearch.internal-yaml-rest-test'
         }
         """
 
@@ -34,7 +34,7 @@ class YamlRestTestPluginFuncTest extends AbstractRestResourcesFuncTest {
         given:
         internalBuild()
         buildFile << """
-            apply plugin: 'elasticsearch.yaml-rest-test'
+            apply plugin: 'elasticsearch.internal-yaml-rest-test'
 
             dependencies {
                yamlRestTestImplementation "junit:junit:4.12"

diff --git a/...nternal/test/rest/YamlRestTestPlugin.java → ...test/rest/InternalYamlRestTestPlugin.java b/...nternal/test/rest/YamlRestTestPlugin.java → ...test/rest/InternalYamlRestTestPlugin.java
@@ -22,7 +22,7 @@
 /**
  * Apply this plugin to run the YAML based REST tests.
  */
-public class YamlRestTestPlugin implements Plugin<Project> {
+public class InternalYamlRestTestPlugin implements Plugin<Project> {
 
     public static final String SOURCE_SET_NAME = "yamlRestTest";
 

diff --git a/build-tools/build.gradle b/build-tools/build.gradle
@@ -61,6 +61,10 @@ gradlePlugin {
             id = 'elasticsearch.test-gradle-policy'
             implementationClass = 'org.elasticsearch.gradle.test.GradleTestPolicySetupPlugin'
         }
+        yamlTests {
+            id = 'elasticsearch.yaml-rest-test'
+            implementationClass = 'org.elasticsearch.gradle.test.YamlRestTestPlugin'
+        }
     }
 }
 

diff --git a/...ools/src/integTest/groovy/org/elasticsearch/gradle/test/YamlRestTestPluginFuncTest.groovy b/...ools/src/integTest/groovy/org/elasticsearch/gradle/test/YamlRestTestPluginFuncTest.groovy
@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.test
+
+import org.elasticsearch.gradle.VersionProperties
+import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
+import org.gradle.testkit.runner.TaskOutcome
+
+class YamlRestTestPluginFuncTest extends AbstractGradleFuncTest {
+
+    def "declares default dependencies"() {
+        given:
+        buildFile << """
+        plugins {
+          id 'elasticsearch.yaml-rest-test'
+        }
+        """
+
+        when:
+        def output = gradleRunner("dependencies").build().output
+        then:
+        output.contains("""
+restTestSpecs
+/--- org.elasticsearch:rest-api-spec:${VersionProperties.elasticsearch} FAILED""")
+        output.contains(normalized("""
+yamlRestTestImplementation - Implementation only dependencies for source set 'yaml rest test'. (n)
+/--- org.elasticsearch.test:framework:${VersionProperties.elasticsearch} (n)"""))
+    }
+
+    def "yamlRestTest does nothing when there are no tests"() {
+        given:
+        buildFile << """
+        plugins {
+          id 'elasticsearch.yaml-rest-test'
+        }
+        
+        repositories {
+            mavenCentral()
+        }
+   
+        dependencies {
+            yamlRestTestImplementation "org.elasticsearch.test:framework:7.14.0"
+            restTestSpecs "org.elasticsearch:rest-api-spec:7.14.0"
+        }
+        """
+
+        when:
+        def result = gradleRunner("yamlRestTest").build()
+        then:
+        result.task(':compileYamlRestTestJava').outcome == TaskOutcome.NO_SOURCE
+        result.task(':processYamlRestTestResources').outcome == TaskOutcome.NO_SOURCE
+        result.task(':yamlRestTest').outcome == TaskOutcome.NO_SOURCE
+    }
+
+}
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/PluginBuildPlugin.java b/build-tools/src/main/java/org/elasticsearch/gradle/plugin/PluginBuildPlugin.java
@@ -59,6 +59,9 @@
  * Encapsulates build configuration for an Elasticsearch plugin.
  */
 public class PluginBuildPlugin implements Plugin<Project> {
+
+    public static final String BUNDLE_PLUGIN_TASK_NAME = "bundlePlugin";
+
     @Override
     public void apply(final Project project) {
         project.getPluginManager().apply(JavaPlugin.class);
@@ -129,7 +132,7 @@ public void apply(final Project project) {
 
         project.getTasks().register("run", RunTask.class, runTask -> {
             runTask.useCluster(runCluster);
-            runTask.dependsOn(project.getTasks().named("bundlePlugin"));
+            runTask.dependsOn(project.getTasks().named(BUNDLE_PLUGIN_TASK_NAME));
         });
     }
 

diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/test/YamlRestTestPlugin.java b/build-tools/src/main/java/org/elasticsearch/gradle/test/YamlRestTestPlugin.java
@@ -0,0 +1,124 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.test;
+
+import org.elasticsearch.gradle.VersionProperties;
+import org.elasticsearch.gradle.plugin.PluginBuildPlugin;
+import org.elasticsearch.gradle.testclusters.ElasticsearchCluster;
+import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask;
+import org.elasticsearch.gradle.testclusters.TestClustersPlugin;
+import org.elasticsearch.gradle.transform.UnzipTransform;
+import org.gradle.api.Action;
+import org.gradle.api.NamedDomainObjectContainer;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.Task;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.artifacts.ConfigurationContainer;
+import org.gradle.api.artifacts.dsl.DependencyHandler;
+import org.gradle.api.artifacts.type.ArtifactTypeDefinition;
+import org.gradle.api.attributes.Attribute;
+import org.gradle.api.internal.artifacts.ArtifactAttributes;
+import org.gradle.api.plugins.JavaBasePlugin;
+import org.gradle.api.tasks.Copy;
+import org.gradle.api.tasks.SourceSet;
+import org.gradle.api.tasks.SourceSetContainer;
+import org.gradle.api.tasks.TaskProvider;
+import org.gradle.api.tasks.bundling.Zip;
+
+import java.io.File;
+
+import static org.elasticsearch.gradle.plugin.PluginBuildPlugin.BUNDLE_PLUGIN_TASK_NAME;
+
+public class YamlRestTestPlugin implements Plugin<Project> {
+
+    public static final String REST_TEST_SPECS_CONFIGURATION_NAME = "restTestSpecs";
+    public static final String YAML_REST_TEST = "yamlRestTest";
+
+    @Override
+    public void apply(Project project) {
+        project.getPluginManager().apply(GradleTestPolicySetupPlugin.class);
+        project.getPluginManager().apply(TestClustersPlugin.class);
+        project.getPluginManager().apply(JavaBasePlugin.class);
+
+        Attribute<Boolean> restAttribute = Attribute.of("restSpecs", Boolean.class);
+        project.getDependencies().getAttributesSchema().attribute(restAttribute);
+        project.getDependencies().getArtifactTypes().maybeCreate(ArtifactTypeDefinition.JAR_TYPE);
+        project.getDependencies().registerTransform(UnzipTransform.class, transformSpec -> {
+            transformSpec.getFrom()
+                .attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.JAR_TYPE)
+                .attribute(restAttribute, true);
+            transformSpec.getTo()
+                .attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE)
+                .attribute(restAttribute, true);
+        });
+
+        ConfigurationContainer configurations = project.getConfigurations();
+        Configuration restTestSpecs = configurations.create(REST_TEST_SPECS_CONFIGURATION_NAME);
+        restTestSpecs.getAttributes().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE);
+        restTestSpecs.getAttributes().attribute(restAttribute, true);
+
+        TaskProvider<Copy> copyRestTestSpecs = project.getTasks().register("copyRestTestSpecs", Copy.class, t -> {
+            t.from(restTestSpecs);
+            t.into(new File(project.getBuildDir(), "restResources/restspec"));
+        });
+
+        var sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
+        var testSourceSet = sourceSets.maybeCreate(YAML_REST_TEST);
+        NamedDomainObjectContainer<ElasticsearchCluster> testClusters = (NamedDomainObjectContainer<ElasticsearchCluster>) project
+            .getExtensions()
+            .getByName(TestClustersPlugin.EXTENSION_NAME);
+
+        testSourceSet.getOutput().dir(copyRestTestSpecs.map(Task::getOutputs));
+        Configuration yamlRestTestImplementation = configurations.getByName(testSourceSet.getImplementationConfigurationName());
+        setupDefaultDependencies(project.getDependencies(), restTestSpecs, yamlRestTestImplementation);
+        var cluster = testClusters.maybeCreate(YAML_REST_TEST);
+        TaskProvider<StandaloneRestIntegTestTask> yamlRestTestTask = setupTestTask(project, testSourceSet, cluster);
+        project.getPlugins().withType(PluginBuildPlugin.class, p -> {
+            TaskProvider<Zip> bundle = project.getTasks().withType(Zip.class).named(BUNDLE_PLUGIN_TASK_NAME);
+            cluster.plugin(bundle.flatMap(Zip::getArchiveFile));
+            yamlRestTestTask.configure(t -> t.dependsOn(bundle));
+        });
+    }
+
+    private static void setupDefaultDependencies(
+        DependencyHandler dependencyHandler,
+        Configuration restTestSpecs,
+        Configuration yamlRestTestImplementation
+    ) {
+        String elasticsearchVersion = VersionProperties.getElasticsearch();
+        yamlRestTestImplementation.defaultDependencies(
+            deps -> deps.add(dependencyHandler.create("org.elasticsearch.test:framework:" + elasticsearchVersion))
+        );
+
+        restTestSpecs.defaultDependencies(
+            deps -> deps.add(dependencyHandler.create("org.elasticsearch:rest-api-spec:" + elasticsearchVersion))
+        );
+    }
+
+    private TaskProvider<StandaloneRestIntegTestTask> setupTestTask(
+        Project project,
+        SourceSet testSourceSet,
+        ElasticsearchCluster cluster
+    ) {
+        return project.getTasks().register("yamlRestTest", StandaloneRestIntegTestTask.class, task -> {
+            task.useCluster(cluster);
+            task.setTestClassesDirs(testSourceSet.getOutput().getClassesDirs());
+            task.setClasspath(testSourceSet.getRuntimeClasspath());
+
+            var nonInputProperties = new SystemPropertyCommandLineArgumentProvider();
+            nonInputProperties.systemProperty("tests.rest.cluster", () -> String.join(",", cluster.getAllHttpSocketURI()));
+            nonInputProperties.systemProperty("tests.cluster", () -> String.join(",", cluster.getAllTransportPortURI()));
+            nonInputProperties.systemProperty("tests.clustername", () -> cluster.getName());
+            task.getJvmArgumentProviders().add(nonInputProperties);
+            task.systemProperty("tests.rest.load_packaged", Boolean.FALSE.toString());
+        });
+    }
+
+}
diff --git a/modules/aggs-matrix-stats/build.gradle b/modules/aggs-matrix-stats/build.gradle
@@ -5,7 +5,7 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.yaml-rest-test'
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
 
 esplugin {
   description 'Adds aggregations whose input are a list of numeric fields and output includes a matrix.'

diff --git a/modules/analysis-common/build.gradle b/modules/analysis-common/build.gradle
@@ -5,7 +5,7 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.yaml-rest-test'
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {

diff --git a/modules/ingest-common/build.gradle b/modules/ingest-common/build.gradle
@@ -5,7 +5,7 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.yaml-rest-test'
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {

diff --git a/modules/ingest-geoip/build.gradle b/modules/ingest-geoip/build.gradle
@@ -8,7 +8,7 @@
 
 import org.apache.tools.ant.taskdefs.condition.Os
 
-apply plugin: 'elasticsearch.yaml-rest-test'
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {

diff --git a/modules/ingest-user-agent/build.gradle b/modules/ingest-user-agent/build.gradle
@@ -5,7 +5,7 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.yaml-rest-test'
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
 
 esplugin {
   description 'Ingest processor that extracts information from a user agent'
@@ -21,4 +21,3 @@ restResources {
 testClusters.all {
   extraConfigFile 'ingest-user-agent/test-regexes.yml', file('src/test/test-regexes.yml')
 }
-
diff --git a/modules/lang-expression/build.gradle b/modules/lang-expression/build.gradle
@@ -5,7 +5,7 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.yaml-rest-test'
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
@@ -30,4 +30,3 @@ tasks.named("dependencyLicenses").configure {
   mapping from: /lucene-.*/, to: 'lucene'
   mapping from: /asm-.*/, to: 'asm'
 }
-
diff --git a/modules/lang-mustache/build.gradle b/modules/lang-mustache/build.gradle
@@ -5,7 +5,7 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.yaml-rest-test'
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
 apply plugin: 'elasticsearch.java-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 

diff --git a/modules/lang-painless/build.gradle b/modules/lang-painless/build.gradle
@@ -8,7 +8,7 @@
 
 import org.elasticsearch.gradle.testclusters.DefaultTestClustersTask;
 apply plugin: 'elasticsearch.validate-rest-spec'
-apply plugin: 'elasticsearch.yaml-rest-test'
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
 
 esplugin {
   description 'An easy, safe and fast scripting language for Elasticsearch'
@@ -312,4 +312,4 @@ tasks.register("regenSuggest") {
             patternset(includes: 'Suggest*.java')
         }
     }
-}
+}
diff --git a/modules/mapper-extras/build.gradle b/modules/mapper-extras/build.gradle
@@ -5,7 +5,7 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.yaml-rest-test'
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {

diff --git a/modules/parent-join/build.gradle b/modules/parent-join/build.gradle
@@ -5,7 +5,7 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.yaml-rest-test'
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {

diff --git a/modules/percolator/build.gradle b/modules/percolator/build.gradle
@@ -5,7 +5,7 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.yaml-rest-test'
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {

diff --git a/modules/rank-eval/build.gradle b/modules/rank-eval/build.gradle
@@ -5,7 +5,7 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.yaml-rest-test'
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {

diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle
@@ -14,7 +14,7 @@ import org.elasticsearch.gradle.internal.test.AntFixture
 
 apply plugin: 'elasticsearch.test-with-dependencies'
 apply plugin: 'elasticsearch.jdk-download'
-apply plugin: 'elasticsearch.yaml-rest-test'
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
 apply plugin: 'elasticsearch.java-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'