Merge remote-tracking branch 'origin/master' into fix/24969

benchmarks/build.gradle

@@ -34,6 +34,10 @@ apply plugin: 'com.github.johnrengelman.shadow'
 // have the shadow plugin provide the runShadow task
 apply plugin: 'application'
 
+// Not published so no need to assemble
+tasks.remove(assemble)
+build.dependsOn.remove('assemble')
+
 archivesBaseName = 'elasticsearch-benchmarks'
 mainClassName = 'org.openjdk.jmh.Main'
 

build.gradle

@@ -152,10 +152,28 @@ task verifyVersions {
   }
 }
 
+/*
+ * When adding backcompat behavior that spans major versions, temporarily
+ * disabling the backcompat tests is necessary. This flag controls
+ * the enabled state of every bwc task. It should be set back to true
+ * after the backport of the backcompat code is complete.
+ */
+allprojects {
+  ext.bwc_tests_enabled = true
+}
+
+task verifyBwcTestsEnabled {
+  doLast {
+    if (project.bwc_tests_enabled == false) {
+      throw new GradleException('Bwc tests are disabled. They must be re-enabled after completing backcompat behavior backporting.')
+    }
+  }
+}
+
 task branchConsistency {
   description 'Ensures this branch is internally consistent. For example, that versions constants match released versions.'
   group 'Verification'
-  dependsOn verifyVersions
+  dependsOn verifyVersions, verifyBwcTestsEnabled
 }
 
 subprojects {
@@ -402,3 +420,17 @@ task run(type: Run) {
   group = 'Verification'
   impliesSubProjects = true
 }
+
+/* Remove assemble on all qa projects because we don't need to publish
+ * artifacts for them. */
+gradle.projectsEvaluated {
+  subprojects {
+    if (project.path.startsWith(':qa')) {
+      Task assemble = project.tasks.findByName('assemble')
+      if (assemble) {
+        project.tasks.remove(assemble)
+        project.build.dependsOn.remove('assemble')
+      }
+    }
+  }
+}

buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingPlugin.groovy

@@ -12,10 +12,38 @@ import org.gradle.api.tasks.testing.Test
 class RandomizedTestingPlugin implements Plugin<Project> {
 
     void apply(Project project) {
+        setupSeed(project)
         replaceTestTask(project.tasks)
         configureAnt(project.ant)
     }
 
+    /**
+     * Pins the test seed at configuration time so it isn't different on every
+     * {@link RandomizedTestingTask} execution. This is useful if random
+     * decisions in one run of {@linkplain RandomizedTestingTask} influence the
+     * outcome of subsequent runs. Pinning the seed up front like this makes
+     * the reproduction line from one run be useful on another run.
+     */
+    static void setupSeed(Project project) {
+        if (project.rootProject.ext.has('testSeed')) {
+            /* Skip this if we've already pinned the testSeed. It is important
+             * that this checks the rootProject so that we know we've only ever
+             * initialized one time. */
+            return
+        }
+        String testSeed = System.getProperty('tests.seed')
+        if (testSeed == null) {
+            long seed = new Random(System.currentTimeMillis()).nextLong()
+            testSeed = Long.toUnsignedString(seed, 16).toUpperCase(Locale.ROOT)
+        }
+        /* Set the testSeed on the root project first so other projects can use
+         * it during initialization. */
+        project.rootProject.ext.testSeed = testSeed
+        project.rootProject.subprojects {
+            project.ext.testSeed = testSeed
+        }
+    }
+
     static void replaceTestTask(TaskContainer tasks) {
         Test oldTestTask = tasks.findByPath('test')
         if (oldTestTask == null) {

buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingTask.groovy

@@ -9,6 +9,7 @@ import org.apache.tools.ant.DefaultLogger
 import org.apache.tools.ant.RuntimeConfigurable
 import org.apache.tools.ant.UnknownElement
 import org.gradle.api.DefaultTask
+import org.gradle.api.InvalidUserDataException
 import org.gradle.api.file.FileCollection
 import org.gradle.api.file.FileTreeElement
 import org.gradle.api.internal.tasks.options.Option
@@ -259,8 +260,13 @@ class RandomizedTestingTask extends DefaultTask {
                     }
                 }
                 for (Map.Entry<String, Object> prop : systemProperties) {
+                    if (prop.getKey().equals('tests.seed')) {
+                        throw new InvalidUserDataException('Seed should be ' +
+                            'set on the project instead of a system property')
+                    }
                     sysproperty key: prop.getKey(), value: prop.getValue().toString()
                 }
+                systemProperty 'tests.seed', project.testSeed
                 for (Map.Entry<String, Object> envvar : environmentVariables) {
                     env key: envvar.getKey(), value: envvar.getValue().toString()
                 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -120,6 +120,7 @@ class BuildPlugin implements Plugin<Project> {
                 println "  JDK Version           : ${gradleJavaVersionDetails}"
                 println "  JAVA_HOME             : ${gradleJavaHome}"
             }
+            println "  Random Testing Seed   : ${project.testSeed}"
 
             // enforce gradle version
             GradleVersion minGradle = GradleVersion.version('3.3')
@@ -393,8 +394,11 @@ class BuildPlugin implements Plugin<Project> {
             project.tasks.withType(GenerateMavenPom.class) { GenerateMavenPom t ->
                 // place the pom next to the jar it is for
                 t.destination = new File(project.buildDir, "distributions/${project.archivesBaseName}-${project.version}.pom")
-                // build poms with assemble
-                project.assemble.dependsOn(t)
+                // build poms with assemble (if the assemble task exists)
+                Task assemble = project.tasks.findByName('assemble')
+                if (assemble) {
+                    assemble.dependsOn(t)
+                }
             }
         }
     }
@@ -525,7 +529,12 @@ class BuildPlugin implements Plugin<Project> {
             systemProperty 'tests.logger.level', 'WARN'
             for (Map.Entry<String, String> property : System.properties.entrySet()) {
                 if (property.getKey().startsWith('tests.') ||
-                    property.getKey().startsWith('es.')) {
+                        property.getKey().startsWith('es.')) {
+                    if (property.getKey().equals('tests.seed')) {
+                        /* The seed is already set on the project so we
+                         * shouldn't attempt to override it. */
+                        continue;
+                    }
                     systemProperty property.getKey(), property.getValue()
                 }
             }

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy

@@ -32,6 +32,9 @@ public class DocsTestPlugin extends RestTestPlugin {
     public void apply(Project project) {
         project.pluginManager.apply('elasticsearch.standalone-rest-test')
         super.apply(project)
+        // Docs are published separately so no need to assemble
+        project.tasks.remove(project.assemble)
+        project.build.dependsOn.remove('assemble')
         Map<String, String> defaultSubstitutions = [
             /* These match up with the asciidoc syntax for substitutions but
              * the values may differ. In particular {version} needs to resolve

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy

@@ -127,6 +127,11 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
          */
         Set<String> unconvertedCandidates = new HashSet<>()
 
+        /**
+         * The last non-TESTRESPONSE snippet.
+         */
+        Snippet previousTest
+
         /**
          * Called each time a snippet is encountered. Tracks the snippets and
          * calls buildTest to actually build the test.
@@ -142,6 +147,7 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
             }
             if (snippet.testSetup) {
                 setup(snippet)
+                previousTest = snippet
                 return
             }
             if (snippet.testResponse) {
@@ -150,6 +156,7 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
             }
             if (snippet.test || snippet.console) {
                 test(snippet)
+                previousTest = snippet
                 return
             }
             // Must be an unmarked snippet....
@@ -158,7 +165,18 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
         private void test(Snippet test) {
             setupCurrent(test)
 
-            if (false == test.continued) {
+            if (test.continued) {
+                /* Catch some difficult to debug errors with // TEST[continued]
+                 * and throw a helpful error message. */
+                if (previousTest == null || previousTest.path != test.path) {
+                    throw new InvalidUserDataException("// TEST[continued] " +
+                        "cannot be on first snippet in a file: $test")
+                }
+                if (previousTest != null && previousTest.testSetup) {
+                    throw new InvalidUserDataException("// TEST[continued] " +
+                        "cannot immediately follow // TESTSETUP: $test")
+                }
+            } else {
                 current.println('---')
                 current.println("\"line_$test.start\":")
                 /* The Elasticsearch test runner doesn't support the warnings

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy

@@ -19,6 +19,7 @@
 package org.elasticsearch.gradle.test
 
 import org.apache.tools.ant.taskdefs.condition.Os
+import org.elasticsearch.gradle.Version
 import org.gradle.api.InvalidUserDataException
 import org.gradle.api.Project
 
@@ -143,7 +144,7 @@ class NodeInfo {
             args.add("${esScript}")
         }
 
-        env = [ 'JAVA_HOME' : project.javaHome ]
+        env = ['JAVA_HOME': project.javaHome]
         args.addAll("-E", "node.portsfile=true")
         String collectedSystemProperties = config.systemProperties.collect { key, value -> "-D${key}=${value}" }.join(" ")
         String esJavaOpts = config.jvmArgs.isEmpty() ? collectedSystemProperties : collectedSystemProperties + " " + config.jvmArgs
@@ -158,7 +159,11 @@ class NodeInfo {
             }
         }
         env.put('ES_JVM_OPTIONS', new File(confDir, 'jvm.options'))
-        args.addAll("-E", "path.conf=${confDir}")
+        if (Version.fromString(nodeVersion).major == 5) {
+            args.addAll("-E", "path.conf=${confDir}")
+        } else {
+            args.addAll("--path.conf", "${confDir}")
+        }
         if (!System.properties.containsKey("tests.es.path.data")) {
             args.addAll("-E", "path.data=${-> dataDir.toString()}")
         }

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantPropertiesExtension.groovy

@@ -25,12 +25,6 @@ class VagrantPropertiesExtension {
     @Input
     List<String> boxes
 
-    @Input
-    Long testSeed
-
-    @Input
-    String formattedTestSeed
-
     @Input
     String upgradeFromVersion
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantSupportPlugin.groovy

@@ -0,0 +1,127 @@
+package org.elasticsearch.gradle.vagrant
+
+import org.gradle.api.GradleException
+import org.gradle.api.InvalidUserDataException
+import org.gradle.api.Plugin
+import org.gradle.api.Project
+import org.gradle.process.ExecResult
+import org.gradle.process.internal.ExecException
+
+/**
+ * Global configuration for if Vagrant tasks are supported in this
+ * build environment.
+ */
+class VagrantSupportPlugin implements Plugin<Project> {
+
+    @Override
+    void apply(Project project) {
+        if (project.rootProject.ext.has('vagrantEnvChecksDone') == false) {
+            Map vagrantInstallation = getVagrantInstallation(project)
+            Map virtualBoxInstallation = getVirtualBoxInstallation(project)
+
+            project.rootProject.ext.vagrantInstallation = vagrantInstallation
+            project.rootProject.ext.virtualBoxInstallation = virtualBoxInstallation
+            project.rootProject.ext.vagrantSupported = vagrantInstallation.supported && virtualBoxInstallation.supported
+            project.rootProject.ext.vagrantEnvChecksDone = true
+
+            // Finding that HOME needs to be set when performing vagrant updates
+            String homeLocation = System.getenv("HOME")
+            if (project.rootProject.ext.vagrantSupported && homeLocation == null) {
+                throw new GradleException("Could not locate \$HOME environment variable. Vagrant is enabled " +
+                        "and requires \$HOME to be set to function properly.")
+            }
+        }
+
+        addVerifyInstallationTasks(project)
+    }
+
+    private Map getVagrantInstallation(Project project) {
+        try {
+            ByteArrayOutputStream pipe = new ByteArrayOutputStream()
+            ExecResult runResult = project.exec {
+                commandLine 'vagrant', '--version'
+                standardOutput pipe
+                ignoreExitValue true
+            }
+            String version = pipe.toString().trim()
+            if (runResult.exitValue == 0) {
+                if (version ==~ /Vagrant 1\.(8\.[6-9]|9\.[0-9])+/) {
+                    return [ 'supported' : true ]
+                } else {
+                    return [ 'supported' : false,
+                             'info' : "Illegal version of vagrant [${version}]. Need [Vagrant 1.8.6+]" ]
+                }
+            } else {
+                return [ 'supported' : false,
+                         'info' : "Could not read installed vagrant version:\n" + version ]
+            }
+        } catch (ExecException e) {
+            // Exec still throws this if it cannot find the command, regardless if ignoreExitValue is set.
+            // Swallow error. Vagrant isn't installed. Don't halt the build here.
+            return [ 'supported' : false, 'info' : "Could not find vagrant: " + e.message ]
+        }
+    }
+
+    private Map getVirtualBoxInstallation(Project project) {
+        try {
+            ByteArrayOutputStream pipe = new ByteArrayOutputStream()
+            ExecResult runResult = project.exec {
+                commandLine 'vboxmanage', '--version'
+                standardOutput = pipe
+                ignoreExitValue true
+            }
+            String version = pipe.toString().trim()
+            if (runResult.exitValue == 0) {
+                try {
+                    String[] versions = version.split('\\.')
+                    int major = Integer.parseInt(versions[0])
+                    int minor = Integer.parseInt(versions[1])
+                    if ((major < 5) || (major == 5 && minor < 1)) {
+                        return [ 'supported' : false,
+                                 'info' : "Illegal version of virtualbox [${version}]. Need [5.1+]" ]
+                    } else {
+                        return [ 'supported' : true ]
+                    }
+                } catch (NumberFormatException | ArrayIndexOutOfBoundsException e) {
+                    return [ 'supported' : false,
+                             'info' : "Unable to parse version of virtualbox [${version}]. Required [5.1+]" ]
+                }
+            } else {
+                return [ 'supported': false, 'info': "Could not read installed virtualbox version:\n" + version ]
+            }
+        } catch (ExecException e) {
+            // Exec still throws this if it cannot find the command, regardless if ignoreExitValue is set.
+            // Swallow error. VirtualBox isn't installed. Don't halt the build here.
+            return [ 'supported' : false, 'info' : "Could not find virtualbox: " + e.message ]
+        }
+    }
+
+    private void addVerifyInstallationTasks(Project project) {
+        createCheckVagrantVersionTask(project)
+        createCheckVirtualBoxVersionTask(project)
+    }
+
+    private void createCheckVagrantVersionTask(Project project) {
+        project.tasks.create('vagrantCheckVersion') {
+            description 'Check the Vagrant version'
+            group 'Verification'
+            doLast {
+                if (project.rootProject.vagrantInstallation.supported == false) {
+                    throw new InvalidUserDataException(project.rootProject.vagrantInstallation.info)
+                }
+            }
+        }
+    }
+
+    private void createCheckVirtualBoxVersionTask(Project project) {
+        project.tasks.create('virtualboxCheckVersion') {
+            description 'Check the Virtualbox version'
+            group 'Verification'
+            doLast {
+                if (project.rootProject.virtualBoxInstallation.supported == false) {
+                    throw new InvalidUserDataException(project.rootProject.virtualBoxInstallation.info)
+                }
+            }
+        }
+    }
+}