Merge branch 'feature-pwd-protected-keystore' into elasticsearch-keys…

…tore-init-passphrase

.ci/bwcVersions

@@ -0,0 +1,10 @@
+BWC_VERSION:
+  - "7.0.0"
+  - "7.0.1"
+  - "7.1.0"
+  - "7.1.1"
+  - "7.2.0"
+  - "7.2.1"
+  - "7.3.0"
+  - "7.4.0"
+  - "8.0.0"

build.gradle

@@ -29,16 +29,13 @@ import org.gradle.util.DistributionLocator
 import org.gradle.plugins.ide.eclipse.model.SourceFolder
 
 plugins {
-    id 'com.gradle.build-scan' version '2.2.1'
+    id 'com.gradle.build-scan' version '2.3'
     id 'base'
     id 'elasticsearch.global-build-info'
 }
-if (Boolean.valueOf(project.findProperty('org.elasticsearch.acceptScanTOS') ?: "false")) {
-    buildScan {
-        termsOfServiceUrl = 'https://gradle.com/terms-of-service'
-        termsOfServiceAgree = 'yes'
-    }
-}
+
+apply plugin: 'nebula.info-scm'
+apply from: 'gradle/build-scan.gradle'
 
 // common maven publishing configuration
 allprojects {
@@ -49,7 +46,6 @@ allprojects {
 
 BuildPlugin.configureRepositories(project)
 
-apply plugin: 'nebula.info-scm'
 String licenseCommit
 if (VersionProperties.elasticsearch.toString().endsWith('-SNAPSHOT')) {
   licenseCommit = scminfo.change ?: "master" // leniency for non git builds
@@ -107,6 +103,17 @@ subprojects {
  * logic in VersionUtils.java. */
 BwcVersions versions = new BwcVersions(file('server/src/main/java/org/elasticsearch/Version.java').readLines('UTF-8'))
 
+task updateCIBwcVersions() {
+    doLast {
+        File yml = file(".ci/bwcVersions")
+        yml.text = ""
+        yml << "BWC_VERSION:\n"
+        versions.indexCompatible.each {
+            yml << "  - \"$it\"\n"
+        }
+    }
+}
+
 // build metadata from previous build, contains eg hashes for bwc builds
 String buildMetadataValue = System.getenv('BUILD_METADATA')
 if (buildMetadataValue == null) {
@@ -150,6 +157,12 @@ task verifyVersions {
                       .collect { Version.fromString(it) }
       )
     }
+    String ciYml = file(".ci/bwcVersions").text
+    bwcVersions.indexCompatible.each {
+        if (ciYml.contains("\"$it\"\n") == false) {
+            throw new Exception(".ci/bwcVersions is outdated, run `./gradlew updateCIBwcVersions` and check in the results");
+        }
+    }
   }
 }
 
@@ -160,8 +173,8 @@ task verifyVersions {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = true
-final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
+boolean bwc_tests_enabled = false
+final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/43197"
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -825,7 +825,7 @@ class BuildPlugin implements Plugin<Project> {
 
                 test.executable = "${ext.get('runtimeJavaHome')}/bin/java"
                 test.workingDir = project.file("${project.buildDir}/testrun/${test.name}")
-                test.maxParallelForks = project.rootProject.extensions.getByType(ExtraPropertiesExtension).get('defaultParallel') as Integer
+                test.maxParallelForks = System.getProperty('tests.jvms', project.rootProject.extensions.extraProperties.get('defaultParallel').toString()) as Integer
 
                 test.exclude '**/*$*.class'
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy

@@ -21,11 +21,9 @@ package org.elasticsearch.gradle.doc
 import org.elasticsearch.gradle.OS
 import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.VersionProperties
-import org.elasticsearch.gradle.test.ClusterFormationTasks
 import org.elasticsearch.gradle.test.RestTestPlugin
 import org.gradle.api.Project
 import org.gradle.api.Task
-
 /**
  * Sets up tests for documentation.
  */
@@ -38,7 +36,7 @@ public class DocsTestPlugin extends RestTestPlugin {
         super.apply(project)
         String distribution = System.getProperty('tests.distribution', 'default')
         // The distribution can be configured with -Dtests.distribution on the command line
-        project.testClusters.integTest.distribution = distribution.toUpperCase()
+        project.testClusters.integTest.testDistribution = distribution.toUpperCase()
         project.testClusters.integTest.nameCustomization = { it.replace("integTest", "node") }
         // Docs are published separately so no need to assemble
         project.tasks.assemble.enabled = false

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy

@@ -353,7 +353,7 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
 
         private void testSetup(Snippet snippet) {
             if (lastDocsPath == snippet.path) {
-                throw new InvalidUserDataException("$snippet: wasn't first")
+                throw new InvalidUserDataException("$snippet: wasn't first. TESTSETUP can only be used in the first snippet of a document.")
             }
             setupCurrent(snippet)
             current.println('---')

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/AntFixture.groovy

@@ -58,6 +58,9 @@ public class AntFixture extends AntTask implements Fixture {
     @Input
     boolean useShell = false
 
+    @Input
+    int maxWaitInSeconds = 30
+
     /**
      * A flag to indicate whether the fixture should be run in the foreground, or spawned.
      * It is protected so subclasses can override (eg RunTask).
@@ -128,7 +131,7 @@ public class AntFixture extends AntTask implements Fixture {
 
         String failedProp = "failed${name}"
         // first wait for resources, or the failure marker from the wrapper script
-        ant.waitfor(maxwait: '30', maxwaitunit: 'second', checkevery: '500', checkeveryunit: 'millisecond', timeoutproperty: failedProp) {
+        ant.waitfor(maxwait: maxWaitInSeconds, maxwaitunit: 'second', checkevery: '500', checkeveryunit: 'millisecond', timeoutproperty: failedProp) {
             or {
                 resourceexists {
                     file(file: failureMarker.toString())

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

@@ -18,15 +18,14 @@
  */
 package org.elasticsearch.gradle.test
 
-import org.elasticsearch.gradle.VersionProperties
+
 import org.elasticsearch.gradle.testclusters.ElasticsearchCluster
 import org.elasticsearch.gradle.testclusters.TestClustersPlugin
 import org.gradle.api.DefaultTask
 import org.gradle.api.Task
 import org.gradle.api.execution.TaskExecutionAdapter
 import org.gradle.api.logging.Logger
 import org.gradle.api.logging.Logging
-import org.gradle.api.specs.Specs
 import org.gradle.api.tasks.Copy
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.TaskState
@@ -37,7 +36,6 @@ import org.gradle.plugins.ide.idea.IdeaPlugin
 import java.nio.charset.StandardCharsets
 import java.nio.file.Files
 import java.util.stream.Stream
-
 /**
  * A wrapper task around setting up a cluster and running rest tests.
  */
@@ -69,8 +67,6 @@ class RestIntegTestTask extends DefaultTask {
         } else {
             project.testClusters {
                 "$name" {
-                    distribution = 'INTEG_TEST'
-                    version = VersionProperties.elasticsearch
                     javaHome = project.file(project.ext.runtimeJavaHome)
                 }
             }

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

@@ -324,9 +324,9 @@ class VagrantTestPlugin implements Plugin<Project> {
                      fi
                      
                      if [ -z "\$SYSTEM_JAVA_HOME" ]; then
-                       export SYSTEM_JAVA_HOME="${-> convertPath(project, linuxSystemJdk.toString()) }"
+                       export SYSTEM_JAVA_HOME="${-> convertLinuxPath(project, linuxSystemJdk.toString()) }"
                      fi
-                     "${-> convertPath(project, linuxGradleJdk.toString()) }"/bin/java -cp "\$PACKAGING_TESTS/*" org.elasticsearch.packaging.VMTestRunner "\${test_args[@]}"
+                     "${-> convertLinuxPath(project, linuxGradleJdk.toString()) }"/bin/java -cp "\$PACKAGING_TESTS/*" org.elasticsearch.packaging.VMTestRunner "\${test_args[@]}"
                      """
         }
         Task createWindowsRunnerScript = project.tasks.create('createWindowsRunnerScript', FileContentsTask) {
@@ -335,14 +335,20 @@ class VagrantTestPlugin implements Plugin<Project> {
             // the use of $args rather than param() here is deliberate because the syntax for array (multivalued) parameters is likely
             // a little trappy for those unfamiliar with powershell
             contents """\
-                     if (\$args.Count -eq 0) {
-                       \$testArgs = @("${-> project.extensions.esvagrant.testClass}")
-                     } else {
-                       \$testArgs = \$args
+                     try {
+                         if (\$args.Count -eq 0) {
+                           \$testArgs = @("${-> project.extensions.esvagrant.testClass}")
+                         } else {
+                           \$testArgs = \$args
+                         }
+                         \$Env:SYSTEM_JAVA_HOME = "${-> convertWindowsPath(project, windowsSystemJdk.toString()) }"
+                         & "${-> convertWindowsPath(project, windowsGradleJdk.toString()) }/bin/java" -cp "\$Env:PACKAGING_TESTS/*" org.elasticsearch.packaging.VMTestRunner @testArgs
+                         exit \$LASTEXITCODE
+                     } catch {
+                         # catch if we have a failure to even run the script at all above, equivalent to set -e, sort of
+                         echo "\$_.Exception.Message"
+                         exit 1
                      }
-                     \$Env:SYSTEM_JAVA_HOME = "${-> convertPath(project, windowsSystemJdk.toString()) }"
-                     & "${-> convertPath(project, windowsGradleJdk.toString()) }"/bin/java -cp "\$Env:PACKAGING_TESTS/*" org.elasticsearch.packaging.VMTestRunner @testArgs
-                     exit \$LASTEXITCODE
                      """
         }
 
@@ -578,7 +584,7 @@ class VagrantTestPlugin implements Plugin<Project> {
 
             if (LINUX_BOXES.contains(box)) {
                 Task batsPackagingTest = project.tasks.create("vagrant${boxTask}#batsPackagingTest", BatsOverVagrantTask) {
-                    remoteCommand "export SYSTEM_JAVA_HOME=\"${-> convertPath(project, linuxSystemJdk.toString())}\"; " + BATS_TEST_COMMAND
+                    remoteCommand "export SYSTEM_JAVA_HOME=\"${-> convertLinuxPath(project, linuxSystemJdk.toString())}\"; " + BATS_TEST_COMMAND
                     boxName box
                     environmentVars vagrantEnvVars
                     dependsOn up, setupPackagingTest, linuxSystemJdk
@@ -625,7 +631,7 @@ class VagrantTestPlugin implements Plugin<Project> {
                 // https://github.com/hashicorp/vagrant/blob/9c299a2a357fcf87f356bb9d56e18a037a53d138/plugins/communicators/winrm/communicator.rb#L195-L225
                 // https://devops-collective-inc.gitbooks.io/secrets-of-powershell-remoting/content/manuscript/accessing-remote-computers.html
                 javaPackagingTest.command = 'winrm'
-                javaPackagingTest.args = ['--elevated', '--command', 'powershell -File "$Env:PACKAGING_TESTS/run-tests.ps1"']
+                javaPackagingTest.args = ['--elevated', '--command', '& "$Env:PACKAGING_TESTS/run-tests.ps1"; exit $LASTEXITCODE']
             }
 
             TaskExecutionAdapter javaPackagingReproListener = createReproListener(project, javaPackagingTest.path)
@@ -658,7 +664,10 @@ class VagrantTestPlugin implements Plugin<Project> {
     }
 
     // convert the given path from an elasticsearch repo path to a VM path
-    private String convertPath(Project project, String path) {
+    private String convertLinuxPath(Project project, String path) {
         return "/elasticsearch/" + project.rootDir.toPath().relativize(Paths.get(path));
     }
+    private String convertWindowsPath(Project project, String path) {
+        return "C:\\elasticsearch\\" + project.rootDir.toPath().relativize(Paths.get(path)).toString().replace('/', '\\');
+    }
 }

buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java

@@ -54,6 +54,7 @@
  */
 public class DistributionDownloadPlugin implements Plugin<Project> {
 
+    private static final String CONTAINER_NAME = "elasticsearch_distributions";
     private static final String FAKE_IVY_GROUP = "elasticsearch-distribution";
     private static final String DOWNLOAD_REPO_NAME = "elasticsearch-downloads";
 
@@ -67,7 +68,7 @@ public void apply(Project project) {
             Configuration extractedConfiguration = project.getConfigurations().create("es_distro_extracted_" + name);
             return new ElasticsearchDistribution(name, project.getObjects(), fileConfiguration, extractedConfiguration);
         });
-        project.getExtensions().add("elasticsearch_distributions", distributionsContainer);
+        project.getExtensions().add(CONTAINER_NAME, distributionsContainer);
 
         setupDownloadServiceRepo(project);
 
@@ -78,6 +79,11 @@ public void apply(Project project) {
         project.afterEvaluate(this::setupDistributions);
     }
 
+    @SuppressWarnings("unchecked")
+    public static NamedDomainObjectContainer<ElasticsearchDistribution> getContainer(Project project) {
+        return (NamedDomainObjectContainer<ElasticsearchDistribution>) project.getExtensions().getByName(CONTAINER_NAME);
+    }
+
     // pkg private for tests
     void setupDistributions(Project project) {
         for (ElasticsearchDistribution distribution : distributionsContainer) {

buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java

@@ -20,14 +20,17 @@
 package org.elasticsearch.gradle;
 
 import org.gradle.api.Buildable;
+import org.gradle.api.Project;
 import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.file.FileTree;
 import org.gradle.api.model.ObjectFactory;
 import org.gradle.api.provider.Property;
 import org.gradle.api.tasks.TaskDependency;
 
 import java.io.File;
 import java.util.Iterator;
 import java.util.Locale;
+import java.util.concurrent.Callable;
 
 public class ElasticsearchDistribution implements Buildable {
 
@@ -65,7 +68,7 @@ public String toString() {
     }
 
     // package private to tests can use
-    static final Platform CURRENT_PLATFORM = OS.<Platform>conditional()
+    public static final Platform CURRENT_PLATFORM = OS.<Platform>conditional()
         .onLinux(() -> Platform.LINUX)
         .onWindows(() -> Platform.WINDOWS)
         .onMac(() -> Platform.DARWIN)
@@ -90,6 +93,10 @@ public TaskDependency getBuildDependencies() {
             return configuration.getBuildDependencies();
         }
 
+        public FileTree getFileTree(Project project) {
+            return project.fileTree((Callable<File>) configuration::getSingleFile);
+        }
+
         @Override
         public String toString() {
             return configuration.getSingleFile().toString();