Merge branch '7.x' into thirdpartytests-7.x

.ci/bwcVersions

@@ -39,6 +39,7 @@ BWC_VERSION:
   - "6.8.7"
   - "6.8.8"
   - "6.8.9"
+  - "6.8.10"
   - "7.0.0"
   - "7.0.1"
   - "7.1.0"
@@ -58,4 +59,5 @@ BWC_VERSION:
   - "7.6.1"
   - "7.6.2"
   - "7.7.0"
+  - "7.7.1"
   - "7.8.0"

CONTRIBUTING.md

@@ -116,10 +116,20 @@ into [disrepair][eclipse].
 
     ./gradlew :run
 
-You can access Elasticsearch with:
+That'll spend a while building Elasticsearch and then it'll start Elasticsearch,
+writing its log above Gradle's status message. We log a lot of stuff on startup,
+specifically these lines tell you that Elasticsearch is ready:
+
+    [2020-05-29T14:50:35,167][INFO ][o.e.h.AbstractHttpServerTransport] [runTask-0] publish_address {127.0.0.1:9200}, bound_addresses {[::1]:9200}, {127.0.0.1:9200}
+    [2020-05-29T14:50:35,169][INFO ][o.e.n.Node               ] [runTask-0] started
+
+But to be honest its typically easier to wait until the console stopps scrolling
+and then run `curl` in another window like this:
 
     curl -u elastic:password localhost:9200
 
+
+
 ### Importing the project into IntelliJ IDEA
 
 Elasticsearch builds using Java 14. When importing into IntelliJ you will need

Vagrantfile

@@ -495,22 +495,12 @@ JAVA
       rm -rf /tmp/bats
     }
 
-    cat \<\<VARS > /etc/profile.d/elasticsearch_vars.sh
-export ZIP=/elasticsearch/distribution/zip/build/distributions
-export TAR=/elasticsearch/distribution/tar/build/distributions
-export RPM=/elasticsearch/distribution/rpm/build/distributions
-export DEB=/elasticsearch/distribution/deb/build/distributions
-export PACKAGING_TESTS=/project/build/packaging/tests
-VARS
     cat \<\<SUDOERS_VARS > /etc/sudoers.d/elasticsearch_vars
-Defaults   env_keep += "ZIP"
-Defaults   env_keep += "TAR"
-Defaults   env_keep += "RPM"
-Defaults   env_keep += "DEB"
-Defaults   env_keep += "PACKAGING_ARCHIVES"
-Defaults   env_keep += "PACKAGING_TESTS"
 Defaults   env_keep += "BATS_UTILS"
 Defaults   env_keep += "BATS_TESTS"
+Defaults   env_keep += "BATS_PLUGINS"
+Defaults   env_keep += "BATS_UPGRADE"
+Defaults   env_keep += "PACKAGE_NAME"
 Defaults   env_keep += "JAVA_HOME"
 Defaults   env_keep += "SYSTEM_JAVA_HOME"
 SUDOERS_VARS

build.gradle

@@ -30,7 +30,6 @@ import org.gradle.plugins.ide.eclipse.model.AccessRule
 import org.gradle.plugins.ide.eclipse.model.SourceFolder
 import org.gradle.util.DistributionLocator
 import org.gradle.util.GradleVersion
-
 import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure
 
 plugins {
@@ -46,6 +45,8 @@ apply from: 'gradle/runtime-jdk-provision.gradle'
 apply from: 'gradle/ide.gradle'
 apply from: 'gradle/forbidden-dependencies.gradle'
 apply from: 'gradle/formatting.gradle'
+apply from: 'gradle/local-distribution.gradle'
+apply from: 'gradle/fips.gradle'
 
 // common maven publishing configuration
 allprojects {
@@ -54,8 +55,6 @@ allprojects {
   description = "Elasticsearch subproject ${project.path}"
 }
 
-BuildPlugin.configureRepositories(project)
-
 String licenseCommit
 if (VersionProperties.elasticsearch.toString().endsWith('-SNAPSHOT')) {
   licenseCommit = BuildParams.gitRevision ?: "master" // leniency for non git builds
@@ -116,12 +115,12 @@ subprojects {
   }
 }
 
-task updateCIBwcVersions() {
+tasks.register("updateCIBwcVersions") {
   doLast {
     File yml = file(".ci/bwcVersions")
     yml.text = ""
     yml << "BWC_VERSION:\n"
-    versions.indexCompatible.each {
+    BuildParams.bwcVersions.indexCompatible.each {
       yml << "  - \"$it\"\n"
     }
   }
@@ -150,7 +149,7 @@ allprojects {
   }
 }
 
-task verifyVersions {
+tasks.register("verifyVersions") {
   doLast {
     if (gradle.startParameter.isOffline()) {
       throw new GradleException("Must run in online mode to verify versions")
@@ -202,18 +201,18 @@ subprojects {
   ext.bwc_tests_enabled = bwc_tests_enabled
 }
 
-task verifyBwcTestsEnabled {
+tasks.register("verifyBwcTestsEnabled") {
   doLast {
     if (bwc_tests_enabled == false) {
       throw new GradleException('Bwc tests are disabled. They must be re-enabled after completing backcompat behavior backporting.')
     }
   }
 }
 
-task branchConsistency {
+tasks.register("branchConsistency") {
   description 'Ensures this branch is internally consistent. For example, that versions constants match released versions.'
   group 'Verification'
-  dependsOn verifyVersions, verifyBwcTestsEnabled
+  dependsOn ":verifyVersions", ":verifyBwcTestsEnabled"
 }
 
 allprojects {
@@ -414,7 +413,7 @@ class Run extends DefaultTask {
   }
 }
 
-task run(type: Run) {
+tasks.register("run", Run) {
   dependsOn ':distribution:run'
   description = 'Runs elasticsearch in the foreground'
   group = 'Verification'
@@ -493,7 +492,7 @@ allprojects {
     if (realTask == null) {
       return
     }
-    project.tasks.create(taskName) {
+    project.tasks.register(taskName) {
       doLast {
         println("${realTask.path} dependencies:")
         for (Task dep : realTask.getTaskDependencies().getDependencies(realTask)) {

buildSrc/build.gradle

@@ -23,13 +23,14 @@ import org.gradle.util.GradleVersion
 plugins {
   id 'java-gradle-plugin'
   id 'groovy'
+  id 'java-test-fixtures'
 }
 
 group = 'org.elasticsearch.gradle'
 
 String minimumGradleVersion = file('src/main/resources/minimumGradleVersion').text.trim()
 if (GradleVersion.current() < GradleVersion.version(minimumGradleVersion)) {
-   throw new GradleException("Gradle ${minimumGradleVersion}+ is required to build elasticsearch")
+  throw new GradleException("Gradle ${minimumGradleVersion}+ is required to build elasticsearch")
 }
 
 if (project == rootProject) {
@@ -48,7 +49,7 @@ if (project == rootProject) {
 Properties props = VersionPropertiesLoader.loadBuildSrcVersion(project.file('version.properties'))
 version = props.getProperty("elasticsearch")
 
-task generateVersionProperties(type: WriteProperties) {
+def generateVersionProperties = tasks.register("generateVersionProperties", WriteProperties) {
   outputFile = "${buildDir}/version.properties"
   comment = 'Generated version properties'
   properties(props)
@@ -69,6 +70,11 @@ if (JavaVersion.current() < JavaVersion.VERSION_11) {
 sourceSets {
   // We have a few classes that need to be compiled for older java versions
   minimumRuntime {}
+
+  integTest {
+    compileClasspath += sourceSets["main"].output + configurations["testRuntimeClasspath"]
+    runtimeClasspath += output + compileClasspath
+  }
 }
 
 configurations {
@@ -122,8 +128,10 @@ dependencies {
   compile 'com.networknt:json-schema-validator:1.0.36'
   compileOnly "com.puppycrawl.tools:checkstyle:${props.getProperty('checkstyle')}"
   testCompile "com.puppycrawl.tools:checkstyle:${props.getProperty('checkstyle')}"
-  testCompile "junit:junit:${props.getProperty('junit')}"
-  testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}"
+  testFixturesApi "junit:junit:${props.getProperty('junit')}"
+  testFixturesApi "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}"
+  testFixturesApi gradleApi()
+  testFixturesApi gradleTestKit()
   testCompile 'com.github.tomakehurst:wiremock-jre8-standalone:2.23.2'
   testCompile 'org.mockito:mockito-core:1.9.5'
   minimumRuntimeCompile "junit:junit:${props.getProperty('junit')}"
@@ -157,7 +165,7 @@ if (project == rootProject) {
 // to enforce precommit checks like forbidden apis, as well as setup publishing
 if (project != rootProject) {
   apply plugin: 'elasticsearch.build'
-  apply plugin: 'nebula.maven-base-publish'
+  apply plugin: 'elasticsearch.publish'
 
   allprojects {
     targetCompatibility = 10
@@ -170,10 +178,11 @@ if (project != rootProject) {
   // build-tools is not ready for primetime with these...
   dependencyLicenses.enabled = false
   dependenciesInfo.enabled = false
-  disableTasks('forbiddenApisMain', 'forbiddenApisMinimumRuntime', 'forbiddenApisTest')
+  disableTasks('forbiddenApisMain', 'forbiddenApisMinimumRuntime',
+    'forbiddenApisTest', 'forbiddenApisIntegTest', 'forbiddenApisTestFixtures')
   jarHell.enabled = false
   thirdPartyAudit.enabled = false
-  if (Boolean.parseBoolean(System.getProperty("tests.fips.enabled"))){
+  if (Boolean.parseBoolean(System.getProperty("tests.fips.enabled"))) {
     test.enabled = false
   }
 
@@ -237,13 +246,15 @@ if (project != rootProject) {
     }
   }
 
-  task integTest(type: Test) {
+  tasks.register("integTest", Test) {
     inputs.dir(file("src/testKit")).withPropertyName("testkit dir").withPathSensitivity(PathSensitivity.RELATIVE)
     systemProperty 'test.version_under_test', version
     onlyIf { org.elasticsearch.gradle.info.BuildParams.inFipsJvm == false }
     maxParallelForks = System.getProperty('tests.jvms', org.elasticsearch.gradle.info.BuildParams.defaultParallel.toString()) as Integer
+    testClassesDirs = sourceSets.integTest.output.classesDirs
+    classpath = sourceSets.integTest.runtimeClasspath
   }
-  check.dependsOn(integTest)
+  check.dependsOn("integTest")
 
   // for now we hardcode the tests for our build to use the gradle jvm.
   tasks.withType(Test).configureEach {
@@ -257,6 +268,11 @@ if (project != rootProject) {
   afterEvaluate {
     generatePomFileForPluginMavenPublication.enabled = false
   }
+
+  publishing.publications.named("nebula").configure {
+    suppressPomMetadataWarningsFor("testFixturesApiElements")
+    suppressPomMetadataWarningsFor("testFixturesRuntimeElements")
+  }
 }
 
 // Define this here because we need it early.

buildSrc/reaper/build.gradle

@@ -1,7 +1,7 @@
 apply plugin: 'java'
 
 jar {
-  archiveName = "${project.name}.jar"
+  archiveFileName = "${project.name}.jar"
   manifest {
     attributes 'Main-Class': 'org.elasticsearch.gradle.reaper.Reaper'
   }

buildSrc/src/test/java/org/elasticsearch/gradle/ExportElasticsearchBuildResourcesTaskIT.java → buildSrc/src/integTest/java/org/elasticsearch/gradle/ExportElasticsearchBuildResourcesTaskIT.java

@@ -40,22 +40,12 @@ public void testUpToDateWithSourcesConfigured() {
         assertBuildFileExists(result, PROJECT_NAME, "build-tools-exported/checkstyle_suppressions.xml");
     }
 
-    public void testImplicitTaskDependencyCopy() {
-        BuildResult result = getGradleRunner(PROJECT_NAME).withArguments("clean", "sampleCopyAll", "-s", "-i").build();
+    public void testOutputAsInput() {
+        BuildResult result = getGradleRunner(PROJECT_NAME).withArguments("clean", "sampleCopy", "-s", "-i").build();
 
-        assertTaskSuccessful(result, ":buildResources");
-        assertTaskSuccessful(result, ":sampleCopyAll");
-        assertBuildFileExists(result, PROJECT_NAME, "sampleCopyAll/checkstyle.xml");
-        // This is a side effect of compile time reference
-        assertBuildFileExists(result, PROJECT_NAME, "sampleCopyAll/checkstyle_suppressions.xml");
-    }
-
-    public void testImplicitTaskDependencyInputFileOfOther() {
-        BuildResult result = getGradleRunner(PROJECT_NAME).withArguments("clean", "sample", "-s", "-i").build();
-
-        assertTaskSuccessful(result, ":sample");
-        assertBuildFileExists(result, PROJECT_NAME, "build-tools-exported/checkstyle.xml");
-        assertBuildFileExists(result, PROJECT_NAME, "build-tools-exported/checkstyle_suppressions.xml");
+        assertTaskSuccessful(result, ":sampleCopy");
+        assertBuildFileExists(result, PROJECT_NAME, "sampleCopy/checkstyle.xml");
+        assertBuildFileExists(result, PROJECT_NAME, "sampleCopy/checkstyle_suppressions.xml");
     }
 
     public void testIncorrectUsage() {