Update the QA project to use test cluster api (#1430)

Also update the project to run fixtures on the java 8 runtime. Java
home can now be set on cluster configs and is used by fixtures.

A cluster can be configured with a dependent ElasticsearchCluster.
This cluster is depended on for any configuration tasks in case its
cluster address is used.

Settings are now stored as objects in cluster configurations instead of
Strings since they may be GStrings with closures inside of them that
must be resolved ONLY at configuration writing time. This is needed to
resolve test cluster http addresses which are only discoverable once
the cluster is started.

In order to make this resolution process simpler, removed most of the
Map<String, String> types in the code and replaced them with the
FileSettings type which can resolve closures as needed when writing the
configuration to a file.

Upgraded Spark QA runtime to 2.3.4 as the older version has been removed
from apache's mirrors.

AbstractClusterTask now extends DefaultTestClustersTask, as it is the
only way to ensure that Elasticsearch is still running when starting the
task.

In order to configure JAVA_HOME consistently, each AbstractClusterTask
now has its environment variables resolved in its super class instead of
with duplicated code in each task.

buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/fixture/hadoop/ConfigFormats.groovy

@@ -19,28 +19,30 @@
 
 package org.elasticsearch.hadoop.gradle.fixture.hadoop
 
+import static org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.SettingsContainer.FileSettings
+
 class ConfigFormats {
 
     static Closure<String> hadoopXML() {
-        return { Map conf ->
-            String props = conf.collect { key, value ->
+        return { FileSettings conf ->
+            String props = conf.resolve().collect { key, value ->
                 "<property>\n\t\t<name>${key}</name>\n\t\t<value>${value}</value>\n\t</property>"
             }.join("\n\t")
             return "<configuration>\n\t${props}\n</configuration>"
         }
     }
 
     static Closure<String> propertyFile() {
-        return { Map conf ->
-            conf.collect { key, value ->
+        return { FileSettings conf ->
+            conf.resolve().collect { key, value ->
                 "${key}=${value}"
             }.join("\n")
         }
     }
 
     static Closure<String> whiteSpaced() {
-        return { Map conf ->
-            conf.collect { key, value ->
+        return { FileSettings conf ->
+            conf.resolve().collect { key, value ->
                 "${key}   ${value}"
             }.join("\n")
         }

buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/fixture/hadoop/HadoopClusterFormationTasks.groovy

@@ -23,6 +23,7 @@ import org.apache.tools.ant.DefaultLogger
 import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.test.Fixture
+import org.elasticsearch.gradle.testclusters.DefaultTestClustersTask
 import org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.HadoopClusterConfiguration
 import org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.InstanceConfiguration
 import org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.RoleConfiguration
@@ -42,6 +43,8 @@ import org.apache.tools.ant.taskdefs.condition.Os
 
 import java.nio.file.Paths
 
+import static org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.SettingsContainer.FileSettings
+
 /**
  * A helper for creating tasks to build a cluster that is used by a task, and tear down the cluster
  * when the task is finished.
@@ -68,8 +71,6 @@ class HadoopClusterFormationTasks {
      * Adds dependent tasks to the given task to start and stop a cluster with the given configuration.
      * <p>
      * Returns a list of NodeInfo objects for each node in the cluster.
-     *
-     * Based on {@link org.elasticsearch.gradle.test.ClusterFormationTasks}
      */
     static List<InstanceInfo> setup(Project project, HadoopClusterConfiguration clusterConfiguration) {
         String prefix = clusterConfiguration.getName()
@@ -342,13 +343,16 @@ class HadoopClusterFormationTasks {
 
     static Task configureWriteConfigTask(String name, Project project, Task setup, InstanceInfo node) {
         // Add all node level configs to node Configuration
-        return project.tasks.create(name: name, type: DefaultTask, dependsOn: setup) {
+        return project.tasks.create(name: name, type: DefaultTestClustersTask, dependsOn: setup) {
             group = 'hadoopFixture'
+            if (node.elasticsearchCluster != null) {
+                useCluster(node.elasticsearchCluster)
+            }
             doFirst {
                 // Write each config file needed
                 node.configFiles.forEach { configFile ->
                     String configName = configFile.getName()
-                    Map<String, String> configFileEntries = node.configContents.get(configName)
+                    FileSettings configFileEntries = node.configContents.get(configName)
                     if (configFileEntries == null) {
                         throw new GradleException("Could not find contents of [${configFile}] settings file from deployment options.")
                     }
@@ -387,7 +391,6 @@ class HadoopClusterFormationTasks {
         return project.tasks.create(name: name, type: LoggedExec, dependsOn: setup) { Exec exec ->
             exec.group = 'hadoopFixture'
             exec.workingDir node.cwd
-            exec.environment 'JAVA_HOME', node.getJavaHome()
             exec.environment(node.env)
 
             // Configure HADOOP_OPTS (or similar env) - adds system properties, assertion flags, remote debug etc

buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/fixture/hadoop/InstanceInfo.groovy

@@ -20,7 +20,7 @@
 package org.elasticsearch.hadoop.gradle.fixture.hadoop
 
 import org.apache.tools.ant.taskdefs.condition.Os
-import org.elasticsearch.gradle.info.BuildParams
+import org.elasticsearch.gradle.testclusters.ElasticsearchCluster
 import org.elasticsearch.hadoop.gradle.util.WaitForURL
 import org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.InstanceConfiguration
 import org.gradle.api.GradleException
@@ -30,6 +30,8 @@ import java.nio.file.Path
 import java.nio.file.Paths
 import java.util.concurrent.TimeUnit
 
+import static org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.SettingsContainer.FileSettings
+
 /**
  * Generic information for any process running in a hadoop ecosystem.
  *
@@ -70,7 +72,7 @@ class InstanceInfo {
     /** The config files */
     List<File> configFiles
 
-    Map<String, Map<String, String>> configContents
+    Map<String, FileSettings> configContents
 
     /** Closure that renders the contents of the config file */
     Closure<String> configFileFormatter
@@ -84,8 +86,8 @@ class InstanceInfo {
     /** stdout/stderr log of the service process for this instance */
     File startLog
 
-    /** Major version of java this node runs with, or {@code null} if using the runtime java version */
-    Integer javaVersion
+    /** Location of the java installation to use when running processes **/
+    String javaHome
 
     /** environment variables to start the node with */
     Map<String, String> env
@@ -108,6 +110,9 @@ class InstanceInfo {
     /** buffer for ant output when starting this node */
     ByteArrayOutputStream buffer = new ByteArrayOutputStream()
 
+    /** Elasticsearch cluster dependency for tasks **/
+    ElasticsearchCluster elasticsearchCluster
+
     /**
      * A closure to call before the cluster is considered ready. The closure is passed the node info,
      * as well as a groovy AntBuilder, to enable running ant condition checks. The default wait
@@ -155,13 +160,16 @@ class InstanceInfo {
         startLog = new File(cwd, 'run.log')
 
         // We just default to the current runtime at this time
-        javaVersion = 8
+        javaHome = config.getJavaHome()
 
         // Prepare Environment
         env = [:]
         env.putAll(config.getEnvironmentVariables())
         config.getServiceDescriptor().finalizeEnv(env, config)
 
+        // Add JAVA_HOME to the environment
+        env['JAVA_HOME'] = javaHome
+
         // Prepare startup command and arguments
         args = []
         List<String> startCommandLine = config.getServiceDescriptor().startCommand(config)
@@ -202,6 +210,8 @@ class InstanceInfo {
         if (Os.isFamily(Os.FAMILY_WINDOWS)) {
             args.add('"') // end the entire command, quoted
         }
+
+        this.elasticsearchCluster = config.getElasticsearchCluster()
     }
 
     Path binPath() {
@@ -235,19 +245,13 @@ class InstanceInfo {
         throw new UnsupportedOperationException("JNAKernal32Library is compiled for Java 10 and up.")
     }
 
-    /** Return the java home used by this node. */
-    String getJavaHome() {
-        return javaVersion == null ? project.runtimeJavaHome : BuildParams.javaVersions.find { it.version == javaVersion }.javaHome.absolutePath
-    }
-
     /** Returns debug string for the command that started this node. */
     String getCommandString() {
         String commandString = "\nService ${config.serviceDescriptor.serviceName()}: ${config.roleDescriptor.roleName()} configuration:\n"
         commandString += "|-----------------------------------------\n"
         commandString += "|  cwd: ${cwd}\n"
         commandString += "|  command: ${executable} ${args.join(' ')}\n"
         commandString += '|  environment:\n'
-        commandString += "|    JAVA_HOME: ${javaHome}\n"
         env.each { k, v -> commandString += "|    ${k}: ${v}\n" }
         commandString += "|\n|  [${backgroundScript.name}]\n"
         backgroundScript.eachLine('UTF-8', { line -> commandString += "    ${line}\n"})

buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/fixture/hadoop/ServiceDescriptor.groovy

@@ -24,6 +24,8 @@ import org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.InstanceConfiguration
 import org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.ServiceConfiguration
 import org.elasticsearch.hadoop.gradle.tasks.ApacheMirrorDownload
 
+import static org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.SettingsContainer.FileSettings
+
 /**
  * Describes deployment characteristics for different Hadoop ecosystem projects.
  *
@@ -100,7 +102,7 @@ interface ServiceDescriptor {
     /**
      * Collect all configuration entries, setting defaults for the service, role, and instance.
      */
-    Map<String, Map<String, String>> collectConfigFilesContents(InstanceConfiguration configuration)
+    Map<String, FileSettings> collectConfigFilesContents(InstanceConfiguration configuration)
 
     /**
      * Closure that formats a configuration map into a String for the config file contents.
@@ -110,7 +112,7 @@ interface ServiceDescriptor {
     /**
      * Produces the HTTP/S URI to reach the web front end for a running instance, or null if there is no web interface.
      */
-    String httpUri(InstanceConfiguration configuration, Map<String, Map<String, String>> configFileContents)
+    String httpUri(InstanceConfiguration configuration, Map<String, FileSettings> configFileContents)
 
     /**
      * The command line to use for starting the given role and instance.

buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/fixture/hadoop/conf/EndProcessConfiguration.groovy

@@ -19,6 +19,9 @@
 
 package org.elasticsearch.hadoop.gradle.fixture.hadoop.conf
 
+import org.elasticsearch.gradle.testclusters.ElasticsearchCluster
+import org.gradle.api.Project
+
 /**
  * Provides defaults and can be slotted in as the last parent configuration in a chain.
  *
@@ -29,8 +32,8 @@ package org.elasticsearch.hadoop.gradle.fixture.hadoop.conf
  */
 class EndProcessConfiguration extends ProcessConfiguration {
 
-    EndProcessConfiguration() {
-        super(null)
+    EndProcessConfiguration(Project project) {
+        super(project)
     }
 
     @Override
@@ -68,6 +71,11 @@ class EndProcessConfiguration extends ProcessConfiguration {
         return Collections.emptyList()
     }
 
+    @Override
+    String getJavaHome() {
+        return project.runtimeJavaHome
+    }
+
     @Override
     String getJvmArgs() {
         return ""
@@ -77,4 +85,9 @@ class EndProcessConfiguration extends ProcessConfiguration {
     boolean getDebug() {
         return false
     }
+
+    @Override
+    ElasticsearchCluster getElasticsearchCluster() {
+        return null
+    }
 }

buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/fixture/hadoop/conf/HadoopClusterConfiguration.groovy

@@ -47,11 +47,10 @@ class HadoopClusterConfiguration extends ProcessConfiguration {
     private static final Map<String, ServiceDescriptor> SUPPORTED_SERVICES = [HADOOP, HIVE, PIG, SPARK]
                     .collectEntries { [(it.id()): it] }
 
-    private static final ProcessConfiguration END = new EndProcessConfiguration()
-
     private final Project project
     private final String name
     private final List<Task> clusterTasks
+    private final ProcessConfiguration defaultConfiguration
     private final Map<String, ServiceConfiguration> serviceConfigurations
     private final List<ServiceConfiguration> serviceCreationOrder
 
@@ -60,6 +59,7 @@ class HadoopClusterConfiguration extends ProcessConfiguration {
         this.project = project
         this.name = name
         this.clusterTasks = []
+        this.defaultConfiguration = new EndProcessConfiguration(project)
         this.serviceConfigurations = [:]
         this.serviceCreationOrder = []
     }
@@ -126,6 +126,6 @@ class HadoopClusterConfiguration extends ProcessConfiguration {
 
     @Override
     protected ProcessConfiguration parent() {
-        return END
+        return defaultConfiguration
     }
 }

buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/fixture/hadoop/conf/ProcessConfiguration.groovy

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.hadoop.gradle.fixture.hadoop.conf
 
+import org.elasticsearch.gradle.testclusters.ElasticsearchCluster
 import org.gradle.api.InvalidUserDataException
 import org.gradle.api.Project
 import org.gradle.api.Task
@@ -38,16 +39,18 @@ abstract class ProcessConfiguration {
         this.project = project
     }
 
-    private final Project project
+    protected final Project project
     private Map<String, String> systemProperties = new HashMap<>()
     private Map<String, String> environmentVariables = new HashMap<>()
     private SettingsContainer settingsContainer = new SettingsContainer()
     private Map<String, Object> extraConfigFiles = new HashMap<>()
     private LinkedHashMap<String, Object[]> setupCommands = new LinkedHashMap<>()
     private List<Object> dependencies = new ArrayList<>()
     private List<Task> clusterTasks = new ArrayList<>()
+    private String javaHome = null
     private String jvmArgs = ''
     private boolean debug = false
+    private ElasticsearchCluster elasticsearchCluster = null
 
     void addSystemProperty(String key, String value) {
         systemProperties.put(key, value)
@@ -77,7 +80,7 @@ abstract class ProcessConfiguration {
         return combined
     }
 
-    void addSetting(String key, String value) {
+    void addSetting(String key, Object value) {
         settingsContainer.addSetting(key, value)
     }
 
@@ -145,6 +148,23 @@ abstract class ProcessConfiguration {
         return combined
     }
 
+    void setJavaHome(String javaHome) {
+        this.javaHome = javaHome
+    }
+
+    String getJavaHome() {
+        if (this.javaHome != null) {
+            return this.javaHome
+        } else {
+            ProcessConfiguration parent = parent()
+            if (parent != null) {
+                return parent.getJavaHome()
+            } else {
+                return null
+            }
+        }
+    }
+
     void setJvmArgs(String jvmArgs) {
         this.jvmArgs = jvmArgs
     }
@@ -178,6 +198,23 @@ abstract class ProcessConfiguration {
         return debug
     }
 
+    void useElasticsearchCluster(ElasticsearchCluster elasticsearchCluster) {
+        this.elasticsearchCluster = elasticsearchCluster
+    }
+
+    ElasticsearchCluster getElasticsearchCluster() {
+        if (this.elasticsearchCluster != null) {
+            return this.elasticsearchCluster
+        } else {
+            ProcessConfiguration parent = parent()
+            if (parent != null) {
+                return parent.getElasticsearchCluster()
+            } else {
+                return null
+            }
+        }
+    }
+
     Task createClusterTask(Map<String, ?> options) throws InvalidUserDataException {
         Task task = project.tasks.create(options)
         addClusterTask(task)