Skip to content

Commit

Permalink
Remove usages of ClusterFormationTasks and replace with test clusters (…
Browse files Browse the repository at this point in the history 
…#1365)
  • Loading branch information
@mark-vieira
mark-vieira authored Oct 14, 2019
1 parent 5b91177 commit 301048e
Show file tree
Hide file tree
Showing 5 changed files with 102 additions and 552 deletions.
5 changes: 3 additions & 2 deletions buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import org.elasticsearch.gradle.info.GenerateGlobalBuildInfoTask
import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin
import org.elasticsearch.gradle.info.JavaHome
import org.elasticsearch.gradle.precommit.LicenseHeadersTask
import org.elasticsearch.gradle.testclusters.RestTestRunnerTask
import org.elasticsearch.hadoop.gradle.util.Resources
import org.gradle.api.GradleException
import org.gradle.api.JavaVersion
Expand Down Expand Up @@ -577,7 +578,7 @@ class BuildPlugin implements Plugin<Project> {
hadoopTestingJar.from(project.sourceSets.main.output)
hadoopTestingJar.from(project.sourceSets.itest.output)

Test integrationTest = project.tasks.create('integrationTest', Test.class)
Test integrationTest = project.tasks.create('integrationTest', RestTestRunnerTask.class)
integrationTest.dependsOn(hadoopTestingJar)

integrationTest.testClassesDirs = project.sourceSets.itest.output.classesDirs
Expand Down Expand Up @@ -713,7 +714,7 @@ class BuildPlugin implements Plugin<Project> {
}

private static void configurePrecommit(Project project) {
if (project != project.rootProject) {
if (project != project.rootProject && project.hasProperty('localRepo') == false) {
LicenseHeadersTask licenseHeaders = project.tasks.create('licenseHeaders', LicenseHeadersTask.class)
project.tasks.getByName('check').dependsOn(licenseHeaders)
}
Expand Down
187 changes: 38 additions & 149 deletions ...src/main/groovy/org/elasticsearch/hadoop/gradle/fixture/ElasticsearchFixturePlugin.groovy
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,13 @@
package org.elasticsearch.hadoop.gradle.fixture

import org.elasticsearch.gradle.Version
import org.elasticsearch.gradle.test.ClusterConfiguration
import org.elasticsearch.gradle.test.ClusterFormationTasks
import org.elasticsearch.gradle.test.NodeInfo
import org.elasticsearch.hadoop.gradle.util.PlaceholderTask
import org.elasticsearch.gradle.testclusters.ElasticsearchCluster
import org.elasticsearch.gradle.testclusters.RestTestRunnerTask
import org.elasticsearch.gradle.testclusters.TestClustersPlugin
import org.elasticsearch.gradle.testclusters.TestDistribution
import org.gradle.api.NamedDomainObjectContainer
import org.gradle.api.Plugin
import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.execution.TaskExecutionAdapter
import org.gradle.api.tasks.TaskState
import org.gradle.util.ConfigureUtil

import java.nio.charset.StandardCharsets
import java.nio.file.Files
import java.util.stream.Stream
import org.gradle.process.CommandLineArgumentProvider

/**
* Plugin that adds the ability to stand up an Elasticsearch cluster for tests.
Expand All @@ -28,92 +21,60 @@ import java.util.stream.Stream
*/
class ElasticsearchFixturePlugin implements Plugin<Project> {

static class ElasticsearchCluster {

Project project
ClusterConfiguration configuration
List<Task> tasks = []

ElasticsearchCluster(Project project) {
this.project = project
this.configuration = new ClusterConfiguration(project)
}

void clusterConf(Closure configClosure) {
ConfigureUtil.configure(configClosure, configuration)
}

void addTask(Task task) {
tasks.add(task)
}
}

@Override
void apply(Project project) {

project.pluginManager.apply(TestClustersPlugin)
def version = project.hasProperty("es.version") ? project.getProperty("es.version") : project.elasticsearchVersion

// Optionally allow user to disable the fixture
def hasLocalRepo = project.hasProperty("localRepo")
def useFixture = !hasLocalRepo && Boolean.parseBoolean(project.hasProperty("tests.fixture.es.enable") ? project.getProperty("tests.fixture.es.enable") : "true")
def useFixture = !hasLocalRepo && Boolean.parseBoolean(project.findProperty("tests.fixture.es.enable") ?: "true")

def integrationTestTask = project.tasks.getByName("integrationTest") as RestTestRunnerTask
if (useFixture) {
// Depends on project already containing an "integrationTest"
// task, as well as javaHome+runtimeJavaHome configured
createClusterFor(project.tasks.getByName("integrationTest"), project, version)
createClusterFor(integrationTestTask, project, version)
} else {
project.tasks.getByName("integrationTest") {
systemProperty "test.disable.local.es", "true"
}
integrationTestTask.systemProperty("test.disable.local.es", "true")
}
}

private static def createClusterFor(Task integrationTest, Project project, String version) {
// Version settings
def majorVersion = version.tokenize(".").get(0).toInteger()

// Init task can be used to prepare cluster
Task clusterInit = project.tasks.create(name: "esCluster#init", dependsOn: project.testClasses)
integrationTest.dependsOn(clusterInit)

ElasticsearchCluster cluster = project.extensions.create("esCluster", ElasticsearchCluster.class, project)
cluster.tasks.add(integrationTest)
ClusterConfiguration clusterConfig = cluster.configuration
private static void createClusterFor(RestTestRunnerTask integrationTest, Project project, String version) {
def clustersContainer = project.extensions.getByName(TestClustersPlugin.EXTENSION_NAME) as NamedDomainObjectContainer<ElasticsearchCluster>
def integTestCluster = clustersContainer.create("integTest") { ElasticsearchCluster cluster ->
cluster.version = version
cluster.testDistribution = TestDistribution.DEFAULT
}

// default settings:
clusterConfig.clusterName = "elasticsearch-fixture"
clusterConfig.numNodes = 1
clusterConfig.httpPort = 9500
clusterConfig.transportPort = 9600
clusterConfig.distribution = "default" // Full Distribution
integrationTest.useCluster(integTestCluster)
// Add the cluster HTTP URI as a system property which isn't tracked as a task input
integrationTest.jvmArgumentProviders.add({ ["-Dtests.rest.cluster=${integTestCluster.httpSocketURI}"] } as CommandLineArgumentProvider)

// Set BWC if not current ES version:
if (version != project.elasticsearchVersion) {
clusterConfig.bwcVersion = Version.fromString(version)
clusterConfig.numBwcNodes = 1
}
// Version settings
def majorVersion = version.tokenize(".").get(0).toInteger()

// Version specific configurations
if (majorVersion <= 2) {
clusterConfig.setting("transport.type","local")
clusterConfig.setting("http.type","netty3")
clusterConfig.setting("script.inline", "true")
clusterConfig.setting("script.indexed", "true")
integTestCluster.setting("transport.type","local")
integTestCluster.setting("http.type","netty3")
integTestCluster.setting("script.inline", "true")
integTestCluster.setting("script.indexed", "true")
} else if (majorVersion == 5) {
clusterConfig.setting("transport.type","netty4")
clusterConfig.setting("http.type","netty4")
clusterConfig.setting("script.inline", "true")
clusterConfig.setting("node.ingest", "true")
clusterConfig.setting("script.max_compilations_rate", null)
integTestCluster.setting("transport.type","netty4")
integTestCluster.setting("http.type","netty4")
integTestCluster.setting("script.inline", "true")
integTestCluster.setting("node.ingest", "true")
integTestCluster.setting("script.max_compilations_rate", null)
} else if (majorVersion == 6) {
clusterConfig.setting("node.ingest", "true")
clusterConfig.setting("http.host", "localhost")
clusterConfig.systemProperty('es.http.cname_in_publish_address', 'true')
integTestCluster.setting("node.ingest", "true")
integTestCluster.setting("http.host", "localhost")
integTestCluster.systemProperty('es.http.cname_in_publish_address', 'true')
} else if (majorVersion >= 7) {
clusterConfig.setting("node.ingest", "true")
clusterConfig.setting("http.host", "localhost")
integTestCluster.setting("node.ingest", "true")
integTestCluster.setting("http.host", "localhost")
// TODO: Remove this when this is the default in 7
clusterConfig.systemProperty('es.http.cname_in_publish_address', 'true')
integTestCluster.systemProperty('es.http.cname_in_publish_address', 'true')
}

// Also write a script to a file for use in tests
Expand All @@ -128,79 +89,7 @@ class ElasticsearchFixturePlugin implements Plugin<Project> {
script = new File(scriptsDir, "increment.painless").setText("ctx._source.counter = ctx._source.getOrDefault('counter', 0) + 1", 'UTF-8')
}
if (script != null) {
clusterConfig.extraConfigFile("script", script)
integTestCluster.extraConfigFile("script", script)
}

project.gradle.projectsEvaluated {
Task clusterMain = new PlaceholderTask()
List<NodeInfo> nodes = ClusterFormationTasks.setup(project, "esCluster", clusterMain, clusterConfig)
project.tasks.getByPath("esCluster#wait").doLast {
integrationTest.systemProperty('tests.rest.cluster', "${nodes.collect{it.httpUri()}.join(",")}")
}

// dump errors and warnings from cluster log on failure
TaskExecutionAdapter logDumpListener = new TaskExecutionAdapter() {
@Override
void afterExecute(Task task, TaskState state) {
if (state.failure != null) {
for (NodeInfo nodeInfo : nodes) {
printLogExcerpt(nodeInfo)
}
}
}
}
for (Task clusterTask : cluster.tasks) {
for (Object dependency : clusterMain.taskDeps) {
clusterTask.dependsOn(dependency)
}
for (Object finalizer : clusterMain.taskFinalizers) {
clusterTask.finalizedBy(finalizer)
}
clusterTask.doFirst {
project.gradle.addListener(logDumpListener)
}
clusterTask.doLast {
project.gradle.removeListener(logDumpListener)
}
}
}
}

/** Print out an excerpt of the log from the given node. */
protected static void printLogExcerpt(NodeInfo nodeInfo) {
File logFile = new File(nodeInfo.homeDir, "logs/${nodeInfo.clusterName}.log")
println("\nCluster ${nodeInfo.clusterName} - node ${nodeInfo.nodeNum} log excerpt:")
println("(full log at ${logFile})")
println('-----------------------------------------')
Stream<String> stream = Files.lines(logFile.toPath(), StandardCharsets.UTF_8)
try {
boolean inStartup = true
boolean inExcerpt = false
int linesSkipped = 0
for (String line : stream) {
if (line.startsWith("[")) {
inExcerpt = false // clear with the next log message
}
if (line =~ /(\[WARN *\])|(\[ERROR *\])/) {
inExcerpt = true // show warnings and errors
}
if (inStartup || inExcerpt) {
if (linesSkipped != 0) {
println("... SKIPPED ${linesSkipped} LINES ...")
}
println(line)
linesSkipped = 0
} else {
++linesSkipped
}
if (line =~ /recovered \[\d+\] indices into cluster_state/) {
inStartup = false
}
}
} finally {
stream.close()
}
println('=========================================')

}
}
2 changes: 1 addition & 1 deletion buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/fixture/hadoop/InstanceInfo.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,7 @@ class InstanceInfo {

/** Return the java home used by this node. */
String getJavaHome() {
return javaVersion == null ? project.runtimeJavaHome : project.javaVersions.get(javaVersion)
return javaVersion == null ? project.runtimeJavaHome : project.javaVersions.find { it.version == javaVersion }.javaHome.absolutePath
}

/** Returns debug string for the command that started this node. */
Expand Down
Loading

0 comments on commit 301048e

Please sign in to comment.