Skip to content

Commit

Permalink
Testsclusters use seprate configurations per version (#41504)
Browse files Browse the repository at this point in the history 
* artifact config tests WIP

* wip version configuration

* Tests for older versions

* use separate configurations per version

* checkstyle

* PR review
  • Loading branch information
@alpar-t
alpar-t committed May 6, 2019
1 parent d54a921 commit 2455295
Show file tree
Hide file tree
Showing 7 changed files with 157 additions and 75 deletions.
16 changes: 9 additions & 7 deletions buildSrc/src/main/java/org/elasticsearch/gradle/Distribution.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,24 +20,26 @@

public enum Distribution {

INTEG_TEST("elasticsearch", "integ-test-zip"),
DEFAULT("elasticsearch", "elasticsearch"),
OSS("elasticsearch-oss", "elasticsearch-oss");
INTEG_TEST("elasticsearch"),
DEFAULT("elasticsearch"),
OSS("elasticsearch-oss");

private final String artifactName;
private final String group;

Distribution(String name, String group) {
Distribution(String name) {
this.artifactName = name;
this.group = group;
}

public String getArtifactName() {
return artifactName;
}

public String getGroup() {
return "org.elasticsearch.distribution." + group;
if (this.equals(INTEG_TEST)) {
return "org.elasticsearch.distribution.integ-test-zip";
} else {
return "org.elasticsearch.distribution." + name().toLowerCase();
}
}

public String getFileExtension() {
Expand Down
4 changes: 3 additions & 1 deletion buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -279,7 +279,9 @@ public boolean isProcessAlive() {
}

void eachVersionedDistribution(BiConsumer<String, Distribution> consumer) {
nodes.forEach(each -> consumer.accept(each.getVersion(), each.getDistribution()));
nodes.forEach(each -> {
consumer.accept(each.getVersion(), each.getDistribution());
});
}

public ElasticsearchNode singleNode() {
Expand Down
2 changes: 1 addition & 1 deletion buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -276,7 +276,7 @@ public synchronized void start() {

Path distroArtifact = artifactsExtractDir
.resolve(distribution.getGroup())
.resolve(distribution.getArtifactName() + "-" + getVersion());
.resolve("elasticsearch-" + getVersion());

if (Files.exists(distroArtifact) == false) {
throw new TestClustersException("Can not start " + this + ", missing: " + distroArtifact);
Expand Down
155 changes: 91 additions & 64 deletions buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,17 +20,18 @@

import groovy.lang.Closure;
import org.elasticsearch.gradle.BwcVersions;
import org.elasticsearch.gradle.Distribution;
import org.elasticsearch.gradle.Version;
import org.elasticsearch.gradle.tool.Boilerplate;
import org.gradle.api.Action;
import org.gradle.api.NamedDomainObjectContainer;
import org.gradle.api.Plugin;
import org.gradle.api.Project;
import org.gradle.api.Task;
import org.gradle.api.artifacts.Configuration;
import org.gradle.api.artifacts.repositories.MavenArtifactRepository;
import org.gradle.api.credentials.HttpHeaderCredentials;
import org.gradle.api.execution.TaskActionListener;
import org.gradle.api.execution.TaskExecutionListener;
import org.gradle.api.file.FileCollection;
import org.gradle.api.file.FileTree;
import org.gradle.api.logging.Logger;
import org.gradle.api.logging.Logging;
Expand All @@ -46,7 +47,6 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
Expand All @@ -56,7 +56,7 @@ public class TestClustersPlugin implements Plugin<Project> {

private static final String LIST_TASK_NAME = "listTestClusters";
private static final String NODE_EXTENSION_NAME = "testClusters";
private static final String HELPER_CONFIGURATION_NAME = "testclusters";
private static final String HELPER_CONFIGURATION_PREFIX = "testclusters";
private static final String SYNC_ARTIFACTS_TASK_NAME = "syncTestClustersArtifacts";
private static final int EXECUTOR_SHUTDOWN_TIMEOUT = 1;
private static final TimeUnit EXECUTOR_SHUTDOWN_TIMEOUT_UNIT = TimeUnit.MINUTES;
Expand All @@ -69,6 +69,10 @@ public class TestClustersPlugin implements Plugin<Project> {
private final Thread shutdownHook = new Thread(this::shutDownAllClusters);
private ExecutorService executorService = Executors.newSingleThreadExecutor();

public static String getHelperConfigurationName(String version) {
return HELPER_CONFIGURATION_PREFIX + "-" + version;
}

@Override
public void apply(Project project) {
Project rootProject = project.getRootProject();
Expand All @@ -82,47 +86,6 @@ public void apply(Project project) {
// create DSL for tasks to mark clusters these use
createUseClusterTaskExtension(project, container);

if (rootProject.getConfigurations().findByName(HELPER_CONFIGURATION_NAME) == null) {
// We use a single configuration on the root project to resolve all testcluster dependencies ( like distros )
// at once, only once without the need to repeat it for each project. This pays off assuming that most
// projects use the same dependencies.
Configuration helperConfiguration = project.getRootProject().getConfigurations().create(HELPER_CONFIGURATION_NAME);
helperConfiguration.setDescription(
"Internal helper configuration used by cluster configuration to download " +
"ES distributions and plugins."
);

// We have a single task to sync the helper configuration to "artifacts dir"
// the clusters will look for artifacts there based on the naming conventions.
// Tasks that use a cluster will add this as a dependency automatically so it's guaranteed to run early in
// the build.
rootProject.getTasks().create(SYNC_ARTIFACTS_TASK_NAME, sync -> {
sync.getInputs().files((Callable<FileCollection>) helperConfiguration::getAsFileTree);
sync.getOutputs().dir(new File(project.getRootProject().getBuildDir(), "testclusters/extract"));
// NOTE: Gradle doesn't allow a lambda here ( fails at runtime )
sync.doLast(new Action<Task>() {
@Override
public void execute(Task task) {
project.sync(spec ->
helperConfiguration.getResolvedConfiguration().getResolvedArtifacts().forEach(resolvedArtifact -> {
final FileTree files;
File file = resolvedArtifact.getFile();
if (file.getName().endsWith(".zip")) {
files = project.zipTree(file);
} else if (file.getName().endsWith("tar.gz")) {
files = project.tarTree(file);
} else {
throw new IllegalArgumentException("Can't extract " + file + " unknown file extension");
}
spec.from(files).into(new File(project.getRootProject().getBuildDir(), "testclusters/extract") + "/" +
resolvedArtifact.getModuleVersion().getId().getGroup()
);
}));
}
});
});
}

// When we know what tasks will run, we claim the clusters of those task to differentiate between clusters
// that are defined in the build script and the ones that will actually be used in this invocation of gradle
// we use this information to determine when the last task that required the cluster executed so that we can
Expand All @@ -143,6 +106,10 @@ public void execute(Task task) {
autoConfigureClusterDependencies(project, rootProject, container);
}

private static File getExtractDir(Project project) {
return new File(project.getRootProject().getBuildDir(), "testclusters/extract/");
}

private NamedDomainObjectContainer<ElasticsearchCluster> createTestClustersContainerExtension(Project project) {
// Create an extensions that allows describing clusters
NamedDomainObjectContainer<ElasticsearchCluster> container = project.container(
Expand Down Expand Up @@ -290,12 +257,59 @@ private static void autoConfigureClusterDependencies(
Project rootProject,
NamedDomainObjectContainer<ElasticsearchCluster> container
) {
// Download integ test distribution from maven central
MavenArtifactRepository mavenCentral = project.getRepositories().mavenCentral();
mavenCentral.content(spec -> {
spec.includeGroupByRegex("org\\.elasticsearch\\.distribution\\..*");
});

// Other distributions from the download service
project.getRepositories().add(
project.getRepositories().ivy(spec -> {
spec.setUrl("https://artifacts.elastic.co/downloads");
spec.patternLayout(p -> p.artifact("elasticsearch/[module]-[revision](-[classifier]).[ext]"));
HttpHeaderCredentials headerConfig = spec.getCredentials(HttpHeaderCredentials.class);
headerConfig.setName("X-Elastic-No-KPI");
headerConfig.setValue("1");
spec.content(c-> c.includeGroupByRegex("org\\.elasticsearch\\.distribution\\..*"));
})
);

// We have a single task to sync the helper configuration to "artifacts dir"
// the clusters will look for artifacts there based on the naming conventions.
// Tasks that use a cluster will add this as a dependency automatically so it's guaranteed to run early in
// the build.
Task sync = Boilerplate.maybeCreate(rootProject.getTasks(), SYNC_ARTIFACTS_TASK_NAME, onCreate -> {
onCreate.getOutputs().dir(getExtractDir(rootProject));
// NOTE: Gradle doesn't allow a lambda here ( fails at runtime )
onCreate.doFirst(new Action<Task>() {
@Override
public void execute(Task task) {
// Clean up the extract dir first to make sure we have no stale files from older
// previous builds of the same distribution
project.delete(getExtractDir(rootProject));
}
});
});

// When the project evaluated we know of all tasks that use clusters.
// Each of these have to depend on the artifacts being synced.
// We need afterEvaluate here despite the fact that container is a domain object, we can't implement this with
// all because fields can change after the fact.
project.afterEvaluate(ip -> container.forEach(esCluster ->
esCluster.eachVersionedDistribution((version, distribution) -> {
Configuration helperConfiguration = Boilerplate.maybeCreate(
rootProject.getConfigurations(),
getHelperConfigurationName(version),
onCreate ->
// We use a single configuration on the root project to resolve all testcluster dependencies ( like distros )
// at once, only once without the need to repeat it for each project. This pays off assuming that most
// projects use the same dependencies.
onCreate.setDescription(
"Internal helper configuration used by cluster configuration to download " +
"ES distributions and plugins for " + version
)
);
BwcVersions.UnreleasedVersionInfo unreleasedInfo;
final List<Version> unreleased;
{
Expand All @@ -320,29 +334,42 @@ private static void autoConfigureClusterDependencies(
projectNotation.put("path", unreleasedInfo.gradleProjectPath);
projectNotation.put("configuration", distribution.getLiveConfiguration());
rootProject.getDependencies().add(
HELPER_CONFIGURATION_NAME,
helperConfiguration.getName(),
project.getDependencies().project(projectNotation)
);
} else {
if (distribution.equals(Distribution.INTEG_TEST)) {
rootProject.getDependencies().add(
HELPER_CONFIGURATION_NAME, "org.elasticsearch.distribution.integ-test-zip:elasticsearch:" + version
);
} else {
// declare dependencies to be downloaded from the download service.
// The BuildPlugin sets up the right repo for this to work
// TODO: move the repo definition in this plugin when ClusterFormationTasks is removed
String dependency = String.format(
"%s:%s:%s:%s@%s",
distribution.getGroup(),
distribution.getArtifactName(),
version,
distribution.getClassifier(),
distribution.getFileExtension()
);
rootProject.getDependencies().add(HELPER_CONFIGURATION_NAME, dependency);
}
rootProject.getDependencies().add(
helperConfiguration.getName(),
distribution.getGroup() + ":" +
distribution.getArtifactName() + ":" +
version +
(distribution.getClassifier().isEmpty() ? "" : ":" + distribution.getClassifier()) + "@" +
distribution.getFileExtension());

}

sync.getInputs().files(helperConfiguration);
// NOTE: Gradle doesn't allow a lambda here ( fails at runtime )
sync.doLast(new Action<Task>() {
@Override
public void execute(Task task) {
project.copy(spec ->
helperConfiguration.getResolvedConfiguration().getResolvedArtifacts().forEach(resolvedArtifact -> {
final FileTree files;
File file = resolvedArtifact.getFile();
if (file.getName().endsWith(".zip")) {
files = project.zipTree(file);
} else if (file.getName().endsWith("tar.gz")) {
files = project.tarTree(file);
} else {
throw new IllegalArgumentException("Can't extract " + file + " unknown file extension");
}

spec.from(files, s -> s.into(resolvedArtifact.getModuleVersion().getId().getGroup()));
spec.into(getExtractDir(project));
}));
}
});
})));
}

Expand Down
19 changes: 19 additions & 0 deletions buildSrc/src/main/java/org/elasticsearch/gradle/tool/Boilerplate.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,33 @@
*/
package org.elasticsearch.gradle.tool;

import org.gradle.api.Action;
import org.gradle.api.NamedDomainObjectContainer;
import org.gradle.api.Project;
import org.gradle.api.plugins.JavaPluginConvention;
import org.gradle.api.tasks.SourceSetContainer;

import java.util.Optional;

public abstract class Boilerplate {

public static SourceSetContainer getJavaSourceSets(Project project) {
return project.getConvention().getPlugin(JavaPluginConvention.class).getSourceSets();
}

public static <T> T maybeCreate(NamedDomainObjectContainer<T> collection, String name) {
return Optional.ofNullable(collection.findByName(name))
.orElse(collection.create(name));

}
public static <T> T maybeCreate(NamedDomainObjectContainer<T> collection, String name, Action<T> action) {
return Optional.ofNullable(collection.findByName(name))
.orElseGet(() -> {
T result = collection.create(name);
action.execute(result);
return result;
});

}

}
8 changes: 8 additions & 0 deletions buildSrc/src/test/java/org/elasticsearch/gradle/testclusters/TestClustersPluginIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,14 @@ public void testMultiProject() {
);
}

public void testReleased() {
BuildResult result = getTestClustersRunner("testReleased").build();
assertTaskSuccessful(result, ":testReleased");
assertStartedAndStoppedOnce(result, "releasedVersionDefault-1");
assertStartedAndStoppedOnce(result, "releasedVersionOSS-1");
assertStartedAndStoppedOnce(result, "releasedVersionIntegTest-1");
}

public void testIncremental() {
BuildResult result = getTestClustersRunner("clean", ":user1").build();
assertTaskSuccessful(result, ":user1");
Expand Down
28 changes: 26 additions & 2 deletions buildSrc/src/testKit/testclusters/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@ allprojects { all ->
url "https://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/" + luceneSnapshotRevision
}
}

jcenter()
}

if (project == rootProject || project.name == "alpha" || project.name == "bravo") {
Expand Down Expand Up @@ -58,6 +56,21 @@ testClusters {
javaHome = file(System.getProperty('java.home'))
numberOfNodes = 3
}
releasedVersionDefault {
version = "7.0.0"
distribution = 'DEFAULT'
javaHome = file(System.getProperty('java.home'))
}
releasedVersionOSS {
version = "7.0.0"
distribution = 'OSS'
javaHome = file(System.getProperty('java.home'))
}
releasedVersionIntegTest {
version = "7.0.0"
distribution = 'INTEG_TEST'
javaHome = file(System.getProperty('java.home'))
}
}

task multiNode {
Expand All @@ -67,6 +80,17 @@ task multiNode {
}
}

task testReleased {
useCluster testClusters.releasedVersionDefault
useCluster testClusters.releasedVersionOSS
useCluster testClusters.releasedVersionIntegTest
doFirst {
println "$path: Cluster running @ ${testClusters.releasedVersionDefault.httpSocketURI}"
println "$path: Cluster running @ ${testClusters.releasedVersionOSS.httpSocketURI}"
println "$path: Cluster running @ ${testClusters.releasedVersionIntegTest.httpSocketURI}"
}
}

task printLog {
useCluster testClusters.myTestCluster
doFirst {
Expand Down

0 comments on commit 2455295

Please sign in to comment.