Skip to content

Commit

Permalink
Clean up configuration when docker isn't available (#42745)
Browse files Browse the repository at this point in the history 
We initially added `requireDocker` for a way for tasks to say that they
absolutely must have it, like the  build docker image tasks.
Projects using the test fixtures plugin are not in this both, as the
intent with these is that they will be skipped if docker and docker-compose
is not available.

Before this change we were lenient, the docker image build would succeed
but produce nothing. The implementation was also confusing as it was not
immediately obvious this was the case due to all the indirection in the
code.

The reason we have this leniency is that when we added the docker image
build, docker was a fairly new requirement for us, and we didn't have
it deployed in CI widely enough nor had CI configured to prefer workers
with docker when possible. We are in a much better position now.
The other reason was other stack teams running `./gradlew assemble`
in their respective CI and the possibility of breaking them if docker is
not installed. We have been advocating for building specific distros for
some time now and I will also send out an additional notice

The PR also removes the use of `requireDocker` from tests that actually
use test fixtures and are ok without it, and fixes a bug in test
fixtures that would cause incorrect configuration and allow some tasks
to run when docker was not available and they shouldn't have.

Closes  #42680 and #42829  see also #42719
  • Loading branch information
@alpar-t
alpar-t committed Jun 10, 2019
1 parent 44aedcf commit 9def454
Show file tree
Hide file tree
Showing 5 changed files with 47 additions and 56 deletions.
6 changes: 1 addition & 5 deletions buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -257,11 +257,7 @@ class BuildPlugin implements Plugin<Project> {
}
}

if (ext.get('buildDocker')) {
(ext.get('requiresDocker') as List<Task>).add(task)
} else {
task.onlyIf { false }
}
(ext.get('requiresDocker') as List<Task>).add(task)
}

protected static void checkDockerVersionRecent(String dockerVersion) {
Expand Down
8 changes: 2 additions & 6 deletions buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,11 +116,7 @@ default void waitForConditions(
} catch (TestClustersException e) {
throw e;
} catch (Exception e) {
if (lastException == null) {
lastException = e;
} else {
lastException = e;
}
throw e;
}
}
if (conditionMet == false) {
Expand All @@ -129,7 +125,7 @@ default void waitForConditions(
if (lastException == null) {
throw new TestClustersException(message);
} else {
throw new TestClustersException(message, lastException);
throw new TestClustersException(message + message, lastException);
}
}
logger.info(
Expand Down
68 changes: 36 additions & 32 deletions buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import org.gradle.api.tasks.TaskContainer;
import org.gradle.api.tasks.testing.Test;

import java.io.File;
import java.util.Collections;
import java.util.function.BiConsumer;

Expand All @@ -56,46 +57,47 @@ public void apply(Project project) {
disableTaskByType(tasks, ThirdPartyAuditTask.class);
disableTaskByType(tasks, JarHellTask.class);

// the project that defined a test fixture can also use it
extension.fixtures.add(project);

Task buildFixture = project.getTasks().create("buildFixture");
Task pullFixture = project.getTasks().create("pullFixture");
Task preProcessFixture = project.getTasks().create("preProcessFixture");
buildFixture.dependsOn(preProcessFixture);
pullFixture.dependsOn(preProcessFixture);
Task postProcessFixture = project.getTasks().create("postProcessFixture");
postProcessFixture.dependsOn(buildFixture);
preProcessFixture.onlyIf(spec -> buildFixture.getEnabled());
postProcessFixture.onlyIf(spec -> buildFixture.getEnabled());

if (dockerComposeSupported(project) == false) {
if (dockerComposeSupported() == false) {
preProcessFixture.setEnabled(false);
postProcessFixture.setEnabled(false);
buildFixture.setEnabled(false);
pullFixture.setEnabled(false);
return;
}
preProcessFixture.onlyIf(spec -> buildFixture.getEnabled());
postProcessFixture.onlyIf(spec -> buildFixture.getEnabled());

project.apply(spec -> spec.plugin(BasePlugin.class));
project.apply(spec -> spec.plugin(DockerComposePlugin.class));
ComposeExtension composeExtension = project.getExtensions().getByType(ComposeExtension.class);
composeExtension.setUseComposeFiles(Collections.singletonList(DOCKER_COMPOSE_YML));
composeExtension.setRemoveContainers(true);
composeExtension.setExecutable(
project.file("/usr/local/bin/docker-compose").exists() ?
"/usr/local/bin/docker-compose" : "/usr/bin/docker-compose"
);
} else {
project.apply(spec -> spec.plugin(BasePlugin.class));
project.apply(spec -> spec.plugin(DockerComposePlugin.class));
ComposeExtension composeExtension = project.getExtensions().getByType(ComposeExtension.class);
composeExtension.setUseComposeFiles(Collections.singletonList(DOCKER_COMPOSE_YML));
composeExtension.setRemoveContainers(true);
composeExtension.setExecutable(
project.file("/usr/local/bin/docker-compose").exists() ?
"/usr/local/bin/docker-compose" : "/usr/bin/docker-compose"
);

buildFixture.dependsOn(tasks.getByName("composeUp"));
pullFixture.dependsOn(tasks.getByName("composePull"));
tasks.getByName("composeUp").mustRunAfter(preProcessFixture);
tasks.getByName("composePull").mustRunAfter(preProcessFixture);
postProcessFixture.dependsOn(buildFixture);
buildFixture.dependsOn(tasks.getByName("composeUp"));
pullFixture.dependsOn(tasks.getByName("composePull"));
tasks.getByName("composeUp").mustRunAfter(preProcessFixture);
tasks.getByName("composePull").mustRunAfter(preProcessFixture);

configureServiceInfoForTask(
postProcessFixture,
project,
(name, port) -> postProcessFixture.getExtensions()
.getByType(ExtraPropertiesExtension.class).set(name, port)
);
extension.fixtures.add(project);
configureServiceInfoForTask(
postProcessFixture,
project,
(name, port) -> postProcessFixture.getExtensions()
.getByType(ExtraPropertiesExtension.class).set(name, port)
);
}
}

extension.fixtures
Expand All @@ -107,7 +109,7 @@ public void apply(Project project) {
conditionTaskByType(tasks, extension, TestingConventionsTasks.class);
conditionTaskByType(tasks, extension, ComposeUp.class);

if (dockerComposeSupported(project) == false) {
if (dockerComposeSupported() == false) {
project.getLogger().warn(
"Tests for {} require docker-compose at /usr/local/bin/docker-compose or /usr/bin/docker-compose " +
"but none could be found so these will be skipped", project.getPath()
Expand Down Expand Up @@ -135,7 +137,9 @@ private void conditionTaskByType(TaskContainer tasks, TestFixtureExtension exten
taskClass,
task -> task.onlyIf(spec ->
extension.fixtures.stream()
.anyMatch(fixtureProject -> fixtureProject.getTasks().getByName("buildFixture").getEnabled() == false) == false
.anyMatch(fixtureProject ->
fixtureProject.getTasks().getByName("buildFixture").getEnabled() == false
) == false
)
);
}
Expand Down Expand Up @@ -168,12 +172,12 @@ private void configureServiceInfoForTask(Task task, Project fixtureProject, BiCo
);
}

public boolean dockerComposeSupported(Project project) {
public static boolean dockerComposeSupported() {
if (OS.current().equals(OS.WINDOWS)) {
return false;
}
final boolean hasDockerCompose = project.file("/usr/local/bin/docker-compose").exists() ||
project.file("/usr/bin/docker-compose").exists();
final boolean hasDockerCompose = (new File("/usr/local/bin/docker-compose")).exists() ||
(new File("/usr/bin/docker-compose").exists());
return hasDockerCompose && Boolean.parseBoolean(System.getProperty("tests.fixture.enabled", "true"));
}

Expand Down
6 changes: 5 additions & 1 deletion distribution/docker/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import org.elasticsearch.gradle.BuildPlugin
import org.elasticsearch.gradle.LoggedExec
import org.elasticsearch.gradle.MavenFilteringHack
import org.elasticsearch.gradle.VersionProperties
import org.elasticsearch.gradle.testfixtures.TestFixturesPlugin

apply plugin: 'base'
apply plugin: 'elasticsearch.test.fixtures'
Expand Down Expand Up @@ -77,7 +78,10 @@ void addCopyDockerContextTask(final boolean oss) {
}

preProcessFixture {
dependsOn assemble
// don't add the tasks to build the docker images if we have no way of testing them
if (TestFixturesPlugin.dockerComposeSupported()) {
dependsOn assemble
}
}

postProcessFixture.doLast {
Expand Down
15 changes: 3 additions & 12 deletions plugins/repository-s3/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -169,24 +169,16 @@ if (useFixture) {

File minioAddressFile = new File(project.buildDir, 'generated-resources/s3Fixture.address')

// We can't lazy evaluate a system property for the Minio address passed to JUnit so we write it to a resource file
// and pass its name instead.
task writeMinioAddress {
thirdPartyTest {
dependsOn tasks.bundlePlugin, tasks.postProcessFixture
outputs.file(minioAddressFile)
doLast {
doFirst {
file(minioAddressFile).text = "${ -> minioAddress.call() }"
}
}

thirdPartyTest {
dependsOn writeMinioAddress
inputs.file(minioAddressFile)
// TODO: this could be a nonInputProperties.systemProperty so we don't need a file
systemProperty 'test.s3.endpoint', minioAddressFile.name
}

BuildPlugin.requireDocker(tasks.thirdPartyTest)

task integTestMinio(type: RestIntegTestTask) {
description = "Runs REST tests using the Minio repository."
dependsOn tasks.bundlePlugin, tasks.postProcessFixture
Expand All @@ -200,7 +192,6 @@ if (useFixture) {
}
}
check.dependsOn(integTestMinio)
BuildPlugin.requireDocker(tasks.integTestMinio)

testClusters.integTestMinio {
keystore 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
Expand Down

0 comments on commit 9def454

Please sign in to comment.