From 1f3d1019e71dcc1aa8158b4a38139ed1227d381e Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Mon, 4 Nov 2019 17:38:25 +0000 Subject: [PATCH] Only define Docker pkg tests if Docker is available (#47640) Closes #47639, and unmutes tests that were muted in b958467. The Docker packaging tests were being defined irrespective of whether Docker was actually available in the current environment. Instead, implement exclude lists so that in environments where Docker is not available, no Docker packaging tests are defined. For CI hosts, the build checks `.ci/dockerOnLinuxExclusions`. The Vagrant VMs can defined the extension property `shouldTestDocker` property to opt-in to packaging tests. As part of this, define a seperate utility class for checking Docker, and call that instead of defining checks in-line in BuildPlugin.groovy --- .ci/dockerOnLinuxExclusions | 12 + .ci/os.sh | 10 + Vagrantfile | 6 +- .../elasticsearch/gradle/BuildPlugin.groovy | 80 +----- .../gradle/test/DistroTestPlugin.java | 133 +++++++++- .../gradle/tool/DockerUtils.java | 239 ++++++++++++++++++ .../org/elasticsearch/gradle/Version.java | 30 ++- .../gradle/BuildPluginTests.java | 11 - .../elasticsearch/gradle/VersionTests.java | 14 +- .../gradle/test/DistroTestPluginTests.java | 85 +++++++ qa/os/centos-7/build.gradle | 1 + qa/os/debian-9/build.gradle | 1 + qa/os/fedora-28/build.gradle | 1 + qa/os/fedora-29/build.gradle | 1 + .../packaging/test/DockerTests.java | 2 - .../elasticsearch/packaging/util/Docker.java | 20 +- qa/os/ubuntu-1604/build.gradle | 1 + qa/os/ubuntu-1804/build.gradle | 1 + 18 files changed, 539 insertions(+), 109 deletions(-) create mode 100644 .ci/dockerOnLinuxExclusions create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/tool/DockerUtils.java create mode 100644 buildSrc/src/test/java/org/elasticsearch/gradle/test/DistroTestPluginTests.java diff --git a/.ci/dockerOnLinuxExclusions b/.ci/dockerOnLinuxExclusions new file mode 100644 index 0000000000000..ab344641baff9 --- /dev/null +++ b/.ci/dockerOnLinuxExclusions @@ -0,0 +1,12 @@ +# This file specifies the Linux OS versions on which we can't build and +# test Docker images for some reason. These values correspond to ID and +# VERSION_ID from /etc/os-release, and a matching value will cause the +# Docker tests to be skipped on that OS. If /etc/os-release doesn't exist +# (as is the case on centos-6, for example) then that OS will again be +# excluded. +centos-6 +debian-8 +opensuse-15-1 +ol-6.10 +ol-7.7 +sles-12 diff --git a/.ci/os.sh b/.ci/os.sh index 8ec110ac183b7..7cb94ab9fa93f 100755 --- a/.ci/os.sh +++ b/.ci/os.sh @@ -6,6 +6,16 @@ if which zypper > /dev/null ; then sudo zypper install -y insserv-compat fi +if [ -e /etc/sysctl.d/99-gce.conf ]; then + # The GCE defaults disable IPv4 forwarding, which breaks the Docker + # build. Workaround this by renaming the file so that it is executed + # earlier than our own overrides. + # + # This ultimately needs to be fixed at the image level - see infra + # issue 15654. + sudo mv /etc/sysctl.d/99-gce.conf /etc/sysctl.d/98-gce.conf +fi + # Required by bats sudo touch /etc/is_vagrant_vm sudo useradd vagrant diff --git a/Vagrantfile b/Vagrantfile index 93b60b46872fd..c86b0a910c239 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -126,7 +126,7 @@ Vagrant.configure(2) do |config| end 'fedora-29'.tap do |box| config.vm.define box, define_opts do |config| - config.vm.box = 'elastic/fedora-28-x86_64' + config.vm.box = 'elastic/fedora-29-x86_64' dnf_common config, box dnf_docker config end @@ -216,6 +216,10 @@ def ubuntu_docker(config) # Add vagrant to the Docker group, so that it can run commands usermod -aG docker vagrant + + # Enable IPv4 forwarding + sed -i '/net.ipv4.ip_forward/s/^#//' /etc/sysctl.conf + systemctl restart networking SHELL end diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index 7956c2c8d85f2..c085952cc0be8 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -76,15 +76,14 @@ import org.gradle.external.javadoc.CoreJavadocOptions import org.gradle.internal.jvm.Jvm import org.gradle.language.base.plugins.LifecycleBasePlugin import org.gradle.process.CommandLineArgumentProvider -import org.gradle.process.ExecResult -import org.gradle.process.ExecSpec import org.gradle.util.GradleVersion import java.nio.charset.StandardCharsets import java.nio.file.Files -import java.util.regex.Matcher import static org.elasticsearch.gradle.tool.Boilerplate.maybeConfigure +import static org.elasticsearch.gradle.tool.DockerUtils.assertDockerIsAvailable +import static org.elasticsearch.gradle.tool.DockerUtils.getDockerPath /** * Encapsulates build configuration for elasticsearch projects. @@ -183,8 +182,7 @@ class BuildPlugin implements Plugin { */ // check if the Docker binary exists and record its path - final List maybeDockerBinaries = ['/usr/bin/docker', '/usr/local/bin/docker'] - final String dockerBinary = maybeDockerBinaries.find { it -> new File(it).exists() } + final String dockerBinary = getDockerPath().orElse(null) final boolean buildDocker final String buildDockerProperty = System.getProperty("build.docker") @@ -203,55 +201,9 @@ class BuildPlugin implements Plugin { ext.set('requiresDocker', []) rootProject.gradle.taskGraph.whenReady { TaskExecutionGraph taskGraph -> final List tasks = taskGraph.allTasks.intersect(ext.get('requiresDocker') as List).collect { " ${it.path}".toString()} - if (tasks.isEmpty() == false) { - /* - * There are tasks in the task graph that require Docker. Now we are failing because either the Docker binary does not - * exist or because execution of a privileged Docker command failed. - */ - if (dockerBinary == null) { - final String message = String.format( - Locale.ROOT, - "Docker (checked [%s]) is required to run the following task%s: \n%s", - maybeDockerBinaries.join(","), - tasks.size() > 1 ? "s" : "", - tasks.join('\n')) - throwDockerRequiredException(message) - } - - // we use a multi-stage Docker build, check the Docker version since 17.05 - final ByteArrayOutputStream dockerVersionOutput = new ByteArrayOutputStream() - LoggedExec.exec( - rootProject, - { ExecSpec it -> - it.commandLine = [dockerBinary, '--version'] - it.standardOutput = dockerVersionOutput - }) - final String dockerVersion = dockerVersionOutput.toString().trim() - checkDockerVersionRecent(dockerVersion) - - final ByteArrayOutputStream dockerImagesErrorOutput = new ByteArrayOutputStream() - // the Docker binary executes, check that we can execute a privileged command - final ExecResult dockerImagesResult = LoggedExec.exec( - rootProject, - { ExecSpec it -> - it.commandLine = [dockerBinary, "images"] - it.errorOutput = dockerImagesErrorOutput - it.ignoreExitValue = true - }) - - if (dockerImagesResult.exitValue != 0) { - final String message = String.format( - Locale.ROOT, - "a problem occurred running Docker from [%s] yet it is required to run the following task%s: \n%s\n" + - "the problem is that Docker exited with exit code [%d] with standard error output [%s]", - dockerBinary, - tasks.size() > 1 ? "s" : "", - tasks.join('\n'), - dockerImagesResult.exitValue, - dockerImagesErrorOutput.toString().trim()) - throwDockerRequiredException(message) - } + if (tasks.isEmpty() == false) { + assertDockerIsAvailable(task.project, tasks) } } } @@ -259,28 +211,6 @@ class BuildPlugin implements Plugin { (ext.get('requiresDocker') as List).add(task) } - protected static void checkDockerVersionRecent(String dockerVersion) { - final Matcher matcher = dockerVersion =~ /Docker version (\d+\.\d+)\.\d+(?:-[a-zA-Z0-9]+)?, build [0-9a-f]{7,40}/ - assert matcher.matches(): dockerVersion - final dockerMajorMinorVersion = matcher.group(1) - final String[] majorMinor = dockerMajorMinorVersion.split("\\.") - if (Integer.parseInt(majorMinor[0]) < 17 - || (Integer.parseInt(majorMinor[0]) == 17 && Integer.parseInt(majorMinor[1]) < 5)) { - final String message = String.format( - Locale.ROOT, - "building Docker images requires Docker version 17.05+ due to use of multi-stage builds yet was [%s]", - dockerVersion) - throwDockerRequiredException(message) - } - } - - private static void throwDockerRequiredException(final String message) { - throw new GradleException( - message + "\nyou can address this by attending to the reported issue, " - + "removing the offending tasks from being executed, " - + "or by passing -Dbuild.docker=false") - } - /** Add a check before gradle execution phase which ensures java home for the given java version is set. */ static void requireJavaHome(Task task, int version) { // use root project for global accounting diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/DistroTestPlugin.java b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/DistroTestPlugin.java index 55f4985748325..417e13400c114 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/DistroTestPlugin.java +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/DistroTestPlugin.java @@ -28,18 +28,22 @@ import org.elasticsearch.gradle.ElasticsearchDistribution.Type; import org.elasticsearch.gradle.Jdk; import org.elasticsearch.gradle.JdkDownloadPlugin; +import org.elasticsearch.gradle.OS; import org.elasticsearch.gradle.Version; import org.elasticsearch.gradle.VersionProperties; import org.elasticsearch.gradle.info.BuildParams; import org.elasticsearch.gradle.vagrant.BatsProgressLogger; import org.elasticsearch.gradle.vagrant.VagrantBasePlugin; import org.elasticsearch.gradle.vagrant.VagrantExtension; +import org.gradle.api.GradleException; import org.gradle.api.NamedDomainObjectContainer; import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.Task; import org.gradle.api.artifacts.Configuration; import org.gradle.api.file.Directory; +import org.gradle.api.logging.Logger; +import org.gradle.api.logging.Logging; import org.gradle.api.plugins.ExtraPropertiesExtension; import org.gradle.api.plugins.JavaBasePlugin; import org.gradle.api.provider.Provider; @@ -52,6 +56,7 @@ import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -66,6 +71,7 @@ import static org.elasticsearch.gradle.vagrant.VagrantMachine.convertWindowsPath; public class DistroTestPlugin implements Plugin { + private static final Logger logger = Logging.getLogger(DistroTestPlugin.class); private static final String SYSTEM_JDK_VERSION = "11.0.2+9"; private static final String SYSTEM_JDK_VENDOR = "openjdk"; @@ -84,6 +90,8 @@ public class DistroTestPlugin implements Plugin { @Override public void apply(Project project) { + final boolean runDockerTests = shouldRunDockerTests(project); + project.getPluginManager().apply(DistributionDownloadPlugin.class); project.getPluginManager().apply(BuildPlugin.class); @@ -95,15 +103,17 @@ public void apply(Project project) { Provider upgradeDir = project.getLayout().getBuildDirectory().dir("packaging/upgrade"); Provider pluginsDir = project.getLayout().getBuildDirectory().dir("packaging/plugins"); - List distributions = configureDistributions(project, upgradeVersion); + List distributions = configureDistributions(project, upgradeVersion, runDockerTests); TaskProvider copyDistributionsTask = configureCopyDistributionsTask(project, distributionsDir); TaskProvider copyUpgradeTask = configureCopyUpgradeTask(project, upgradeVersion, upgradeDir); TaskProvider copyPluginsTask = configureCopyPluginsTask(project, pluginsDir); TaskProvider destructiveDistroTest = project.getTasks().register("destructiveDistroTest"); for (ElasticsearchDistribution distribution : distributions) { - TaskProvider destructiveTask = configureDistroTest(project, distribution); - destructiveDistroTest.configure(t -> t.dependsOn(destructiveTask)); + if (distribution.getType() != Type.DOCKER || runDockerTests == true) { + TaskProvider destructiveTask = configureDistroTest(project, distribution); + destructiveDistroTest.configure(t -> t.dependsOn(destructiveTask)); + } } Map> batsTests = new HashMap<>(); batsTests.put("bats oss", configureBatsTest(project, "oss", distributionsDir, copyDistributionsTask)); @@ -129,7 +139,23 @@ public void apply(Project project) { TaskProvider vmTask = configureVMWrapperTask(vmProject, distribution.getName() + " distribution", destructiveTaskName, vmDependencies); vmTask.configure(t -> t.dependsOn(distribution)); - distroTest.configure(t -> t.dependsOn(vmTask)); + + distroTest.configure(t -> { + // Only VM sub-projects that are specifically opted-in to testing Docker should + // have the Docker task added as a dependency. Although we control whether Docker + // is installed in the VM via `Vagrantfile` and we could auto-detect its presence + // in the VM, the test tasks e.g. `destructiveDistroTest.default-docker` are defined + // on the host during Gradle's configuration phase and not in the VM, so + // auto-detection doesn't work. + // + // The shouldTestDocker property could be null, hence we use Boolean.TRUE.equals() + boolean shouldExecute = distribution.getType() != Type.DOCKER + || Boolean.TRUE.equals(vmProject.findProperty("shouldTestDocker")) == true; + + if (shouldExecute) { + t.dependsOn(vmTask); + } + }); } } @@ -321,17 +347,17 @@ private static TaskProvider configureBatsTest(Project project, Str }); } - private List configureDistributions(Project project, Version upgradeVersion) { + private List configureDistributions(Project project, Version upgradeVersion, boolean runDockerTests) { NamedDomainObjectContainer distributions = DistributionDownloadPlugin.getContainer(project); List currentDistros = new ArrayList<>(); List upgradeDistros = new ArrayList<>(); - // Docker disabled for https://github.com/elastic/elasticsearch/issues/47639 - for (Type type : Arrays.asList(Type.DEB, Type.RPM /*,Type.DOCKER*/)) { + for (Type type : List.of(Type.DEB, Type.RPM, Type.DOCKER)) { for (Flavor flavor : Flavor.values()) { for (boolean bundledJdk : Arrays.asList(true, false)) { - // We should never add a Docker distro with bundledJdk == false - boolean skip = type == Type.DOCKER && bundledJdk == false; + // All our Docker images include a bundled JDK so it doesn't make sense to test without one + boolean skip = type == Type.DOCKER && (runDockerTests == false || bundledJdk == false); + if (skip == false) { addDistro(distributions, type, null, flavor, bundledJdk, VersionProperties.getElasticsearch(), currentDistros); } @@ -345,6 +371,7 @@ private List configureDistributions(Project project, addDistro(distributions, type, null, Flavor.OSS, true, upgradeVersion.toString(), upgradeDistros); } } + for (Platform platform : Arrays.asList(Platform.LINUX, Platform.WINDOWS)) { for (Flavor flavor : Flavor.values()) { for (boolean bundledJdk : Arrays.asList(true, false)) { @@ -405,4 +432,92 @@ private static String destructiveDistroTestTaskName(ElasticsearchDistribution di distro.getFlavor(), distro.getBundledJdk()); } + + static Map parseOsRelease(final List osReleaseLines) { + final Map values = new HashMap<>(); + + osReleaseLines.stream().map(String::trim).filter(line -> (line.isEmpty() || line.startsWith("#")) == false).forEach(line -> { + final String[] parts = line.split("=", 2); + final String key = parts[0]; + // remove optional leading and trailing quotes and whitespace + final String value = parts[1].replaceAll("^['\"]?\\s*", "").replaceAll("\\s*['\"]?$", ""); + + values.put(key, value); + }); + + return values; + } + + static String deriveId(final Map osRelease) { + return osRelease.get("ID") + "-" + osRelease.get("VERSION_ID"); + } + + private static List getLinuxExclusionList(Project project) { + final String exclusionsFilename = "dockerOnLinuxExclusions"; + final Path exclusionsPath = project.getRootDir().toPath().resolve(Path.of(".ci", exclusionsFilename)); + + try { + return Files.readAllLines(exclusionsPath) + .stream() + .map(String::trim) + .filter(line -> (line.isEmpty() || line.startsWith("#")) == false) + .collect(Collectors.toList()); + } catch (IOException e) { + throw new GradleException("Failed to read .ci/" + exclusionsFilename, e); + } + } + + /** + * The {@link DistroTestPlugin} generates a number of test tasks, some + * of which are Docker packaging tests. When running on the host OS or in CI + * i.e. not in a Vagrant VM, only certain operating systems are supported. This + * method determines whether the Docker tests should be run on the host + * OS. Essentially, unless an OS and version is specifically excluded, we expect + * to be able to run Docker and test the Docker images. + * @param project + */ + private static boolean shouldRunDockerTests(Project project) { + switch (OS.current()) { + case WINDOWS: + // Not yet supported. + return false; + + case MAC: + // Assume that Docker for Mac is installed, since Docker is part of the dev workflow. + return true; + + case LINUX: + // Only some hosts in CI are configured with Docker. We attempt to work out the OS + // and version, so that we know whether to expect to find Docker. We don't attempt + // to probe for whether Docker is available, because that doesn't tell us whether + // Docker is unavailable when it should be. + final Path osRelease = Paths.get("/etc/os-release"); + + if (Files.exists(osRelease)) { + Map values; + + try { + final List osReleaseLines = Files.readAllLines(osRelease); + values = parseOsRelease(osReleaseLines); + } catch (IOException e) { + throw new GradleException("Failed to read /etc/os-release", e); + } + + final String id = deriveId(values); + + final boolean shouldExclude = getLinuxExclusionList(project).contains(id); + + logger.warn("Linux OS id [" + id + "] is " + (shouldExclude ? "" : "not ") + "present in the Docker exclude list"); + + return shouldExclude == false; + } + + logger.warn("/etc/os-release does not exist!"); + return false; + + default: + logger.warn("Unknown OS [" + OS.current() + "], answering false to shouldRunDockerTests()"); + return false; + } + } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/tool/DockerUtils.java b/buildSrc/src/main/java/org/elasticsearch/gradle/tool/DockerUtils.java new file mode 100644 index 0000000000000..2442ffce427c8 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/tool/DockerUtils.java @@ -0,0 +1,239 @@ +package org.elasticsearch.gradle.tool; + +import org.elasticsearch.gradle.Version; +import org.gradle.api.GradleException; +import org.gradle.api.Project; +import org.gradle.process.ExecResult; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.util.List; +import java.util.Locale; +import java.util.Optional; + +/** + * Contains utilities for checking whether Docker is installed, is executable, + * has a recent enough version, and appears to be functional. The Elasticsearch build + * requires Docker >= 17.05 as it uses a multi-stage build. + */ +public class DockerUtils { + /** + * Defines the possible locations of the Docker CLI. These will be searched in order. + */ + private static String[] DOCKER_BINARIES = { "/usr/bin/docker", "/usr/local/bin/docker" }; + + /** + * Searches the entries in {@link #DOCKER_BINARIES} for the Docker CLI. This method does + * not check whether the Docker installation appears usable, see {@link #getDockerAvailability(Project)} + * instead. + * + * @return the path to a CLI, if available. + */ + public static Optional getDockerPath() { + // Check if the Docker binary exists + return List.of(DOCKER_BINARIES) + .stream() + .filter(path -> new File(path).exists()) + .findFirst(); + } + + /** + * Searches for a functional Docker installation, and returns information about the search. + * @return the results of the search. + */ + private static DockerAvailability getDockerAvailability(Project project) { + String dockerPath = null; + Result lastResult = null; + Version version = null; + boolean isVersionHighEnough = false; + + // Check if the Docker binary exists + final Optional dockerBinary = getDockerPath(); + + if (dockerBinary.isPresent()) { + dockerPath = dockerBinary.get(); + + // Since we use a multi-stage Docker build, check the Docker version since 17.05 + lastResult = runCommand(project, dockerPath, "version", "--format", "{{.Server.Version}}"); + + if (lastResult.isSuccess() == true) { + version = Version.fromString(lastResult.stdout.trim(), Version.Mode.RELAXED); + + isVersionHighEnough = version.onOrAfter("17.05.0"); + + if (isVersionHighEnough == true) { + // Check that we can execute a privileged command + lastResult = runCommand(project, dockerPath, "images"); + } + } + } + + boolean isAvailable = isVersionHighEnough && lastResult.isSuccess() == true; + + return new DockerAvailability(isAvailable, isVersionHighEnough, dockerPath, version, lastResult); + } + + /** + * An immutable class that represents the results of a Docker search from {@link #getDockerAvailability(Project)}}. + */ + private static class DockerAvailability { + /** + * Indicates whether Docker is available and meets the required criteria. + * True if, and only if, Docker is: + *
    + *
  • Installed
    • + *
  • Executable
    • + *
  • Is at least version 17.05
    • + *
  • Can execute a command that requires privileges
    • + *
+ */ + final boolean isAvailable; + + /** + * True if the installed Docker version is >= 17.05 + */ + final boolean isVersionHighEnough; + + /** + * The path to the Docker CLI, or null + */ + public final String path; + + /** + * The installed Docker version, or null + */ + public final Version version; + + /** + * Information about the last command executes while probing Docker, or null. + */ + final Result lastCommand; + + DockerAvailability(boolean isAvailable, boolean isVersionHighEnough, String path, Version version, Result lastCommand) { + this.isAvailable = isAvailable; + this.isVersionHighEnough = isVersionHighEnough; + this.path = path; + this.version = version; + this.lastCommand = lastCommand; + } + } + + /** + * Given a list of tasks that requires Docker, check whether Docker is available, otherwise + * throw an exception. + * @param project a Gradle project + * @param tasks the tasks that require Docker + * @throws GradleException if Docker is not available. The exception message gives the reason. + */ + public static void assertDockerIsAvailable(Project project, List tasks) { + DockerAvailability availability = getDockerAvailability(project); + + if (availability.isAvailable == true) { + return; + } + + /* + * There are tasks in the task graph that require Docker. + * Now we are failing because either the Docker binary does + * not exist or because execution of a privileged Docker + * command failed. + */ + if (availability.path == null) { + final String message = String.format( + Locale.ROOT, + "Docker (checked [%s]) is required to run the following task%s: \n%s", + String.join(", ", DOCKER_BINARIES), + tasks.size() > 1 ? "s" : "", + String.join("\n", tasks)); + throwDockerRequiredException(message); + } + + if (availability.version == null) { + final String message = String.format( + Locale.ROOT, + "Docker is required to run the following task%s, but it doesn't appear to be running: \n%s", + tasks.size() > 1 ? "s" : "", + String.join("\n", tasks)); + throwDockerRequiredException(message); + } + + if (availability.isVersionHighEnough == false) { + final String message = String.format( + Locale.ROOT, + "building Docker images requires Docker version 17.05+ due to use of multi-stage builds yet was [%s]", + availability.version); + throwDockerRequiredException(message); + } + + // Some other problem, print the error + final String message = String.format( + Locale.ROOT, + "a problem occurred running Docker from [%s] yet it is required to run the following task%s: \n%s\n" + + "the problem is that Docker exited with exit code [%d] with standard error output [%s]", + availability.path, + tasks.size() > 1 ? "s" : "", + String.join("\n", tasks), + availability.lastCommand.exitCode, + availability.lastCommand.stderr.trim()); + throwDockerRequiredException(message); + } + + private static void throwDockerRequiredException(final String message) { + throwDockerRequiredException(message, null); + } + + private static void throwDockerRequiredException(final String message, Exception e) { + throw new GradleException( + message + "\nyou can address this by attending to the reported issue, " + + "removing the offending tasks from being executed, " + + "or by passing -Dbuild.docker=false", e); + } + + /** + * Runs a command and captures the exit code, standard output and standard error. + * @param args the command and any arguments to execute + * @return a object that captures the result of running the command. If an exception occurring + * while running the command, or the process was killed after reaching the 10s timeout, + * then the exit code will be -1. + */ + private static Result runCommand(Project project, String... args) { + if (args.length == 0) { + throw new IllegalArgumentException("Cannot execute with no command"); + } + + ByteArrayOutputStream stdout = new ByteArrayOutputStream(); + ByteArrayOutputStream stderr = new ByteArrayOutputStream(); + + final ExecResult execResult = project.exec(spec -> { + // The redundant cast is to silence a compiler warning. + spec.setCommandLine((Object[]) args); + spec.setStandardOutput(stdout); + spec.setErrorOutput(stderr); + }); + + return new Result(execResult.getExitValue(), stdout.toString(), stderr.toString()); + } + + /** + * This class models the result of running a command. It captures the exit code, standard output and standard error. + */ + private static class Result { + final int exitCode; + final String stdout; + final String stderr; + + Result(int exitCode, String stdout, String stderr) { + this.exitCode = exitCode; + this.stdout = stdout; + this.stderr = stderr; + } + + boolean isSuccess() { + return exitCode == 0; + } + + public String toString() { + return "exitCode = [" + exitCode + "] " + "stdout = [" + stdout.trim() + "] " + "stderr = [" + stderr.trim() + "]"; + } + } +} diff --git a/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/Version.java b/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/Version.java index 31738f140878d..d31e15b842b2c 100644 --- a/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/Version.java +++ b/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/Version.java @@ -13,9 +13,28 @@ public final class Version implements Comparable { private final int revision; private final int id; + /** + * Specifies how a version string should be parsed. + */ + public enum Mode { + /** + * Strict parsing only allows known suffixes after the patch number: "-alpha", "-beta" or "-rc". The + * suffix "-SNAPSHOT" is also allowed, either after the patch number, or after the other suffices. + */ + STRICT, + + /** + * Relaxed parsing allows any alphanumeric suffix after the patch number. + */ + RELAXED + } + private static final Pattern pattern = Pattern.compile("(\\d)+\\.(\\d+)\\.(\\d+)(-alpha\\d+|-beta\\d+|-rc\\d+)?(-SNAPSHOT)?"); + private static final Pattern relaxedPattern = + Pattern.compile("(\\d)+\\.(\\d+)\\.(\\d+)(-[a-zA-Z0-9_]+)*?"); + public Version(int major, int minor, int revision) { Objects.requireNonNull(major, "major version can't be null"); Objects.requireNonNull(minor, "minor version can't be null"); @@ -36,11 +55,18 @@ private static int parseSuffixNumber(String substring) { } public static Version fromString(final String s) { + return fromString(s, Mode.STRICT); + } + + public static Version fromString(final String s, final Mode mode) { Objects.requireNonNull(s); - Matcher matcher = pattern.matcher(s); + Matcher matcher = mode == Mode.STRICT ? pattern.matcher(s) : relaxedPattern.matcher(s); if (matcher.matches() == false) { + String expected = mode == Mode.STRICT == true + ? "major.minor.revision[-(alpha|beta|rc)Number][-SNAPSHOT]" + : "major.minor.revision[-extra]"; throw new IllegalArgumentException( - "Invalid version format: '" + s + "'. Should be major.minor.revision[-(alpha|beta|rc)Number][-SNAPSHOT]" + "Invalid version format: '" + s + "'. Should be " + expected ); } diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/BuildPluginTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/BuildPluginTests.java index c61a0a3935898..8d6d0be00dbe8 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/BuildPluginTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/BuildPluginTests.java @@ -28,17 +28,6 @@ public class BuildPluginTests extends GradleUnitTestCase { - public void testPassingDockerVersions() { - BuildPlugin.checkDockerVersionRecent("Docker version 18.06.1-ce, build e68fc7a215d7"); - BuildPlugin.checkDockerVersionRecent("Docker version 17.05.0, build e68fc7a"); - BuildPlugin.checkDockerVersionRecent("Docker version 17.05.1, build e68fc7a"); - } - - @Test(expected = GradleException.class) - public void testFailingDockerVersions() { - BuildPlugin.checkDockerVersionRecent("Docker version 17.04.0, build e68fc7a"); - } - @Test(expected = GradleException.class) public void testRepositoryURIThatUsesHttpScheme() throws URISyntaxException { final URI uri = new URI("http://s3.amazonaws.com/artifacts.elastic.co/maven"); diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/VersionTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/VersionTests.java index 3394285157e17..ae2fb0e6215db 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/VersionTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/VersionTests.java @@ -40,6 +40,14 @@ public void testVersionParsing() { assertVersionEquals("6.1.2-beta1-SNAPSHOT", 6, 1, 2); } + public void testRelaxedVersionParsing() { + assertVersionEquals("6.1.2", 6, 1, 2, Version.Mode.RELAXED); + assertVersionEquals("6.1.2-SNAPSHOT", 6, 1, 2, Version.Mode.RELAXED); + assertVersionEquals("6.1.2-beta1-SNAPSHOT", 6, 1, 2, Version.Mode.RELAXED); + assertVersionEquals("6.1.2-foo", 6, 1, 2, Version.Mode.RELAXED); + assertVersionEquals("6.1.2-foo-bar", 6, 1, 2, Version.Mode.RELAXED); + } + public void testCompareWithStringVersions() { assertTrue("1.10.20 is not interpreted as before 2.0.0", Version.fromString("1.10.20").before("2.0.0") @@ -100,7 +108,11 @@ private void assertOrder(Version smaller, Version bigger) { } private void assertVersionEquals(String stringVersion, int major, int minor, int revision) { - Version version = Version.fromString(stringVersion); + assertVersionEquals(stringVersion, major, minor, revision, Version.Mode.STRICT); + } + + private void assertVersionEquals(String stringVersion, int major, int minor, int revision, Version.Mode mode) { + Version version = Version.fromString(stringVersion, mode); assertEquals(major, version.getMajor()); assertEquals(minor, version.getMinor()); assertEquals(revision, version.getRevision()); diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/test/DistroTestPluginTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/test/DistroTestPluginTests.java new file mode 100644 index 0000000000000..f88a4c11415bc --- /dev/null +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/test/DistroTestPluginTests.java @@ -0,0 +1,85 @@ +package org.elasticsearch.gradle.test; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.gradle.test.DistroTestPlugin.deriveId; +import static org.elasticsearch.gradle.test.DistroTestPlugin.parseOsRelease; +import static org.hamcrest.CoreMatchers.equalTo; + +public class DistroTestPluginTests extends GradleIntegrationTestCase { + + public void testParseOsReleaseOnOracle() { + final List lines = List + .of( + "NAME=\"Oracle Linux Server\"", + "VERSION=\"6.10\"", + "ID=\"ol\"", + "VERSION_ID=\"6.10\"", + "PRETTY_NAME=\"Oracle Linux Server 6.10\"", + "ANSI_COLOR=\"0;31\"", + "CPE_NAME=\"cpe:/o:oracle:linux:6:10:server\"", + "HOME_URL" + "=\"https://linux.oracle.com/\"", + "BUG_REPORT_URL=\"https://bugzilla.oracle.com/\"", + "", + "ORACLE_BUGZILLA_PRODUCT" + "=\"Oracle Linux 6\"", + "ORACLE_BUGZILLA_PRODUCT_VERSION=6.10", + "ORACLE_SUPPORT_PRODUCT=\"Oracle Linux\"", + "ORACLE_SUPPORT_PRODUCT_VERSION=6.10" + ); + + final Map results = parseOsRelease(lines); + + final Map expected = new HashMap<>(); + expected.put("ANSI_COLOR", "0;31"); + expected.put("BUG_REPORT_URL", "https://bugzilla.oracle.com/"); + expected.put("CPE_NAME", "cpe:/o:oracle:linux:6:10:server"); + expected.put("HOME_URL" + "", "https://linux.oracle.com/"); + expected.put("ID", "ol"); + expected.put("NAME", "Oracle Linux Server"); + expected.put("ORACLE_BUGZILLA_PRODUCT" + "", "Oracle Linux 6"); + expected.put("ORACLE_BUGZILLA_PRODUCT_VERSION", "6.10"); + expected.put("ORACLE_SUPPORT_PRODUCT", "Oracle Linux"); + expected.put("ORACLE_SUPPORT_PRODUCT_VERSION", "6.10"); + expected.put("PRETTY_NAME", "Oracle Linux Server 6.10"); + expected.put("VERSION", "6.10"); + expected.put("VERSION_ID", "6.10"); + + assertThat(expected, equalTo(results)); + } + + /** + * Trailing whitespace should be removed + */ + public void testRemoveTrailingWhitespace() { + final List lines = List.of("NAME=\"Oracle Linux Server\" "); + + final Map results = parseOsRelease(lines); + + final Map expected = Map.of("NAME", "Oracle Linux Server"); + + assertThat(expected, equalTo(results)); + } + + /** + * Comments should be removed + */ + public void testRemoveComments() { + final List lines = List.of("# A comment", "NAME=\"Oracle Linux Server\""); + + final Map results = parseOsRelease(lines); + + final Map expected = Map.of("NAME", "Oracle Linux Server"); + + assertThat(expected, equalTo(results)); + } + + public void testDeriveIdOnOracle() { + final Map osRelease = new HashMap<>(); + osRelease.put("ID", "ol"); + osRelease.put("VERSION_ID", "6.10"); + + assertThat("ol-6.10", equalTo(deriveId(osRelease))); + } +} diff --git a/qa/os/centos-7/build.gradle b/qa/os/centos-7/build.gradle index e69de29bb2d1d..814b04d4aec5f 100644 --- a/qa/os/centos-7/build.gradle +++ b/qa/os/centos-7/build.gradle @@ -0,0 +1 @@ +project.ext.shouldTestDocker = true diff --git a/qa/os/debian-9/build.gradle b/qa/os/debian-9/build.gradle index e69de29bb2d1d..814b04d4aec5f 100644 --- a/qa/os/debian-9/build.gradle +++ b/qa/os/debian-9/build.gradle @@ -0,0 +1 @@ +project.ext.shouldTestDocker = true diff --git a/qa/os/fedora-28/build.gradle b/qa/os/fedora-28/build.gradle index e69de29bb2d1d..814b04d4aec5f 100644 --- a/qa/os/fedora-28/build.gradle +++ b/qa/os/fedora-28/build.gradle @@ -0,0 +1 @@ +project.ext.shouldTestDocker = true diff --git a/qa/os/fedora-29/build.gradle b/qa/os/fedora-29/build.gradle index e69de29bb2d1d..814b04d4aec5f 100644 --- a/qa/os/fedora-29/build.gradle +++ b/qa/os/fedora-29/build.gradle @@ -0,0 +1 @@ +project.ext.shouldTestDocker = true diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java index daad67f7fb111..52205263d3ed3 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java @@ -28,7 +28,6 @@ import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; -import org.junit.Ignore; import java.nio.file.Files; import java.nio.file.Path; @@ -55,7 +54,6 @@ import static org.hamcrest.Matchers.emptyString; import static org.junit.Assume.assumeTrue; -@Ignore("https://github.com/elastic/elasticsearch/issues/47639") public class DockerTests extends PackagingTestCase { protected DockerShell sh; diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java index d78b60236bc4a..8e6bc4a46e13c 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java @@ -137,20 +137,24 @@ private static void waitForElasticsearchToStart() throws InterruptedException { boolean isElasticsearchRunning = false; int attempt = 0; + String psOutput; + do { - String psOutput = dockerShell.run("ps ax").stdout; + // Give the container a chance to crash out + Thread.sleep(1000); + + psOutput = dockerShell.run("ps ax").stdout; - if (psOutput.contains("/usr/share/elasticsearch/jdk/bin/java -X")) { + if (psOutput.contains("/usr/share/elasticsearch/jdk/bin/java")) { isElasticsearchRunning = true; break; } - Thread.sleep(1000); } while (attempt++ < 5); if (!isElasticsearchRunning) { - final String logs = sh.run("docker logs " + containerId).stdout; - fail("Elasticsearch container did start successfully.\n\n" + logs); + final String dockerLogs = sh.run("docker logs " + containerId).stdout; + fail("Elasticsearch container did start successfully.\n\n" + psOutput + "\n\n" + dockerLogs); } } @@ -240,7 +244,7 @@ public static void assertPermissionsAndOwnership(Path path, Set