Only define Docker pkg tests if Docker is available

.ci/dockerOnLinuxExclusions

@@ -0,0 +1,13 @@
+# This file specifies the Linux OS versions on which we can't build and
+# test Docker images for some reason. These values correspond to ID and
+# VERSION_ID from /etc/os-release, and a matching value will cause the
+# Docker tests to be skipped on that OS. If /etc/os-release doesn't exist
+# (as is the case on centos-6, for example) then that OS will again be
+# excluded.
+
+centos-6
+debian-8
+opensuse-15-1
+oraclelinux-6
+oraclelinux-7
+sles-12

.ci/os.sh

@@ -6,6 +6,16 @@ if which zypper > /dev/null ; then
     sudo zypper install -y insserv-compat
 fi
 
+if [ -e /etc/sysctl.d/99-gce.conf ]; then
+  # The GCE defaults disable IPv4 forwarding, which breaks the Docker
+  # build. Workaround this by renaming the file so that it is executed
+  # earlier than our own overrides.
+  #
+  # This ultimately needs to be fixed at the image level - see infra
+  # issue 15654.
+  sudo mv /etc/sysctl.d/99-gce.conf /etc/sysctl.d/98-gce.conf
+fi
+
 # Required by bats
 sudo touch /etc/is_vagrant_vm
 sudo useradd vagrant

Vagrantfile

@@ -126,7 +126,7 @@ Vagrant.configure(2) do |config|
   end
   'fedora-29'.tap do |box|
     config.vm.define box, define_opts do |config|
-      config.vm.box = 'elastic/fedora-28-x86_64'
+      config.vm.box = 'elastic/fedora-29-x86_64'
       dnf_common config, box
       dnf_docker config
     end
@@ -216,6 +216,10 @@ def ubuntu_docker(config)
 
     # Add vagrant to the Docker group, so that it can run commands
     usermod -aG docker vagrant
+
+    # Enable IPv4 forwarding
+    sed -i '/net.ipv4.ip_forward/s/^#//' /etc/sysctl.conf
+    systemctl restart networking
   SHELL
 end
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -79,15 +79,14 @@ import org.gradle.external.javadoc.CoreJavadocOptions
 import org.gradle.internal.jvm.Jvm
 import org.gradle.language.base.plugins.LifecycleBasePlugin
 import org.gradle.process.CommandLineArgumentProvider
-import org.gradle.process.ExecResult
-import org.gradle.process.ExecSpec
 import org.gradle.util.GradleVersion
 
 import java.nio.charset.StandardCharsets
 import java.nio.file.Files
-import java.util.regex.Matcher
 
 import static org.elasticsearch.gradle.tool.Boilerplate.maybeConfigure
+import static org.elasticsearch.gradle.tool.DockerUtils.assertDockerIsAvailable
+import static org.elasticsearch.gradle.tool.DockerUtils.getDockerPath
 
 /**
  * Encapsulates build configuration for elasticsearch projects.
@@ -188,8 +187,7 @@ class BuildPlugin implements Plugin<Project> {
              */
 
             // check if the Docker binary exists and record its path
-            final List<String> maybeDockerBinaries = ['/usr/bin/docker', '/usr/local/bin/docker']
-            final String dockerBinary = maybeDockerBinaries.find { it -> new File(it).exists() }
+            final String dockerBinary = getDockerPath().orElse(null)
 
             final boolean buildDocker
             final String buildDockerProperty = System.getProperty("build.docker")
@@ -208,84 +206,16 @@ class BuildPlugin implements Plugin<Project> {
             ext.set('requiresDocker', [])
             rootProject.gradle.taskGraph.whenReady { TaskExecutionGraph taskGraph ->
                 final List<String> tasks = taskGraph.allTasks.intersect(ext.get('requiresDocker') as List<Task>).collect { "  ${it.path}".toString()}
-                if (tasks.isEmpty() == false) {
-                    /*
-                     * There are tasks in the task graph that require Docker. Now we are failing because either the Docker binary does not
-                     * exist or because execution of a privileged Docker command failed.
-                     */
-                    if (dockerBinary == null) {
-                        final String message = String.format(
-                                Locale.ROOT,
-                                "Docker (checked [%s]) is required to run the following task%s: \n%s",
-                                maybeDockerBinaries.join(","),
-                                tasks.size() > 1 ? "s" : "",
-                                tasks.join('\n'))
-                        throwDockerRequiredException(message)
-                    }
-
-                    // we use a multi-stage Docker build, check the Docker version since 17.05
-                    final ByteArrayOutputStream dockerVersionOutput = new ByteArrayOutputStream()
-                    LoggedExec.exec(
-                            rootProject,
-                            { ExecSpec it ->
-                                it.commandLine = [dockerBinary, '--version']
-                                it.standardOutput = dockerVersionOutput
-                            })
-                    final String dockerVersion = dockerVersionOutput.toString().trim()
-                    checkDockerVersionRecent(dockerVersion)
-
-                    final ByteArrayOutputStream dockerImagesErrorOutput = new ByteArrayOutputStream()
-                    // the Docker binary executes, check that we can execute a privileged command
-                    final ExecResult dockerImagesResult = LoggedExec.exec(
-                            rootProject,
-                            { ExecSpec it ->
-                                it.commandLine = [dockerBinary, "images"]
-                                it.errorOutput = dockerImagesErrorOutput
-                                it.ignoreExitValue = true
-                            })
-
-                    if (dockerImagesResult.exitValue != 0) {
-                        final String message = String.format(
-                                Locale.ROOT,
-                                "a problem occurred running Docker from [%s] yet it is required to run the following task%s: \n%s\n" +
-                                        "the problem is that Docker exited with exit code [%d] with standard error output [%s]",
-                                dockerBinary,
-                                tasks.size() > 1 ? "s" : "",
-                                tasks.join('\n'),
-                                dockerImagesResult.exitValue,
-                                dockerImagesErrorOutput.toString().trim())
-                        throwDockerRequiredException(message)
-                    }
 
+                if (tasks.isEmpty() == false) {
+                    assertDockerIsAvailable(task.project, tasks)
                 }
             }
         }
 
         (ext.get('requiresDocker') as List<Task>).add(task)
     }
 
-    protected static void checkDockerVersionRecent(String dockerVersion) {
-        final Matcher matcher = dockerVersion =~ /Docker version (\d+\.\d+)\.\d+(?:-[a-zA-Z0-9]+)?, build [0-9a-f]{7,40}/
-        assert matcher.matches(): dockerVersion
-        final dockerMajorMinorVersion = matcher.group(1)
-        final String[] majorMinor = dockerMajorMinorVersion.split("\\.")
-        if (Integer.parseInt(majorMinor[0]) < 17
-                || (Integer.parseInt(majorMinor[0]) == 17 && Integer.parseInt(majorMinor[1]) < 5)) {
-            final String message = String.format(
-                    Locale.ROOT,
-                    "building Docker images requires Docker version 17.05+ due to use of multi-stage builds yet was [%s]",
-                    dockerVersion)
-            throwDockerRequiredException(message)
-        }
-    }
-
-    private static void throwDockerRequiredException(final String message) {
-        throw new GradleException(
-                message + "\nyou can address this by attending to the reported issue, "
-                        + "removing the offending tasks from being executed, "
-                        + "or by passing -Dbuild.docker=false")
-    }
-
     /** Add a check before gradle execution phase which ensures java home for the given java version is set. */
     static void requireJavaHome(Task task, int version) {
         // use root project for global accounting

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/DistroTestPlugin.java

@@ -28,17 +28,21 @@
 import org.elasticsearch.gradle.ElasticsearchDistribution.Type;
 import org.elasticsearch.gradle.Jdk;
 import org.elasticsearch.gradle.JdkDownloadPlugin;
+import org.elasticsearch.gradle.OS;
 import org.elasticsearch.gradle.Version;
 import org.elasticsearch.gradle.VersionProperties;
 import org.elasticsearch.gradle.vagrant.BatsProgressLogger;
 import org.elasticsearch.gradle.vagrant.VagrantBasePlugin;
 import org.elasticsearch.gradle.vagrant.VagrantExtension;
+import org.gradle.api.GradleException;
 import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.Task;
 import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.file.Directory;
+import org.gradle.api.logging.Logger;
+import org.gradle.api.logging.Logging;
 import org.gradle.api.plugins.ExtraPropertiesExtension;
 import org.gradle.api.plugins.JavaBasePlugin;
 import org.gradle.api.provider.Provider;
@@ -51,6 +55,7 @@
 import java.io.UncheckedIOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -65,6 +70,7 @@
 import static org.elasticsearch.gradle.vagrant.VagrantMachine.convertWindowsPath;
 
 public class DistroTestPlugin implements Plugin<Project> {
+    private static final Logger logger = Logging.getLogger(DistroTestPlugin.class);
 
     private static final String SYSTEM_JDK_VERSION = "11.0.2+9";
     private static final String SYSTEM_JDK_VENDOR = "openjdk";
@@ -83,6 +89,8 @@ public class DistroTestPlugin implements Plugin<Project> {
 
     @Override
     public void apply(Project project) {
+        final boolean runDockerTests = shouldRunDockerTests(project);
+
         project.getPluginManager().apply(DistributionDownloadPlugin.class);
         project.getPluginManager().apply(BuildPlugin.class);
 
@@ -94,15 +102,17 @@ public void apply(Project project) {
         Provider<Directory> upgradeDir = project.getLayout().getBuildDirectory().dir("packaging/upgrade");
         Provider<Directory> pluginsDir = project.getLayout().getBuildDirectory().dir("packaging/plugins");
 
-        List<ElasticsearchDistribution> distributions = configureDistributions(project, upgradeVersion);
+        List<ElasticsearchDistribution> distributions = configureDistributions(project, upgradeVersion, runDockerTests);
         TaskProvider<Copy> copyDistributionsTask = configureCopyDistributionsTask(project, distributionsDir);
         TaskProvider<Copy> copyUpgradeTask = configureCopyUpgradeTask(project, upgradeVersion, upgradeDir);
         TaskProvider<Copy> copyPluginsTask = configureCopyPluginsTask(project, pluginsDir);
 
         TaskProvider<Task> destructiveDistroTest = project.getTasks().register("destructiveDistroTest");
         for (ElasticsearchDistribution distribution : distributions) {
-            TaskProvider<?> destructiveTask = configureDistroTest(project, distribution);
-            destructiveDistroTest.configure(t -> t.dependsOn(destructiveTask));
+            if (distribution.getType() != Type.DOCKER || runDockerTests == true) {
+                TaskProvider<?> destructiveTask = configureDistroTest(project, distribution);
+                destructiveDistroTest.configure(t -> t.dependsOn(destructiveTask));
+            }
         }
         Map<String, TaskProvider<?>> batsTests = new HashMap<>();
         batsTests.put("bats oss", configureBatsTest(project, "oss", distributionsDir, copyDistributionsTask));
@@ -128,7 +138,18 @@ public void apply(Project project) {
                     TaskProvider<GradleDistroTestTask> vmTask =
                         configureVMWrapperTask(vmProject, distribution.getName() + " distribution", destructiveTaskName, vmDependencies);
                     vmTask.configure(t -> t.dependsOn(distribution));
-                    distroTest.configure(t -> t.dependsOn(vmTask));
+
+                    distroTest.configure(t -> {
+                        // Only VM sub-projects that are specifically opted-in to testing Docker should
+                        // have the Docker task added as a dependency. The shouldTestDocker property could
+                        // be null, hence we use Boolean.TRUE.equals()
+                        boolean shouldExecute = distribution.getType() != Type.DOCKER
+                            || Boolean.TRUE.equals(vmProject.findProperty("shouldTestDocker")) == true;
+
+                        if (shouldExecute) {
+                            t.dependsOn(vmTask);
+                        }
+                    });
                 }
             }
 
@@ -321,17 +342,17 @@ private static TaskProvider<BatsTestTask> configureBatsTest(Project project, Str
             });
     }
 
-    private List<ElasticsearchDistribution> configureDistributions(Project project, Version upgradeVersion) {
+    private List<ElasticsearchDistribution> configureDistributions(Project project, Version upgradeVersion, boolean runDockerTests) {
         NamedDomainObjectContainer<ElasticsearchDistribution> distributions = DistributionDownloadPlugin.getContainer(project);
         List<ElasticsearchDistribution> currentDistros = new ArrayList<>();
         List<ElasticsearchDistribution> upgradeDistros = new ArrayList<>();
 
-        // Docker disabled for https://github.com/elastic/elasticsearch/issues/47639
-        for (Type type : Arrays.asList(Type.DEB, Type.RPM /*,Type.DOCKER*/)) {
+        for (Type type : List.of(Type.DEB, Type.RPM, Type.DOCKER)) {
             for (Flavor flavor : Flavor.values()) {
                 for (boolean bundledJdk : Arrays.asList(true, false)) {
-                    // We should never add a Docker distro with bundledJdk == false
-                    boolean skip = type == Type.DOCKER && bundledJdk == false;
+                    // All our Docker images include a bundled JDK so it doesn't make sense to test without one
+                    boolean skip = type == Type.DOCKER && (runDockerTests == false || bundledJdk == false);
+
                     if (skip == false) {
                         addDistro(distributions, type, null, flavor, bundledJdk, VersionProperties.getElasticsearch(), currentDistros);
                     }
@@ -345,6 +366,7 @@ private List<ElasticsearchDistribution> configureDistributions(Project project,
                 addDistro(distributions, type, null, Flavor.OSS, true, upgradeVersion.toString(), upgradeDistros);
             }
         }
+
         for (Platform platform : Arrays.asList(Platform.LINUX, Platform.WINDOWS)) {
             for (Flavor flavor : Flavor.values()) {
                 for (boolean bundledJdk : Arrays.asList(true, false)) {
@@ -405,4 +427,98 @@ private static String destructiveDistroTestTaskName(ElasticsearchDistribution di
             distro.getFlavor(),
             distro.getBundledJdk());
     }
+
+    static Map<String, String> parseOsRelease(final List<String> osReleaseLines) {
+        final Map<String, String> values = new HashMap<>();
+
+        for (String line : osReleaseLines) {
+            final String trimmed = line.trim();
+
+            // ignore empty and comment lines
+            if (trimmed.isEmpty() || trimmed.startsWith("#")) {
+                continue;
+            }
+
+            final String[] parts = line.split("=", 2);
+            final String key = parts[0];
+            // remove optional leading and trailing quotes and whitespace
+            final String value = parts[1].replaceAll("^['\"]?\\s*", "").replaceAll("\\s*['\"]?$", "");
+            values.put(key, value);
+        }
+
+        return values;
+    }
+
+    static String deriveId(final Map<String, String> osRelease) {
+        return osRelease.get("ID") + "-" + osRelease.get("VERSION_ID");
+    }
+
+    private static List<String> getLinuxExclusionList(Project project) {
+        final String exclusionsFilename = "dockerOnLinuxExclusions";
+        final Path exclusionsPath = project.getRootDir().toPath().resolve(Path.of(".ci", exclusionsFilename));
+
+        try {
+            return Files.readAllLines(exclusionsPath)
+                .stream()
+                .map(String::trim)
+                .filter(line -> line.startsWith("#"))
+                .collect(Collectors.toList());
+        } catch (IOException e) {
+            throw new GradleException("Failed to read .ci/" + exclusionsFilename, e);
+        }
+    }
+
+    /**
+     * The {@link DistroTestPlugin} generates a number of test tasks, some
+     * of which are Docker packaging tests. When running on the host OS or in CI
+     * i.e. not in a Vagrant VM, only certain operating systems are supported. This
+     * method determines whether the Docker tests should be run on the host
+     * OS. Essentially, unless an OS and version is specifically excluded, we expect
+     * to be able to run Docker and test the Docker images.
+     * @param project
+     */
+    private static boolean shouldRunDockerTests(Project project) {
+        switch (OS.current()) {
+            case WINDOWS:
+                // Not yet supported.
+                return false;
+
+            case MAC:
+                // Assume that Docker for Mac is installed, since Docker is part of the dev workflow.
+                return true;
+
+            case LINUX:
+                // Only some hosts in CI are configured with Docker. We attempt to work out the OS
+                // and version, so that we know whether to expect to find Docker. We don't attempt
+                // to probe for whether Docker is available, because that doesn't tell us whether
+                // Docker is unavailable when it should be.
+                final Path osRelease = Paths.get("/etc/os-release");
+
+                if (Files.exists(osRelease)) {
+                    Map<String, String> values;
+
+                    try {
+                        final List<String> osReleaseLines = Files.readAllLines(osRelease);
+                        values = parseOsRelease(osReleaseLines);
+                    } catch (IOException e) {
+                        throw new GradleException("Failed to read /etc/os-release", e);
+                    }
+
+                    final String id = deriveId(values);
+
+                    final boolean shouldExclude = getLinuxExclusionList(project).contains(id);
+
+                    logger.info("Linux OS id [" + id + "] is " + (shouldExclude ? "" : "not ") + "present in the exclude list");
+
+                    return shouldExclude == false;
+                }
+
+                logger.warn("/etc/os-release does not exist!");
+                return false;
+
+            default:
+                logger.warn("Unknown OS [" + OS.current() + "], answering false to shouldRunDockerTests()");
+                return false;
+        }
+    }
 }