Only define Docker pkg tests if Docker is available (#47640)

Closes #47639, and unmutes tests that were muted in b958467.

The Docker packaging tests were being defined irrespective of whether
Docker was actually available in the current environment. Instead,
implement exclude lists so that in environments where Docker is not
available, no Docker packaging tests are defined. For CI hosts, the build
checks `.ci/dockerOnLinuxExclusions`. The Vagrant VMs can defined the 
extension property `shouldTestDocker` property to opt-in to packaging
tests.

As part of this, define a seperate utility class for checking Docker,
and call that instead of defining checks in-line in BuildPlugin.groovy

.ci/dockerOnLinuxExclusions

@@ -0,0 +1,12 @@
+# This file specifies the Linux OS versions on which we can't build and
+# test Docker images for some reason. These values correspond to ID and
+# VERSION_ID from /etc/os-release, and a matching value will cause the
+# Docker tests to be skipped on that OS. If /etc/os-release doesn't exist
+# (as is the case on centos-6, for example) then that OS will again be
+# excluded.
+centos-6
+debian-8
+opensuse-15-1
+ol-6.10
+ol-7.7
+sles-12

.ci/os.sh

@@ -6,6 +6,16 @@ if which zypper > /dev/null ; then
     sudo zypper install -y insserv-compat
 fi
 
+if [ -e /etc/sysctl.d/99-gce.conf ]; then
+  # The GCE defaults disable IPv4 forwarding, which breaks the Docker
+  # build. Workaround this by renaming the file so that it is executed
+  # earlier than our own overrides.
+  #
+  # This ultimately needs to be fixed at the image level - see infra
+  # issue 15654.
+  sudo mv /etc/sysctl.d/99-gce.conf /etc/sysctl.d/98-gce.conf
+fi
+
 # Required by bats
 sudo touch /etc/is_vagrant_vm
 sudo useradd vagrant

Vagrantfile

@@ -126,7 +126,7 @@ Vagrant.configure(2) do |config|
   end
   'fedora-29'.tap do |box|
     config.vm.define box, define_opts do |config|
-      config.vm.box = 'elastic/fedora-28-x86_64'
+      config.vm.box = 'elastic/fedora-29-x86_64'
       dnf_common config, box
       dnf_docker config
     end
@@ -216,6 +216,10 @@ def ubuntu_docker(config)
 
     # Add vagrant to the Docker group, so that it can run commands
     usermod -aG docker vagrant
+
+    # Enable IPv4 forwarding
+    sed -i '/net.ipv4.ip_forward/s/^#//' /etc/sysctl.conf
+    systemctl restart networking
   SHELL
 end
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -76,15 +76,14 @@ import org.gradle.external.javadoc.CoreJavadocOptions
 import org.gradle.internal.jvm.Jvm
 import org.gradle.language.base.plugins.LifecycleBasePlugin
 import org.gradle.process.CommandLineArgumentProvider
-import org.gradle.process.ExecResult
-import org.gradle.process.ExecSpec
 import org.gradle.util.GradleVersion
 
 import java.nio.charset.StandardCharsets
 import java.nio.file.Files
-import java.util.regex.Matcher
 
 import static org.elasticsearch.gradle.tool.Boilerplate.maybeConfigure
+import static org.elasticsearch.gradle.tool.DockerUtils.assertDockerIsAvailable
+import static org.elasticsearch.gradle.tool.DockerUtils.getDockerPath
 
 /**
  * Encapsulates build configuration for elasticsearch projects.
@@ -183,8 +182,7 @@ class BuildPlugin implements Plugin<Project> {
              */
 
             // check if the Docker binary exists and record its path
-            final List<String> maybeDockerBinaries = ['/usr/bin/docker', '/usr/local/bin/docker']
-            final String dockerBinary = maybeDockerBinaries.find { it -> new File(it).exists() }
+            final String dockerBinary = getDockerPath().orElse(null)
 
             final boolean buildDocker
             final String buildDockerProperty = System.getProperty("build.docker")
@@ -203,84 +201,16 @@ class BuildPlugin implements Plugin<Project> {
             ext.set('requiresDocker', [])
             rootProject.gradle.taskGraph.whenReady { TaskExecutionGraph taskGraph ->
                 final List<String> tasks = taskGraph.allTasks.intersect(ext.get('requiresDocker') as List<Task>).collect { "  ${it.path}".toString()}
-                if (tasks.isEmpty() == false) {
-                    /*
-                     * There are tasks in the task graph that require Docker. Now we are failing because either the Docker binary does not
-                     * exist or because execution of a privileged Docker command failed.
-                     */
-                    if (dockerBinary == null) {
-                        final String message = String.format(
-                                Locale.ROOT,
-                                "Docker (checked [%s]) is required to run the following task%s: \n%s",
-                                maybeDockerBinaries.join(","),
-                                tasks.size() > 1 ? "s" : "",
-                                tasks.join('\n'))
-                        throwDockerRequiredException(message)
-                    }
-
-                    // we use a multi-stage Docker build, check the Docker version since 17.05
-                    final ByteArrayOutputStream dockerVersionOutput = new ByteArrayOutputStream()
-                    LoggedExec.exec(
-                            rootProject,
-                            { ExecSpec it ->
-                                it.commandLine = [dockerBinary, '--version']
-                                it.standardOutput = dockerVersionOutput
-                            })
-                    final String dockerVersion = dockerVersionOutput.toString().trim()
-                    checkDockerVersionRecent(dockerVersion)
-
-                    final ByteArrayOutputStream dockerImagesErrorOutput = new ByteArrayOutputStream()
-                    // the Docker binary executes, check that we can execute a privileged command
-                    final ExecResult dockerImagesResult = LoggedExec.exec(
-                            rootProject,
-                            { ExecSpec it ->
-                                it.commandLine = [dockerBinary, "images"]
-                                it.errorOutput = dockerImagesErrorOutput
-                                it.ignoreExitValue = true
-                            })
-
-                    if (dockerImagesResult.exitValue != 0) {
-                        final String message = String.format(
-                                Locale.ROOT,
-                                "a problem occurred running Docker from [%s] yet it is required to run the following task%s: \n%s\n" +
-                                        "the problem is that Docker exited with exit code [%d] with standard error output [%s]",
-                                dockerBinary,
-                                tasks.size() > 1 ? "s" : "",
-                                tasks.join('\n'),
-                                dockerImagesResult.exitValue,
-                                dockerImagesErrorOutput.toString().trim())
-                        throwDockerRequiredException(message)
-                    }
 
+                if (tasks.isEmpty() == false) {
+                    assertDockerIsAvailable(task.project, tasks)
                 }
             }
         }
 
         (ext.get('requiresDocker') as List<Task>).add(task)
     }
 
-    protected static void checkDockerVersionRecent(String dockerVersion) {
-        final Matcher matcher = dockerVersion =~ /Docker version (\d+\.\d+)\.\d+(?:-[a-zA-Z0-9]+)?, build [0-9a-f]{7,40}/
-        assert matcher.matches(): dockerVersion
-        final dockerMajorMinorVersion = matcher.group(1)
-        final String[] majorMinor = dockerMajorMinorVersion.split("\\.")
-        if (Integer.parseInt(majorMinor[0]) < 17
-                || (Integer.parseInt(majorMinor[0]) == 17 && Integer.parseInt(majorMinor[1]) < 5)) {
-            final String message = String.format(
-                    Locale.ROOT,
-                    "building Docker images requires Docker version 17.05+ due to use of multi-stage builds yet was [%s]",
-                    dockerVersion)
-            throwDockerRequiredException(message)
-        }
-    }
-
-    private static void throwDockerRequiredException(final String message) {
-        throw new GradleException(
-                message + "\nyou can address this by attending to the reported issue, "
-                        + "removing the offending tasks from being executed, "
-                        + "or by passing -Dbuild.docker=false")
-    }
-
     /** Add a check before gradle execution phase which ensures java home for the given java version is set. */
     static void requireJavaHome(Task task, int version) {
         // use root project for global accounting

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/DistroTestPlugin.java

@@ -28,18 +28,22 @@
 import org.elasticsearch.gradle.ElasticsearchDistribution.Type;
 import org.elasticsearch.gradle.Jdk;
 import org.elasticsearch.gradle.JdkDownloadPlugin;
+import org.elasticsearch.gradle.OS;
 import org.elasticsearch.gradle.Version;
 import org.elasticsearch.gradle.VersionProperties;
 import org.elasticsearch.gradle.info.BuildParams;
 import org.elasticsearch.gradle.vagrant.BatsProgressLogger;
 import org.elasticsearch.gradle.vagrant.VagrantBasePlugin;
 import org.elasticsearch.gradle.vagrant.VagrantExtension;
+import org.gradle.api.GradleException;
 import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.Task;
 import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.file.Directory;
+import org.gradle.api.logging.Logger;
+import org.gradle.api.logging.Logging;
 import org.gradle.api.plugins.ExtraPropertiesExtension;
 import org.gradle.api.plugins.JavaBasePlugin;
 import org.gradle.api.provider.Provider;
@@ -52,6 +56,7 @@
 import java.io.UncheckedIOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -66,6 +71,7 @@
 import static org.elasticsearch.gradle.vagrant.VagrantMachine.convertWindowsPath;
 
 public class DistroTestPlugin implements Plugin<Project> {
+    private static final Logger logger = Logging.getLogger(DistroTestPlugin.class);
 
     private static final String SYSTEM_JDK_VERSION = "11.0.2+9";
     private static final String SYSTEM_JDK_VENDOR = "openjdk";
@@ -84,6 +90,8 @@ public class DistroTestPlugin implements Plugin<Project> {
 
     @Override
     public void apply(Project project) {
+        final boolean runDockerTests = shouldRunDockerTests(project);
+
         project.getPluginManager().apply(DistributionDownloadPlugin.class);
         project.getPluginManager().apply(BuildPlugin.class);
 
@@ -95,15 +103,17 @@ public void apply(Project project) {
         Provider<Directory> upgradeDir = project.getLayout().getBuildDirectory().dir("packaging/upgrade");
         Provider<Directory> pluginsDir = project.getLayout().getBuildDirectory().dir("packaging/plugins");
 
-        List<ElasticsearchDistribution> distributions = configureDistributions(project, upgradeVersion);
+        List<ElasticsearchDistribution> distributions = configureDistributions(project, upgradeVersion, runDockerTests);
         TaskProvider<Copy> copyDistributionsTask = configureCopyDistributionsTask(project, distributionsDir);
         TaskProvider<Copy> copyUpgradeTask = configureCopyUpgradeTask(project, upgradeVersion, upgradeDir);
         TaskProvider<Copy> copyPluginsTask = configureCopyPluginsTask(project, pluginsDir);
 
         TaskProvider<Task> destructiveDistroTest = project.getTasks().register("destructiveDistroTest");
         for (ElasticsearchDistribution distribution : distributions) {
-            TaskProvider<?> destructiveTask = configureDistroTest(project, distribution);
-            destructiveDistroTest.configure(t -> t.dependsOn(destructiveTask));
+            if (distribution.getType() != Type.DOCKER || runDockerTests == true) {
+                TaskProvider<?> destructiveTask = configureDistroTest(project, distribution);
+                destructiveDistroTest.configure(t -> t.dependsOn(destructiveTask));
+            }
         }
         Map<String, TaskProvider<?>> batsTests = new HashMap<>();
         batsTests.put("bats oss", configureBatsTest(project, "oss", distributionsDir, copyDistributionsTask));
@@ -129,7 +139,23 @@ public void apply(Project project) {
                     TaskProvider<GradleDistroTestTask> vmTask =
                         configureVMWrapperTask(vmProject, distribution.getName() + " distribution", destructiveTaskName, vmDependencies);
                     vmTask.configure(t -> t.dependsOn(distribution));
-                    distroTest.configure(t -> t.dependsOn(vmTask));
+
+                    distroTest.configure(t -> {
+                        // Only VM sub-projects that are specifically opted-in to testing Docker should
+                        // have the Docker task added as a dependency. Although we control whether Docker
+                        // is installed in the VM via `Vagrantfile` and we could auto-detect its presence
+                        // in the VM, the test tasks e.g. `destructiveDistroTest.default-docker` are defined
+                        // on the host during Gradle's configuration phase and not in the VM, so
+                        // auto-detection doesn't work.
+                        //
+                        // The shouldTestDocker property could be null, hence we use Boolean.TRUE.equals()
+                        boolean shouldExecute = distribution.getType() != Type.DOCKER
+                            || Boolean.TRUE.equals(vmProject.findProperty("shouldTestDocker")) == true;
+
+                        if (shouldExecute) {
+                            t.dependsOn(vmTask);
+                        }
+                    });
                 }
             }
 
@@ -321,17 +347,17 @@ private static TaskProvider<BatsTestTask> configureBatsTest(Project project, Str
             });
     }
 
-    private List<ElasticsearchDistribution> configureDistributions(Project project, Version upgradeVersion) {
+    private List<ElasticsearchDistribution> configureDistributions(Project project, Version upgradeVersion, boolean runDockerTests) {
         NamedDomainObjectContainer<ElasticsearchDistribution> distributions = DistributionDownloadPlugin.getContainer(project);
         List<ElasticsearchDistribution> currentDistros = new ArrayList<>();
         List<ElasticsearchDistribution> upgradeDistros = new ArrayList<>();
 
-        // Docker disabled for https://github.com/elastic/elasticsearch/issues/47639
-        for (Type type : Arrays.asList(Type.DEB, Type.RPM /*,Type.DOCKER*/)) {
+        for (Type type : List.of(Type.DEB, Type.RPM, Type.DOCKER)) {
             for (Flavor flavor : Flavor.values()) {
                 for (boolean bundledJdk : Arrays.asList(true, false)) {
-                    // We should never add a Docker distro with bundledJdk == false
-                    boolean skip = type == Type.DOCKER && bundledJdk == false;
+                    // All our Docker images include a bundled JDK so it doesn't make sense to test without one
+                    boolean skip = type == Type.DOCKER && (runDockerTests == false || bundledJdk == false);
+
                     if (skip == false) {
                         addDistro(distributions, type, null, flavor, bundledJdk, VersionProperties.getElasticsearch(), currentDistros);
                     }
@@ -345,6 +371,7 @@ private List<ElasticsearchDistribution> configureDistributions(Project project,
                 addDistro(distributions, type, null, Flavor.OSS, true, upgradeVersion.toString(), upgradeDistros);
             }
         }
+
         for (Platform platform : Arrays.asList(Platform.LINUX, Platform.WINDOWS)) {
             for (Flavor flavor : Flavor.values()) {
                 for (boolean bundledJdk : Arrays.asList(true, false)) {
@@ -405,4 +432,92 @@ private static String destructiveDistroTestTaskName(ElasticsearchDistribution di
             distro.getFlavor(),
             distro.getBundledJdk());
     }
+
+    static Map<String, String> parseOsRelease(final List<String> osReleaseLines) {
+        final Map<String, String> values = new HashMap<>();
+
+        osReleaseLines.stream().map(String::trim).filter(line -> (line.isEmpty() || line.startsWith("#")) == false).forEach(line -> {
+            final String[] parts = line.split("=", 2);
+            final String key = parts[0];
+            // remove optional leading and trailing quotes and whitespace
+            final String value = parts[1].replaceAll("^['\"]?\\s*", "").replaceAll("\\s*['\"]?$", "");
+
+            values.put(key, value);
+        });
+
+        return values;
+    }
+
+    static String deriveId(final Map<String, String> osRelease) {
+        return osRelease.get("ID") + "-" + osRelease.get("VERSION_ID");
+    }
+
+    private static List<String> getLinuxExclusionList(Project project) {
+        final String exclusionsFilename = "dockerOnLinuxExclusions";
+        final Path exclusionsPath = project.getRootDir().toPath().resolve(Path.of(".ci", exclusionsFilename));
+
+        try {
+            return Files.readAllLines(exclusionsPath)
+                .stream()
+                .map(String::trim)
+                .filter(line -> (line.isEmpty() || line.startsWith("#")) == false)
+                .collect(Collectors.toList());
+        } catch (IOException e) {
+            throw new GradleException("Failed to read .ci/" + exclusionsFilename, e);
+        }
+    }
+
+    /**
+     * The {@link DistroTestPlugin} generates a number of test tasks, some
+     * of which are Docker packaging tests. When running on the host OS or in CI
+     * i.e. not in a Vagrant VM, only certain operating systems are supported. This
+     * method determines whether the Docker tests should be run on the host
+     * OS. Essentially, unless an OS and version is specifically excluded, we expect
+     * to be able to run Docker and test the Docker images.
+     * @param project
+     */
+    private static boolean shouldRunDockerTests(Project project) {
+        switch (OS.current()) {
+            case WINDOWS:
+                // Not yet supported.
+                return false;
+
+            case MAC:
+                // Assume that Docker for Mac is installed, since Docker is part of the dev workflow.
+                return true;
+
+            case LINUX:
+                // Only some hosts in CI are configured with Docker. We attempt to work out the OS
+                // and version, so that we know whether to expect to find Docker. We don't attempt
+                // to probe for whether Docker is available, because that doesn't tell us whether
+                // Docker is unavailable when it should be.
+                final Path osRelease = Paths.get("/etc/os-release");
+
+                if (Files.exists(osRelease)) {
+                    Map<String, String> values;
+
+                    try {
+                        final List<String> osReleaseLines = Files.readAllLines(osRelease);
+                        values = parseOsRelease(osReleaseLines);
+                    } catch (IOException e) {
+                        throw new GradleException("Failed to read /etc/os-release", e);
+                    }
+
+                    final String id = deriveId(values);
+
+                    final boolean shouldExclude = getLinuxExclusionList(project).contains(id);
+
+                    logger.warn("Linux OS id [" + id + "] is " + (shouldExclude ? "" : "not ") + "present in the Docker exclude list");
+
+                    return shouldExclude == false;
+                }
+
+                logger.warn("/etc/os-release does not exist!");
+                return false;
+
+            default:
+                logger.warn("Unknown OS [" + OS.current() + "], answering false to shouldRunDockerTests()");
+                return false;
+        }
+    }
 }