Introduce packaging tests for Docker

Vagrantfile

@@ -1,5 +1,5 @@
 # -*- mode: ruby -*-
-# vi: set ft=ruby :
+# vi:ft=ruby ts=2 sw=2 sts=2 et:
 
 # This Vagrantfile exists to test packaging. Read more about its use in the
 # vagrant section in TESTING.asciidoc.
@@ -183,6 +183,35 @@ def deb_common(config, name, extra: '')
     install_command: 'apt-get install -y',
     extra: extra_with_lintian
   )
+  deb_docker(config)
+end
+
+def deb_docker(config)
+  config.vm.provision 'install Docker', run: 'always', type: 'shell', inline: <<-SHELL
+    # Install packages to allow apt to use a repository over HTTPS
+    apt-get install -y \
+      apt-transport-https \
+      ca-certificates \
+      curl \
+      gnupg2 \
+      software-properties-common
+
+    # Add Docker’s official GPG key
+    curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add -
+
+    # Set up the stable Docker repository
+    add-apt-repository \
+      "deb [arch=amd64] https://download.docker.com/linux/debian \
+      $(lsb_release -cs) \
+      stable"
+
+    # Install Docker. Unlike Fedora and CentOS, this also start the daemon.
+    apt-get update
+    apt-get install -y docker-ce docker-ce-cli containerd.io
+
+    # Add vagrant to the Docker group, so that it can run commands
+    usermod -aG docker vagrant
+  SHELL
 end
 
 def rpm_common(config, name)
@@ -193,6 +222,26 @@ def rpm_common(config, name)
     update_tracking_file: '/var/cache/yum/last_update',
     install_command: 'yum install -y'
   )
+  rpm_docker(config)
+end
+
+def rpm_docker(config)
+  config.vm.provision 'install Docker', run: 'always', type: 'shell', inline: <<-SHELL
+    # Install prerequisites
+    yum install -y yum-utils device-mapper-persistent-data lvm2
+
+    # Add repository
+    yum-config-manager -y --add-repo https://download.docker.com/linux/centos/docker-ce.repo
+
+    # Install Docker
+    yum install -y docker-ce docker-ce-cli containerd.io
+
+    # Start Docker
+    systemctl start docker
+
+    # Add vagrant to the Docker group, so that it can run commands
+    usermod -aG docker vagrant
+  SHELL
 end
 
 def dnf_common(config, name)
@@ -209,6 +258,26 @@ def dnf_common(config, name)
     install_command: 'dnf install -y',
     install_command_retries: 5
   )
+  dnf_docker(config)
+end
+
+def dnf_docker(config)
+  config.vm.provision 'install Docker', run: 'always', type: 'shell', inline: <<-SHELL
+    # Install prerequisites
+    dnf -y install dnf-plugins-core
+
+    # Add repository
+    dnf config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo
+
+    # Install Docker
+    dnf install -y docker-ce docker-ce-cli containerd.io
+
+    # Start Docker
+    systemctl start docker
+
+    # Add vagrant to the Docker group, so that it can run commands
+    usermod -aG docker vagrant
+  SHELL
 end
 
 def suse_common(config, name, extra: '')

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/DistroTestPlugin.java

@@ -172,7 +172,7 @@ private static List<Object> configureVM(Project project) {
         String box = project.getName();
 
         // setup jdks used by the distro tests, and by gradle executing
-        
+
         NamedDomainObjectContainer<Jdk> jdksContainer = JdkDownloadPlugin.getContainer(project);
         String platform = box.contains("windows") ? "windows" : "linux";
         Jdk systemJdk = createJdk(jdksContainer, "system", SYSTEM_JDK_VERSION, platform);
@@ -309,13 +309,13 @@ private static TaskProvider<BatsTestTask> configureBatsTest(Project project, Str
                 }
             });
     }
-    
+
     private List<ElasticsearchDistribution> configureDistributions(Project project, Version upgradeVersion) {
         NamedDomainObjectContainer<ElasticsearchDistribution> distributions = DistributionDownloadPlugin.getContainer(project);
         List<ElasticsearchDistribution> currentDistros = new ArrayList<>();
         List<ElasticsearchDistribution> upgradeDistros = new ArrayList<>();
 
-        for (Type type : Arrays.asList(Type.DEB, Type.RPM)) {
+        for (Type type : Arrays.asList(Type.DEB, Type.RPM, Type.DOCKER)) {
             for (Flavor flavor : Flavor.values()) {
                 for (boolean bundledJdk : Arrays.asList(true, false)) {
                     addDistro(distributions, type, null, flavor, bundledJdk, VersionProperties.getElasticsearch(), currentDistros);
@@ -366,7 +366,8 @@ private static void addDistro(NamedDomainObjectContainer<ElasticsearchDistributi
             if (type == Type.ARCHIVE) {
                 d.setPlatform(platform);
             }
-            d.setBundledJdk(bundledJdk);
+            // We don't test Docker images with a non-bundled JDK
+            d.setBundledJdk(type == Type.DOCKER || bundledJdk);
             d.setVersion(version);
         });
         container.add(distro);
@@ -378,10 +379,17 @@ private static boolean isWindows(Project project) {
     }
 
     private static String distroId(Type type, Platform platform, Flavor flavor, boolean bundledJdk) {
-        return flavor + "-" + (type == Type.ARCHIVE ? platform + "-" : "") + type + (bundledJdk ? "" : "-no-jdk");
+        // We don't test Docker images with a non-bundled JDK
+        return flavor + "-" + (type == Type.ARCHIVE ? platform + "-" : "") + type + (bundledJdk || type == Type.DOCKER ? "" : "-no-jdk");
     }
 
     private static String destructiveDistroTestTaskName(ElasticsearchDistribution distro) {
-        return "destructiveDistroTest." + distroId(distro.getType(), distro.getPlatform(), distro.getFlavor(), distro.getBundledJdk());
+        Type type = distro.getType();
+        return "destructiveDistroTest." + distroId(
+            type,
+            distro.getPlatform(),
+            distro.getFlavor(),
+            // We don't test Docker images with a non-bundled JDK
+            type == Type.DOCKER || distro.getBundledJdk());
     }
 }

buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java

@@ -93,8 +93,9 @@ void setupDistributions(Project project) {
             // for the distribution as a file, just depend on the artifact directly
             dependencies.add(distribution.configuration.getName(), dependencyNotation(project, distribution));
 
-            // no extraction allowed for rpm or deb
-            if (distribution.getType() != Type.RPM && distribution.getType() != Type.DEB) {
+            // no extraction allowed for rpm, deb or docker
+            Type distroType = distribution.getType();
+            if (distroType != Type.RPM && distroType != Type.DEB && distroType != Type.DOCKER) {
                 // for the distribution extracted, add a root level task that does the extraction, and depend on that
                 // extracted configuration as an artifact consisting of the extracted distribution directory
                 dependencies.add(distribution.getExtracted().configuration.getName(),
@@ -214,7 +215,6 @@ private Object dependencyNotation(Project project, ElasticsearchDistribution dis
     }
 
     private static Dependency projectDependency(Project project, String projectPath, String projectConfig) {
-
         if (project.findProject(projectPath) == null) {
             throw new GradleException("no project [" + projectPath + "], project names: " + project.getRootProject().getAllprojects());
         }
@@ -226,11 +226,20 @@ private static Dependency projectDependency(Project project, String projectPath,
 
     private static String distributionProjectPath(ElasticsearchDistribution distribution) {
         String projectPath = ":distribution";
-        if (distribution.getType() == Type.INTEG_TEST_ZIP) {
-            projectPath += ":archives:integ-test-zip";
-        } else {
-            projectPath += distribution.getType() == Type.ARCHIVE ? ":archives:" : ":packages:";
-            projectPath += distributionProjectName(distribution);
+        switch (distribution.getType()) {
+            case INTEG_TEST_ZIP:
+                projectPath += ":archives:integ-test-zip";
+                break;
+
+            case DOCKER:
+                projectPath += ":docker:";
+                projectPath += distributionProjectName(distribution);
+                break;
+
+            default:
+                projectPath += distribution.getType() == Type.ARCHIVE ? ":archives:" : ":packages:";
+                projectPath += distributionProjectName(distribution);
+                break;
         }
         return projectPath;
     }
@@ -243,9 +252,12 @@ private static String distributionProjectName(ElasticsearchDistribution distribu
         if (distribution.getBundledJdk() == false) {
             projectName += "no-jdk-";
         }
+
         if (distribution.getType() == Type.ARCHIVE) {
             Platform platform = distribution.getPlatform();
             projectName += platform.toString() + (platform == Platform.WINDOWS ? "-zip" : "-tar");
+        } else if (distribution.getType() == Type.DOCKER) {
+            projectName += "docker-export";
         } else {
             projectName += distribution.getType();
         }

buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java

@@ -46,7 +46,8 @@ public enum Type {
         INTEG_TEST_ZIP,
         ARCHIVE,
         RPM,
-        DEB;
+        DEB,
+        DOCKER;
 
         @Override
         public String toString() {
@@ -171,11 +172,16 @@ public String toString() {
     }
 
     public Extracted getExtracted() {
-        if (getType() == Type.RPM || getType() == Type.DEB) {
-            throw new UnsupportedOperationException("distribution type [" + getType() + "] for " +
-                "elasticsearch distribution [" + name + "] cannot be extracted");
+        switch (getType()) {
+            case DEB:
+            case DOCKER:
+            case RPM:
+                throw new UnsupportedOperationException("distribution type [" + getType() + "] for " +
+                    "elasticsearch distribution [" + name + "] cannot be extracted");
+
+            default:
+                return extracted;
         }
-        return extracted;
     }
 
     @Override
@@ -217,7 +223,7 @@ void finalizeValues() {
             if (platform.isPresent() == false) {
                 platform.set(CURRENT_PLATFORM);
             }
-        } else { // rpm or deb
+        } else { // rpm, deb or docker
             if (platform.isPresent()) {
                 throw new IllegalArgumentException("platform not allowed for elasticsearch distribution ["
                     + name + "] of type [" + getType() + "]");

distribution/docker/build.gradle

@@ -170,6 +170,32 @@ void addBuildDockerImage(final boolean oss) {
   BuildPlugin.requireDocker(buildDockerImageTask)
 }
 
+/**
+ * Exports the generated Docker image to disk, so that it can be easily
+ * reloaded, for example into a VM. Although this involves writing out
+ * the entire image, it's still quicker than rebuilding the main archive
+ * in the VM.
+ */
+void addExportDockerImage(final boolean oss) {
+  def exportTaskName = taskName("export", oss, "DockerImage")
+  def tarFile = "${buildDir}/elasticsearch${oss ? '-oss' : ''}_test.docker.tar"
+
+  task(exportTaskName, type: LoggedExec) {
+    executable 'docker'
+    args "save",
+      "-o",
+      tarFile,
+      "elasticsearch${oss ? '-oss' : ''}:test"
+  }
+
+  artifacts.add(oss ? 'archives' : 'default', file(tarFile)) {
+      type 'tar'
+      name "elasticsearch${oss ? '-oss' : ''}"
+      builtBy exportTaskName
+  }
+}
+
+
 for (final boolean oss : [false, true]) {
   addCopyDockerContextTask(oss)
   addBuildDockerImage(oss)
@@ -184,3 +210,33 @@ assemble.dependsOn "buildDockerImage"
 if (tasks.findByName("composePull")) {
     tasks.composePull.enabled = false 
 }
+
+subprojects { Project subProject ->
+  if (subProject.name.contains('docker-export')) {
+    apply plugin: 'distribution'
+
+    def oss = subProject.name.startsWith('oss')
+
+    def exportTaskName = taskName("export", oss, "DockerImage")
+    def buildTaskName = taskName("build", oss, "DockerImage")
+    def tarFile = "${parent.buildDir}/elasticsearch${oss ? '-oss' : ''}_test.${VersionProperties.elasticsearch}.docker.tar"
+
+    final Task exportDockerImageTask = task(exportTaskName, type: LoggedExec) {
+      executable 'docker'
+      args "save",
+        "-o",
+        tarFile,
+        "elasticsearch${oss ? '-oss' : ''}:test"
+    }
+
+    exportDockerImageTask.dependsOn(parent.tasks.getByName(buildTaskName))
+
+    artifacts.add('default', file(tarFile)) {
+        type 'tar'
+        name "elasticsearch${oss ? '-oss' : ''}"
+        builtBy exportTaskName
+    }
+
+    assemble.dependsOn exportTaskName
+  }
+}

distribution/docker/docker-export/build.gradle

@@ -0,0 +1,2 @@
+// This file is intentionally blank. All configuration of the
+// export is done in the parent project.

distribution/docker/oss-docker-export/build.gradle

@@ -0,0 +1,2 @@
+// This file is intentionally blank. All configuration of the
+// export is done in the parent project.