Use Cloudflare's zlib in Docker images

distribution/docker/build.gradle

@@ -16,6 +16,8 @@ apply plugin: 'elasticsearch.test.fixtures'
 apply plugin: 'elasticsearch.internal-distribution-download'
 apply plugin: 'elasticsearch.rest-resources'
 
+ext.cloudflareZlibVersion = '1.2.8'
+
 String buildId = providers.systemProperty('build.id').forUseAtConfigurationTime().getOrNull()
 boolean useLocalArtifacts = buildId != null && buildId.isBlank() == false
 
@@ -34,6 +36,15 @@ repositories {
     content { includeGroup 'krallin' }
   }
 
+  ivy {
+    url 'https://github.com/'
+    patternLayout {
+      artifact '/[organisation]/[module]/archive/refs/tags/v[revision].[ext]'
+    }
+    metadataSources { artifact() }
+    content { includeGroup 'cloudflare' }
+  }
+
   // Cloud builds bundle some beats
   ivy {
     if (useLocalArtifacts) {
@@ -63,6 +74,7 @@ configurations {
   nonRepositoryPlugins
   filebeat
   metricbeat
+  cloudflareZlib
 }
 
 String beatsArch = Architecture.current() == Architecture.AARCH64 ? 'arm64' : 'x86_64'
@@ -77,6 +89,7 @@ dependencies {
   nonRepositoryPlugins project(path: ':plugins', configuration: 'nonRepositoryPlugins')
   filebeat "beats:filebeat:${VersionProperties.elasticsearch}:${beatsArch}@tar.gz"
   metricbeat "beats:metricbeat:${VersionProperties.elasticsearch}:${beatsArch}@tar.gz"
+  cloudflareZlib "cloudflare:zlib:${cloudflareZlibVersion}@tar.gz"
 }
 
 ext.expansions = { Architecture architecture, DockerBase base ->
@@ -100,7 +113,8 @@ ext.expansions = { Architecture architecture, DockerBase base ->
     'docker_base'        : base.name().toLowerCase(),
     'version'            : VersionProperties.elasticsearch,
     'major_minor_version': "${major}.${minor}",
-    'retry'              : ShellRetry
+    'retry'              : ShellRetry,
+    'zlib_version'       : cloudflareZlibVersion
   ]
 }
 
@@ -259,6 +273,7 @@ void addBuildDockerContextTask(Architecture architecture, DockerBase base) {
         boolean includeBeats = VersionProperties.isElasticsearchSnapshot() == true || buildId != null
 
         from configurations.repositoryPlugins
+
         if (includeBeats) {
           from configurations.filebeat
           from configurations.metricbeat
@@ -289,7 +304,10 @@ void addTransformDockerContextTask(Architecture architecture, DockerBase base) {
     from(tarTree("${project.buildDir}/distributions/${archiveName}.tar.gz")) {
       eachFile { FileCopyDetails details ->
         if (details.name.equals("Dockerfile")) {
-          filter { it.replaceAll('^RUN curl.*artifacts-no-kpi.*$', "COPY ${distributionName} /tmp/elasticsearch.tar.gz") }
+          filter { String filename ->
+            String result = filename.replaceAll('^RUN curl.*artifacts-no-kpi.*$', "COPY ${distributionName} /tmp/elasticsearch.tar.gz")
+            return result.replaceAll('^RUN curl.*cloudflare/zlib.*$', "COPY zlib-${cloudflareZlibVersion}.tar.gz /tmp")
+          }
         }
       }
     }
@@ -302,6 +320,8 @@ void addTransformDockerContextTask(Architecture architecture, DockerBase base) {
       from configurations.dockerSource
     }
 
+    from configurations.cloudflareZlib
+
     if (base == DockerBase.IRON_BANK) {
       from (configurations.tini) {
         rename { _ -> 'tini' }

distribution/docker/src/docker/Dockerfile

@@ -20,28 +20,51 @@
 */ %>
 
 <% if (docker_base == 'iron_bank') { %>
-################################################################################
-# Build stage 0 `builder`:
-# Extract Elasticsearch artifact
-################################################################################
-
 ARG BASE_REGISTRY=registry1.dso.mil
 ARG BASE_IMAGE=ironbank/redhat/ubi/ubi8
 ARG BASE_TAG=8.4
+<% } %>
+
+################################################################################
+# Build stage 0 `zlib`:
+# Compile zlib for the current architecture
+################################################################################
+
+FROM ${base_image} AS zlib
+
+<% if (docker_base == 'default' || docker_base == 'cloud') { %>
+RUN <%= retry.loop(package_manager, "${package_manager} update && DEBIAN_FRONTEND=noninteractive ${package_manager} install -y curl gcc make") %>
+<% } else { %>
+RUN <%= retry.loop(package_manager, "${package_manager} install -y curl gcc make tar gzip") %>
+<% } %>
+
+<%
+// Fetch the zlib source.  Keep this command on one line - it is replaced
+// with a `COPY` during local builds.
+%>
+WORKDIR /tmp
+RUN curl --retry 10 -S -L -o zlib-${zlib_version}.tar.gz https://github.com/cloudflare/zlib/archive/refs/tags/v${zlib_version}.tar.gz
+RUN echo "7e393976368975446b263ae4143fb404bc33bf3b436e72007700b5b88e5be332cd461cdec42d31a4b6dffdca2368550f01b9fa1165d81c0aa818bbf2b1ac191e zlib-${zlib_version}.tar.gz" \\
+      | sha512sum --check -
+RUN tar xf zlib-${zlib_version}.tar.gz
+
+WORKDIR /tmp/zlib-${zlib_version}
+RUN ./configure --prefix=/usr/local/cloudflare-zlib && make && make install
+
+################################################################################
+# Build stage 1 `builder`:
+# Extract Elasticsearch artifact
+################################################################################
 
 FROM ${base_image} AS builder
 
+<% if (docker_base == 'iron_bank') { %>
 # `tini` is a tiny but valid init for containers. This is used to cleanly
 # control how ES and any child processes are shut down.
 COPY tini /bin/tini
 RUN chmod 0555 /bin/tini
 
 <% } else { %>
-################################################################################
-# Build stage 0 `builder`:
-# Extract Elasticsearch artifact
-################################################################################
-FROM ${base_image} AS builder
 
 # Install required packages to extract the Elasticsearch distribution
 <% if (docker_base == 'default' || docker_base == 'cloud') { %>
@@ -138,9 +161,10 @@ RUN chmod -R 0555 /opt/plugins
 <% } %>
 
 ################################################################################
-# Build stage 1 (the actual Elasticsearch image):
+# Build stage 2 (the actual Elasticsearch image):
 #
-# Copy elasticsearch from stage 0
+# Copy zlib from stage 2
+# Copy elasticsearch from stage 1
 # Add entrypoint
 ################################################################################
 
@@ -198,6 +222,7 @@ WORKDIR /usr/share/elasticsearch
 COPY --from=builder --chown=0:0 /usr/share/elasticsearch /usr/share/elasticsearch
 
 COPY --from=builder --chown=0:0 /bin/tini /bin/tini
+COPY --from=zlib --chown=0:0 /usr/local/cloudflare-zlib /usr/local/cloudflare-zlib
 
 <% if (docker_base == 'default' || docker_base == 'cloud') { %>
 COPY --from=builder --chown=0:0 /etc/ssl/certs/java/cacerts /etc/ssl/certs/java/cacerts
@@ -208,6 +233,7 @@ COPY --from=builder --chown=0:0 /opt /opt
 <% } %>
 
 ENV PATH /usr/share/elasticsearch/bin:\$PATH
+ENV ES_ZLIB_PATH /usr/local/cloudflare-zlib/lib
 
 COPY ${bin_dir}/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
 

distribution/src/bin/elasticsearch

@@ -52,6 +52,14 @@ if [ -z "$LIBFFI_TMPDIR" ]; then
   export LIBFFI_TMPDIR
 fi
 
+if [ -n "$ES_ZLIB_PATH" ]; then
+  if [ ! -d "$ES_ZLIB_PATH" ]; then
+    echo "zlib path specified in ES_ZLIB_PATH does not exist or is not a directory: $ES_ZLIB_PATH" >&2
+    exit 1
+  fi
+  export LD_LIBRARY_PATH="$ES_ZLIB_PATH:$LD_LIBRARY_PATH"
+fi
+
 # get keystore password before setting java options to avoid
 # conflicting GC configurations for the keystore tools
 unset KEYSTORE_PASSWORD

docs/changelog/81245.yaml

@@ -0,0 +1,6 @@
+pr: 81245
+summary: Use Cloudflare's zlib in Docker images
+area: Packaging
+type: enhancement
+issues:
+ - 81208

qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java

@@ -415,6 +415,17 @@ public void test050BasicApiTests() throws Exception {
         runElasticsearchTestsAsElastic(PASSWORD);
     }
 
+    /**
+     * Check that the JDK uses the Cloudflare zlib, instead of the default one.
+     */
+    public void test060JavaUsesCloudflareZlib() {
+        waitForElasticsearch(installation, "elastic", PASSWORD);
+
+        final String output = sh.run("bash -c 'pmap -p $(pidof java)'").stdout;
+
+        assertThat("Expected java to be using cloudflare-zlib", output, containsString("cloudflare-zlib"));
+    }
+
     /**
      * Check that the default config can be overridden using a bind mount, and that env vars are respected
      */