diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index 86c2b88c67aad..0bba047581af1 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -16,6 +16,8 @@ apply plugin: 'elasticsearch.test.fixtures' apply plugin: 'elasticsearch.internal-distribution-download' apply plugin: 'elasticsearch.rest-resources' +ext.cloudflareZlibVersion = '1.2.8' + String buildId = providers.systemProperty('build.id').forUseAtConfigurationTime().getOrNull() boolean useLocalArtifacts = buildId != null && buildId.isBlank() == false @@ -34,6 +36,15 @@ repositories { content { includeGroup 'krallin' } } + ivy { + url 'https://github.com/' + patternLayout { + artifact '/[organisation]/[module]/archive/refs/tags/v[revision].[ext]' + } + metadataSources { artifact() } + content { includeGroup 'cloudflare' } + } + // Cloud builds bundle some beats ivy { if (useLocalArtifacts) { @@ -63,6 +74,7 @@ configurations { nonRepositoryPlugins filebeat metricbeat + cloudflareZlib } String beatsArch = Architecture.current() == Architecture.AARCH64 ? 'arm64' : 'x86_64' @@ -77,6 +89,7 @@ dependencies { nonRepositoryPlugins project(path: ':plugins', configuration: 'nonRepositoryPlugins') filebeat "beats:filebeat:${VersionProperties.elasticsearch}:${beatsArch}@tar.gz" metricbeat "beats:metricbeat:${VersionProperties.elasticsearch}:${beatsArch}@tar.gz" + cloudflareZlib "cloudflare:zlib:${cloudflareZlibVersion}@tar.gz" } ext.expansions = { Architecture architecture, DockerBase base -> @@ -100,7 +113,8 @@ ext.expansions = { Architecture architecture, DockerBase base -> 'docker_base' : base.name().toLowerCase(), 'version' : VersionProperties.elasticsearch, 'major_minor_version': "${major}.${minor}", - 'retry' : ShellRetry + 'retry' : ShellRetry, + 'zlib_version' : cloudflareZlibVersion ] } @@ -259,6 +273,7 @@ void addBuildDockerContextTask(Architecture architecture, DockerBase base) { boolean includeBeats = VersionProperties.isElasticsearchSnapshot() == true || buildId != null from configurations.repositoryPlugins + if (includeBeats) { from configurations.filebeat from configurations.metricbeat @@ -289,7 +304,10 @@ void addTransformDockerContextTask(Architecture architecture, DockerBase base) { from(tarTree("${project.buildDir}/distributions/${archiveName}.tar.gz")) { eachFile { FileCopyDetails details -> if (details.name.equals("Dockerfile")) { - filter { it.replaceAll('^RUN curl.*artifacts-no-kpi.*$', "COPY ${distributionName} /tmp/elasticsearch.tar.gz") } + filter { String filename -> + String result = filename.replaceAll('^RUN curl.*artifacts-no-kpi.*$', "COPY ${distributionName} /tmp/elasticsearch.tar.gz") + return result.replaceAll('^RUN curl.*cloudflare/zlib.*$', "COPY zlib-${cloudflareZlibVersion}.tar.gz /tmp") + } } } } @@ -302,6 +320,8 @@ void addTransformDockerContextTask(Architecture architecture, DockerBase base) { from configurations.dockerSource } + from configurations.cloudflareZlib + if (base == DockerBase.IRON_BANK) { from (configurations.tini) { rename { _ -> 'tini' } diff --git a/distribution/docker/src/docker/Dockerfile b/distribution/docker/src/docker/Dockerfile index 98716557d3d3b..30dba569b54c7 100644 --- a/distribution/docker/src/docker/Dockerfile +++ b/distribution/docker/src/docker/Dockerfile @@ -20,28 +20,51 @@ */ %> <% if (docker_base == 'iron_bank') { %> -################################################################################ -# Build stage 0 `builder`: -# Extract Elasticsearch artifact -################################################################################ - ARG BASE_REGISTRY=registry1.dso.mil ARG BASE_IMAGE=ironbank/redhat/ubi/ubi8 ARG BASE_TAG=8.4 +<% } %> + +################################################################################ +# Build stage 0 `zlib`: +# Compile zlib for the current architecture +################################################################################ + +FROM ${base_image} AS zlib + +<% if (docker_base == 'default' || docker_base == 'cloud') { %> +RUN <%= retry.loop(package_manager, "${package_manager} update && DEBIAN_FRONTEND=noninteractive ${package_manager} install -y curl gcc make") %> +<% } else { %> +RUN <%= retry.loop(package_manager, "${package_manager} install -y curl gcc make tar gzip") %> +<% } %> + +<% +// Fetch the zlib source. Keep this command on one line - it is replaced +// with a `COPY` during local builds. +%> +WORKDIR /tmp +RUN curl --retry 10 -S -L -o zlib-${zlib_version}.tar.gz https://github.com/cloudflare/zlib/archive/refs/tags/v${zlib_version}.tar.gz +RUN echo "7e393976368975446b263ae4143fb404bc33bf3b436e72007700b5b88e5be332cd461cdec42d31a4b6dffdca2368550f01b9fa1165d81c0aa818bbf2b1ac191e zlib-${zlib_version}.tar.gz" \\ + | sha512sum --check - +RUN tar xf zlib-${zlib_version}.tar.gz + +WORKDIR /tmp/zlib-${zlib_version} +RUN ./configure --prefix=/usr/local/cloudflare-zlib && make && make install + +################################################################################ +# Build stage 1 `builder`: +# Extract Elasticsearch artifact +################################################################################ FROM ${base_image} AS builder +<% if (docker_base == 'iron_bank') { %> # `tini` is a tiny but valid init for containers. This is used to cleanly # control how ES and any child processes are shut down. COPY tini /bin/tini RUN chmod 0555 /bin/tini <% } else { %> -################################################################################ -# Build stage 0 `builder`: -# Extract Elasticsearch artifact -################################################################################ -FROM ${base_image} AS builder # Install required packages to extract the Elasticsearch distribution <% if (docker_base == 'default' || docker_base == 'cloud') { %> @@ -147,9 +170,10 @@ RUN chmod -R 0555 /opt/plugins <% } %> ################################################################################ -# Build stage 1 (the actual Elasticsearch image): +# Build stage 2 (the actual Elasticsearch image): # -# Copy elasticsearch from stage 0 +# Copy zlib from stage 2 +# Copy elasticsearch from stage 1 # Add entrypoint ################################################################################ @@ -207,6 +231,7 @@ WORKDIR /usr/share/elasticsearch COPY --from=builder --chown=0:0 /usr/share/elasticsearch /usr/share/elasticsearch COPY --from=builder --chown=0:0 /bin/tini /bin/tini +COPY --from=zlib --chown=0:0 /usr/local/cloudflare-zlib /usr/local/cloudflare-zlib <% if (docker_base == 'default' || docker_base == 'cloud') { %> COPY --from=builder --chown=0:0 /etc/ssl/certs/java/cacerts /etc/ssl/certs/java/cacerts @@ -217,6 +242,7 @@ COPY --from=builder --chown=0:0 /opt /opt <% } %> ENV PATH /usr/share/elasticsearch/bin:\$PATH +ENV ES_ZLIB_PATH /usr/local/cloudflare-zlib/lib COPY ${bin_dir}/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh diff --git a/distribution/src/bin/elasticsearch b/distribution/src/bin/elasticsearch index a4bcde1a92af6..1ac3c56cf8795 100755 --- a/distribution/src/bin/elasticsearch +++ b/distribution/src/bin/elasticsearch @@ -52,6 +52,14 @@ if [ -z "$LIBFFI_TMPDIR" ]; then export LIBFFI_TMPDIR fi +if [ -n "$ES_ZLIB_PATH" ]; then + if [ ! -d "$ES_ZLIB_PATH" ]; then + echo "zlib path specified in ES_ZLIB_PATH does not exist or is not a directory: $ES_ZLIB_PATH" >&2 + exit 1 + fi + export LD_LIBRARY_PATH="$ES_ZLIB_PATH:$LD_LIBRARY_PATH" +fi + # get keystore password before setting java options to avoid # conflicting GC configurations for the keystore tools unset KEYSTORE_PASSWORD diff --git a/docs/changelog/81245.yaml b/docs/changelog/81245.yaml new file mode 100644 index 0000000000000..068bfa1c1eb7a --- /dev/null +++ b/docs/changelog/81245.yaml @@ -0,0 +1,6 @@ +pr: 81245 +summary: Use Cloudflare's zlib in Docker images +area: Packaging +type: enhancement +issues: + - 81208 diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java index 33c9ec77e7821..46fb67c70fc62 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java @@ -415,6 +415,17 @@ public void test050BasicApiTests() throws Exception { runElasticsearchTestsAsElastic(PASSWORD); } + /** + * Check that the JDK uses the Cloudflare zlib, instead of the default one. + */ + public void test060JavaUsesCloudflareZlib() { + waitForElasticsearch(installation, "elastic", PASSWORD); + + final String output = sh.run("bash -c 'pmap -p $(pidof java)'").stdout; + + assertThat("Expected java to be using cloudflare-zlib", output, containsString("cloudflare-zlib")); + } + /** * Check that the default config can be overridden using a bind mount, and that env vars are respected */