Skip to content

Commit

Permalink
Use Cloudflare's zlib in Docker images
Browse files Browse the repository at this point in the history 
Closes elastic#81208. Elasticsearch uses zlib for two purposes:

   * Compression of stored fields with `index.codec: best_compression`,
     which we use for observability and security data.
   * Request / response compression.

Historically, zlib was packaged within the JDK, so that users wouldn't
have to have zlib installed for basic usage of Java. However, the
original zlib optimizes for portability and misses a number of important
optimizations such as leveraging vectorization support for x86 and ARM
architectures. Several forks have been created in order to address this.

Since version 9, the JDK uses the system's zlib when available and falls
back to the zlib that is packaged within the JDK if a system zlib cannot
be found.

This commit changes the Docker image to install the Cloudflare fork of
zlib, and run Java using the fork instead of the original zlib, so that
users of the Docker image can get better performance.

Other ES distribution types are out-of-scope, since configuring the JVM
to use an alternative zlib requires an environment config as well as
installed another zlib, and Docker is the only distribution type where
we can control both.
  • Loading branch information
@pugnascotia
pugnascotia committed Dec 2, 2021
1 parent e999ad0 commit 0e35975
Show file tree
Hide file tree
Showing 3 changed files with 63 additions and 14 deletions.
24 changes: 22 additions & 2 deletions distribution/docker/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ apply plugin: 'elasticsearch.test.fixtures'
apply plugin: 'elasticsearch.internal-distribution-download'
apply plugin: 'elasticsearch.rest-resources'

ext.cloudflareZlibVersion = '1.2.8'

String buildId = providers.systemProperty('build.id').forUseAtConfigurationTime().getOrNull()
boolean useLocalArtifacts = buildId != null && buildId.isBlank() == false

Expand All @@ -34,6 +36,15 @@ repositories {
content { includeGroup 'krallin' }
}

ivy {
url 'https://github.com/'
patternLayout {
artifact '/[organisation]/[module]/archive/refs/tags/v[revision].[ext]'
}
metadataSources { artifact() }
content { includeGroup 'cloudflare' }
}

// Cloud builds bundle some beats
ivy {
if (useLocalArtifacts) {
Expand Down Expand Up @@ -63,6 +74,7 @@ configurations {
nonRepositoryPlugins
filebeat
metricbeat
cloudflareZlib
}

String beatsArch = Architecture.current() == Architecture.AARCH64 ? 'arm64' : 'x86_64'
Expand All @@ -77,6 +89,7 @@ dependencies {
nonRepositoryPlugins project(path: ':plugins', configuration: 'nonRepositoryPlugins')
filebeat "beats:filebeat:${VersionProperties.elasticsearch}:${beatsArch}@tar.gz"
metricbeat "beats:metricbeat:${VersionProperties.elasticsearch}:${beatsArch}@tar.gz"
cloudflareZlib "cloudflare:zlib:${cloudflareZlibVersion}@tar.gz"
}

ext.expansions = { Architecture architecture, DockerBase base ->
Expand All @@ -100,7 +113,8 @@ ext.expansions = { Architecture architecture, DockerBase base ->
'docker_base' : base.name().toLowerCase(),
'version' : VersionProperties.elasticsearch,
'major_minor_version': "${major}.${minor}",
'retry' : ShellRetry
'retry' : ShellRetry,
'zlib_version' : cloudflareZlibVersion
]
}

Expand Down Expand Up @@ -259,6 +273,7 @@ void addBuildDockerContextTask(Architecture architecture, DockerBase base) {
boolean includeBeats = VersionProperties.isElasticsearchSnapshot() == true || buildId != null

from configurations.repositoryPlugins

if (includeBeats) {
from configurations.filebeat
from configurations.metricbeat
Expand Down Expand Up @@ -289,7 +304,10 @@ void addTransformDockerContextTask(Architecture architecture, DockerBase base) {
from(tarTree("${project.buildDir}/distributions/${archiveName}.tar.gz")) {
eachFile { FileCopyDetails details ->
if (details.name.equals("Dockerfile")) {
filter { it.replaceAll('^RUN curl.*artifacts-no-kpi.*$', "COPY ${distributionName} /tmp/elasticsearch.tar.gz") }
filter { String filename ->
String result = filename.replaceAll('^RUN curl.*artifacts-no-kpi.*$', "COPY ${distributionName} /tmp/elasticsearch.tar.gz")
return result.replaceAll('^RUN curl.*cloudflare/zlib.*$', "COPY zlib-${cloudflareZlibVersion}.tar.gz /tmp")
}
}
}
}
Expand All @@ -302,6 +320,8 @@ void addTransformDockerContextTask(Architecture architecture, DockerBase base) {
from configurations.dockerSource
}

from configurations.cloudflareZlib

if (base == DockerBase.IRON_BANK) {
from (configurations.tini) {
rename { _ -> 'tini' }
Expand Down
49 changes: 37 additions & 12 deletions distribution/docker/src/docker/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,28 +20,51 @@
*/ %>
<% if (docker_base == 'iron_bank') { %>
################################################################################
# Build stage 0 `builder`:
# Extract Elasticsearch artifact
################################################################################
ARG BASE_REGISTRY=registry1.dso.mil
ARG BASE_IMAGE=ironbank/redhat/ubi/ubi8
ARG BASE_TAG=8.4
<% } %>
################################################################################
# Build stage 0 `zlib`:
# Compile zlib for the current architecture
################################################################################
FROM ${base_image} AS zlib
<% if (docker_base == 'default' || docker_base == 'cloud') { %>
RUN <%= retry.loop(package_manager, "${package_manager} update && DEBIAN_FRONTEND=noninteractive ${package_manager} install -y curl gcc make") %>
<% } else { %>
RUN <%= retry.loop(package_manager, "${package_manager} install -y curl gcc make tar gzip") %>
<% } %>
<%
// Fetch the zlib source. Keep this command on one line - it is replaced
// with a `COPY` during local builds.
%>
WORKDIR /tmp
RUN curl --retry 10 -S -L -o zlib-${zlib_version}.tar.gz https://github.com/cloudflare/zlib/archive/refs/tags/v${zlib_version}.tar.gz
RUN echo "7e393976368975446b263ae4143fb404bc33bf3b436e72007700b5b88e5be332cd461cdec42d31a4b6dffdca2368550f01b9fa1165d81c0aa818bbf2b1ac191e zlib-${zlib_version}.tar.gz" \\
| sha512sum --check -
RUN tar xf zlib-${zlib_version}.tar.gz
WORKDIR /tmp/zlib-${zlib_version}
RUN ./configure --prefix=/usr/local/cloudflare-zlib && make && make install
################################################################################
# Build stage 1 `builder`:
# Extract Elasticsearch artifact
################################################################################
FROM ${base_image} AS builder
<% if (docker_base == 'iron_bank') { %>
# `tini` is a tiny but valid init for containers. This is used to cleanly
# control how ES and any child processes are shut down.
COPY tini /bin/tini
RUN chmod 0555 /bin/tini
<% } else { %>
################################################################################
# Build stage 0 `builder`:
# Extract Elasticsearch artifact
################################################################################
FROM ${base_image} AS builder
# Install required packages to extract the Elasticsearch distribution
<% if (docker_base == 'default' || docker_base == 'cloud') { %>
Expand Down Expand Up @@ -138,9 +161,10 @@ RUN chmod -R 0555 /opt/plugins
<% } %>
################################################################################
# Build stage 1 (the actual Elasticsearch image):
# Build stage 2 (the actual Elasticsearch image):
#
# Copy elasticsearch from stage 0
# Copy zlib from stage 2
# Copy elasticsearch from stage 1
# Add entrypoint
################################################################################
Expand Down Expand Up @@ -198,6 +222,7 @@ WORKDIR /usr/share/elasticsearch
COPY --from=builder --chown=0:0 /usr/share/elasticsearch /usr/share/elasticsearch

COPY --from=builder --chown=0:0 /bin/tini /bin/tini
COPY --from=zlib --chown=0:0 /usr/local/cloudflare-zlib /usr/local/cloudflare-zlib

<% if (docker_base == 'default' || docker_base == 'cloud') { %>
COPY --from=builder --chown=0:0 /etc/ssl/certs/java/cacerts /etc/ssl/certs/java/cacerts
Expand Down
4 changes: 4 additions & 0 deletions distribution/docker/src/docker/bin/docker-entrypoint.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,10 @@ if [[ -n "$ES_LOG_STYLE" ]]; then
esac
fi

if [[ -d /usr/local/cloudflare-zlib/lib ]]; then
export LD_LIBRARY_PATH=/usr/local/cloudflare-zlib/lib
fi

# Signal forwarding and child reaping is handled by `tini`, which is the
# actual entrypoint of the container
exec /usr/share/elasticsearch/bin/elasticsearch <<<"$KEYSTORE_PASSWORD"

0 comments on commit 0e35975

Please sign in to comment.