Make it possible to use Stack logging in Docker

distribution/docker/build.gradle

@@ -30,6 +30,24 @@ dependencies {
   ossDockerSource project(path: ":distribution:archives:oss-linux-tar", configuration:"default")
 }
 
+/**
+ * Used in the Dockerfile template to wrap commands in a retry loop. It can't be
+ * a closure because Gradle can't effectively cache those.
+ */
+class Retry {
+  static String loop(name, command, indentSize = 4, exitKeyword = 'exit') {
+    String indent = ' ' * indentSize
+    String commandWithRetry = """for iter in {1..10}; do
+${indent}  ${command} &&
+${indent}  exit_code=0 && break ||
+${indent}    exit_code=\$? && echo "${name} error: retry \$iter in 10s" && sleep 10;
+${indent}done;
+${indent}${exitKeyword} \$exit_code"""
+
+    return commandWithRetry.replaceAll(" *\n", " \\\\\n")
+  }
+}
+
 ext.expansions = { Architecture architecture, boolean oss, DockerBase base, boolean local ->
   String classifier
   if (local) {
@@ -75,18 +93,6 @@ RUN curl --retry 8 -S -L \\
 
   def (major,minor) = VersionProperties.elasticsearch.split("\\.")
 
-  def retry_loop = { name, command, indentSize = 4, exitKeyword = 'exit'  ->
-    String indent = ' ' * indentSize
-    String commandWithRetry = """for iter in {1..10}; do
-${indent}  ${command} &&
-${indent}  exit_code=0 && break ||
-${indent}    exit_code=\$? && echo "${name} error: retry \$iter in 10s" && sleep 10;
-${indent}done;
-${indent}${exitKeyword} \$exit_code"""
-
-    return commandWithRetry.replaceAll(" *\n", " \\\\\n")
-  }
-
   return [
     'base_image'          : base.getImage(),
     'bin_dir'             : base == DockerBase.IRON_BANK ? 'scripts' : 'bin',
@@ -100,7 +106,7 @@ ${indent}${exitKeyword} \$exit_code"""
     'docker_base'         : base.name().toLowerCase(),
     'version'             : VersionProperties.elasticsearch,
     'major_minor_version' : "${major}.${minor}",
-    'retry_loop'          : retry_loop
+    'retry'               : Retry,
   ]
 }
 

distribution/docker/src/docker/Dockerfile

@@ -27,7 +27,7 @@
 FROM ${base_image} AS builder
 
 # Install required packages to extract the Elasticsearch distribution
-RUN <%= retry_loop(package_manager, "${package_manager} install -y findutils tar gzip") %>
+RUN <%= retry.loop(package_manager, "${package_manager} install -y findutils tar gzip") %>
 
 # `tini` is a tiny but valid init for containers. This is used to cleanly
 # control how ES and any child processes are shut down.
@@ -74,7 +74,7 @@ ENV TARBALL_URL https://curl.haxx.se/download/curl-\${VERSION}.tar.xz
 ENV TARBALL_PATH curl-\${VERSION}.tar.xz
 
 # Install dependencies
-RUN <%= retry_loop('apk', 'apk add gnupg gcc make musl-dev openssl-dev openssl-libs-static file') %>
+RUN <%= retry.loop('apk', 'apk add gnupg gcc make musl-dev openssl-dev openssl-libs-static file') %>
 
 RUN mkdir /work
 WORKDIR /work
@@ -83,7 +83,7 @@ WORKDIR /work
 RUN function retry_wget() { \\
       local URL="\$1" ; \\
       local DEST="\$2" ; \\
-      <%= retry_loop('wget', 'wget "\$URL\" -O "\$DEST"', 6, 'return') %> ; \\
+      <%= retry.loop('wget', 'wget "\$URL\" -O "\$DEST"', 6, 'return') %> ; \\
     } ; \\
     retry_wget "https://daniel.haxx.se/mykey.asc" "curl-gpg.pub" && \\
     retry_wget "\${TARBALL_URL}.asc" "\${TARBALL_PATH}.asc" && \\
@@ -223,21 +223,25 @@ ${source_elasticsearch}
 RUN tar -zxf /opt/elasticsearch.tar.gz --strip-components=1
 
 # The distribution includes a `config` directory, no need to create it
-COPY ${config_dir}/elasticsearch.yml ${config_dir}/log4j2.properties config/
+COPY ${config_dir}/elasticsearch.yml ${config_dir}/log4j2.docker.properties config/
 
 # 1. Configure the distribution for Docker
 # 2. Ensure directories are created. Most already are, but make sure
 # 3. Apply correct permissions
-# 4. Apply more correct permissions
-# 5. The JDK's directories' permissions don't allow `java` to be executed under a different
+# 4. Move the stack logging config aside
+# 5. Rename the docker logging config to be used by default
+# 6. Apply more correct permissions
+# 7. The JDK's directories' permissions don't allow `java` to be executed under a different
 #    group to the default. Fix this.
-# 6. Ensure that there are no files with setuid or setgid, in order to mitigate "stackclash" attacks.
-# 7. Ensure all files are world-readable by default. It should be possible to
+# 8. Ensure that there are no files with setuid or setgid, in order to mitigate "stackclash" attacks.
+# 9. Ensure all files are world-readable by default. It should be possible to
 #    examine the contents of the image under any UID:GID
-RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' /usr/share/elasticsearch/bin/elasticsearch-env && \\
+RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' bin/elasticsearch-env && \\
     mkdir -p config/jvm.options.d data logs plugins && \\
     chmod 0775 config config/jvm.options.d data logs plugins && \\
-    chmod 0660 config/elasticsearch.yml config/log4j2.properties && \\
+    mv config/log4j2.properties config/log4j2.stack.properties && \\
+    mv config/log4j2.docker.properties config/log4j2.properties && \\
+    chmod 0660 config/elasticsearch.yml config/log4j2*.properties && \\
     find ./jdk -type d -exec chmod 0755 {} + && \\
     find . -xdev -perm -4000 -exec chmod ug-s {} + && \\
     find . -type f -exec chmod o+r {} +
@@ -255,7 +259,7 @@ FROM ${base_image}
 
 <% if (docker_base == "ubi") { %>
 
-RUN <%= retry_loop(
+RUN <%= retry.loop(
     package_manager,
       "${package_manager} update --setopt=tsflags=nodocs -y && \n" +
       "      ${package_manager} install --setopt=tsflags=nodocs -y \n" +

distribution/docker/src/docker/bin/docker-entrypoint.sh

@@ -57,6 +57,21 @@ if [[ -f bin/elasticsearch-users ]]; then
   fi
 fi
 
+if [[ -n "$ES_LOG_STYLE" ]]; then
+  case "$ES_LOG_STYLE" in
+    docker)
+      # This is the default. Nothing to do.
+      ;;
+    stack)
+      # Overwrite the default config with the stack config
+      mv /usr/share/elasticsearch/config/log4j2.stack.properties /usr/share/elasticsearch/config/log4j2.properties
+      ;;
+    *)
+      echo "ERROR: ES_LOG_STYLE set to [$ES_LOG_STYLE]. Expected [docker] or [stack]" >&2
+      exit 1 ;;
+  esac
+fi
+
 # Signal forwarding and child reaping is handled by `tini`, which is the
 # actual entrypoint of the container
 exec /usr/share/elasticsearch/bin/elasticsearch <<<"$KEYSTORE_PASSWORD"

docs/reference/setup/install/docker.asciidoc

@@ -122,7 +122,9 @@ curl -X GET "localhost:9200/_cat/nodes?v&pretty"
 // NOTCONSOLE
 
 Log messages go to the console and are handled by the configured Docker logging driver.
-By default you can access logs with `docker logs`.
+By default you can access logs with `docker logs`. If you would prefer the {es}
+container to write logs to disk, set the `ES_LOG_STYLE` environment variable to `stack`.
+This causes {es} to use the same logging configuration as other {es} distribution formats.
 
 To stop the cluster, run `docker-compose down`.
 The data in the Docker volumes is preserved and loaded

qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java

@@ -36,8 +36,10 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import static java.nio.file.attribute.PosixFilePermissions.fromString;
 import static org.elasticsearch.packaging.util.Docker.chownWithPrivilegeEscalation;
@@ -68,8 +70,10 @@
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.matchesPattern;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.Matchers.startsWith;
 import static org.junit.Assume.assumeFalse;
 import static org.junit.Assume.assumeTrue;
 
@@ -639,6 +643,47 @@ public void test120DockerLogsIncludeElasticsearchLogs() throws Exception {
         assertThat("Container logs don't contain INFO level messages", containerLogs.stdout, containsString("INFO"));
     }
 
+    /**
+     * Check that it is possible to use the Stack logging config
+     */
+    public void test121CanUseStackLoggingConfig() throws Exception {
+        runContainer(distribution(), builder().envVars(Map.of("ES_LOG_STYLE", "stack")));
+
+        waitForElasticsearch(installation);
+
+        final Result containerLogs = getContainerLogs();
+        final List<String> stdout = containerLogs.stdout.lines().collect(Collectors.toList());
+
+        assertThat(
+            "Container logs should be formatted using the stack config",
+            stdout.get(stdout.size() - 1),
+            matchesPattern("^\\[\\d\\d\\d\\d-.*")
+        );
+    }
+
+    /**
+     * Check that the Docker logging config can be explicitly selected.
+     */
+    public void test122CanUseDockerLoggingConfig() throws Exception {
+        runContainer(distribution(), builder().envVars(Map.of("ES_LOG_STYLE", "docker")));
+
+        waitForElasticsearch(installation);
+
+        final Result containerLogs = getContainerLogs();
+        final List<String> stdout = containerLogs.stdout.lines().collect(Collectors.toList());
+
+        assertThat("Container logs should be formatted using the docker config", stdout.get(stdout.size() - 1), startsWith("{\""));
+    }
+
+    /**
+     * Check that an unknown logging config is rejected
+     */
+    public void test123CannotUseUnknownLoggingConfig() {
+        final Result result = runContainerExpectingFailure(distribution(), builder().envVars(Map.of("ES_LOG_STYLE", "unknown")));
+
+        assertThat(result.stderr, containsString("ERROR: ES_LOG_STYLE set to [unknown]. Expected [docker] or [stack]"));
+    }
+
     /**
      * Check that the Java process running inside the container has the expected UID, GID and username.
      */

qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java

@@ -200,7 +200,7 @@ private static void waitForElasticsearchToExit() {
 
         if (isElasticsearchRunning) {
             final Shell.Result dockerLogs = getContainerLogs();
-            fail("Elasticsearch container did exit.\n\nStdout:\n" + dockerLogs.stdout + "\n\nStderr:\n" + dockerLogs.stderr);
+            fail("Elasticsearch container didn't exit.\n\nStdout:\n" + dockerLogs.stdout + "\n\nStderr:\n" + dockerLogs.stderr);
         }
     }