Make it possible to use Stack logging in Docker

distribution/docker/build.gradle

@@ -21,13 +21,33 @@ configurations {
   dockerSource
   aarch64OssDockerSource
   ossDockerSource
+  transformLog4jJar
 }
 
 dependencies {
   aarch64DockerSource project(path: ":distribution:archives:linux-aarch64-tar", configuration:"default")
   dockerSource project(path: ":distribution:archives:linux-tar", configuration:"default")
   aarch64OssDockerSource project(path: ":distribution:archives:oss-linux-aarch64-tar", configuration:"default")
   ossDockerSource project(path: ":distribution:archives:oss-linux-tar", configuration:"default")
+  transformLog4jJar project(path: ":distribution:docker:transform-log4j-config", configuration: "default")
+}
+
+/**
+ * Used in the Dockerfile template to wrap commands in a retry loop. It can't be
+ * a closure because Gradle can't effectively cache those.
+ */
+class Retry {
+  static String loop(name, command, indentSize = 4, exitKeyword = 'exit') {
+    String indent = ' ' * indentSize
+    String commandWithRetry = """for iter in {1..10}; do
+${indent}  ${command} &&
+${indent}  exit_code=0 && break ||
+${indent}    exit_code=\$? && echo "${name} error: retry \$iter in 10s" && sleep 10;
+${indent}done;
+${indent}${exitKeyword} \$exit_code"""
+
+    return commandWithRetry.replaceAll(" *\n", " \\\\\n")
+  }
 }
 
 ext.expansions = { Architecture architecture, boolean oss, DockerBase base, boolean local ->
@@ -75,18 +95,6 @@ RUN curl --retry 8 -S -L \\
 
   def (major,minor) = VersionProperties.elasticsearch.split("\\.")
 
-  def retry_loop = { name, command, indentSize = 4, exitKeyword = 'exit'  ->
-    String indent = ' ' * indentSize
-    String commandWithRetry = """for iter in {1..10}; do
-${indent}  ${command} &&
-${indent}  exit_code=0 && break ||
-${indent}    exit_code=\$? && echo "${name} error: retry \$iter in 10s" && sleep 10;
-${indent}done;
-${indent}${exitKeyword} \$exit_code"""
-
-    return commandWithRetry.replaceAll(" *\n", " \\\\\n")
-  }
-
   return [
     'base_image'          : base.getImage(),
     'bin_dir'             : base == DockerBase.IRON_BANK ? 'scripts' : 'bin',
@@ -100,7 +108,7 @@ ${indent}${exitKeyword} \$exit_code"""
     'docker_base'         : base.name().toLowerCase(),
     'version'             : VersionProperties.elasticsearch,
     'major_minor_version' : "${major}.${minor}",
-    'retry_loop'          : retry_loop
+    'retry'               : Retry,
   ]
 }
 
@@ -185,6 +193,10 @@ void addCopyDockerContextTask(Architecture architecture, boolean oss, DockerBase
 
     with dockerBuildContext(architecture, oss, base, true)
 
+    into(base == DockerBase.IRON_BANK ? 'scripts' : 'bin') {
+      from configurations.transformLog4jJar
+    }
+
     if (architecture == Architecture.AARCH64) {
       if (oss) {
         from configurations.aarch64OssDockerSource
@@ -219,46 +231,6 @@ tasks.register("copyKeystore", Sync) {
   }
 }
 
-tasks.register("checkSecurityAuditLayoutPatternIdentical") {
-  // the two log4j2.properties files containing security audit configuration for archive and docker builds respectively
-  def originalLog4j = project(":x-pack:plugin:core").file('src/main/config/log4j2.properties')
-  def dockerLog4j = project.file("src/docker/config/log4j2.properties")
-  inputs.files(originalLog4j, dockerLog4j)
-  def patternPropertyKey = "appender.audit_rolling.layout.pattern"
-  doLast {
-    def coreLog4jProperties = new Properties()
-    originalLog4j.withInputStream { input ->
-      coreLog4jProperties.load(input)
-    }
-
-    if (false == coreLog4jProperties.containsKey(patternPropertyKey)) {
-      throw new GradleException("The [${originalLog4j.getPath()}] file changed such that the layout pattern is not " +
-              "referred to by the property named [${patternPropertyKey}]. Please update the task [${name}] " +
-              "definition from project [${path}] to reflect the new name for the layout pattern property.")
-    }
-
-    def dockerLog4jProperties = new Properties()
-    dockerLog4j.withInputStream { input ->
-      dockerLog4jProperties.load(input)
-    }
-
-    if (false == dockerLog4jProperties.containsKey(patternPropertyKey)) {
-      throw new GradleException("The [${dockerLog4j.getPath()}] file changed such that the layout pattern is not " +
-              "referred to by the property named [${patternPropertyKey}]. Please update the task [${name}] " +
-              "definition from project [${path}] to reflect the new name for the layout pattern property.")
-    }
-
-    if (false == coreLog4jProperties.getProperty(patternPropertyKey).equals(dockerLog4jProperties.getProperty(patternPropertyKey))) {
-      throw new GradleException("The property value for the layout pattern [${patternPropertyKey}] is NOT identical " +
-              "between the [${originalLog4j.getPath()}] and the [${dockerLog4j.getPath()}] files.")
-    }
-  }
-}
-
-tasks.named("precommit").configure {
-  dependsOn 'checkSecurityAuditLayoutPatternIdentical'
-}
-
 elasticsearch_distributions {
   Architecture.values().each { eachArchitecture ->
     Flavor.values().each { distroFlavor ->

distribution/docker/src/docker/Dockerfile

@@ -27,7 +27,7 @@
 FROM ${base_image} AS builder
 
 # Install required packages to extract the Elasticsearch distribution
-RUN <%= retry_loop(package_manager, "${package_manager} install -y findutils tar gzip") %>
+RUN <%= retry.loop(package_manager, "${package_manager} install -y findutils tar gzip") %>
 
 # `tini` is a tiny but valid init for containers. This is used to cleanly
 # control how ES and any child processes are shut down.
@@ -74,7 +74,7 @@ ENV TARBALL_URL https://curl.haxx.se/download/curl-\${VERSION}.tar.xz
 ENV TARBALL_PATH curl-\${VERSION}.tar.xz
 
 # Install dependencies
-RUN <%= retry_loop('apk', 'apk add gnupg gcc make musl-dev openssl-dev openssl-libs-static file') %>
+RUN <%= retry.loop('apk', 'apk add gnupg gcc make musl-dev openssl-dev openssl-libs-static file') %>
 
 RUN mkdir /work
 WORKDIR /work
@@ -83,7 +83,7 @@ WORKDIR /work
 RUN function retry_wget() { \\
       local URL="\$1" ; \\
       local DEST="\$2" ; \\
-      <%= retry_loop('wget', 'wget "\$URL\" -O "\$DEST"', 6, 'return') %> ; \\
+      <%= retry.loop('wget', 'wget "\$URL\" -O "\$DEST"', 6, 'return') %> ; \\
     } ; \\
     retry_wget "https://daniel.haxx.se/mykey.asc" "curl-gpg.pub" && \\
     retry_wget "\${TARBALL_URL}.asc" "\${TARBALL_PATH}.asc" && \\
@@ -223,21 +223,26 @@ ${source_elasticsearch}
 RUN tar -zxf /opt/elasticsearch.tar.gz --strip-components=1
 
 # The distribution includes a `config` directory, no need to create it
-COPY ${config_dir}/elasticsearch.yml ${config_dir}/log4j2.properties config/
+COPY ${config_dir}/elasticsearch.yml config/
+COPY ${bin_dir}/transform-log4j-config-${version}.jar /tmp/
 
 # 1. Configure the distribution for Docker
 # 2. Ensure directories are created. Most already are, but make sure
 # 3. Apply correct permissions
-# 4. Apply more correct permissions
-# 5. The JDK's directories' permissions don't allow `java` to be executed under a different
+# 4. Move the distribution's default logging config aside
+# 5. Generate a docker logging config, to be used by default
+# 6. Apply more correct permissions
+# 7. The JDK's directories' permissions don't allow `java` to be executed under a different
 #    group to the default. Fix this.
-# 6. Ensure that there are no files with setuid or setgid, in order to mitigate "stackclash" attacks.
-# 7. Ensure all files are world-readable by default. It should be possible to
+# 8. Ensure that there are no files with setuid or setgid, in order to mitigate "stackclash" attacks.
+# 9. Ensure all files are world-readable by default. It should be possible to
 #    examine the contents of the image under any UID:GID
-RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' /usr/share/elasticsearch/bin/elasticsearch-env && \\
+RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' bin/elasticsearch-env && \\
     mkdir -p config/jvm.options.d data logs plugins && \\
     chmod 0775 config config/jvm.options.d data logs plugins && \\
-    chmod 0660 config/elasticsearch.yml config/log4j2.properties && \\
+    mv config/log4j2.properties config/log4j2.file.properties && \\
+    jdk/bin/java -jar /tmp/transform-log4j-config-${version}.jar config/log4j2.file.properties > config/log4j2.properties && \\
+    chmod 0660 config/elasticsearch.yml config/log4j2*.properties && \\
     find ./jdk -type d -exec chmod 0755 {} + && \\
     find . -xdev -perm -4000 -exec chmod ug-s {} + && \\
     find . -type f -exec chmod o+r {} +
@@ -255,7 +260,7 @@ FROM ${base_image}
 
 <% if (docker_base == "ubi") { %>
 
-RUN <%= retry_loop(
+RUN <%= retry.loop(
     package_manager,
       "${package_manager} update --setopt=tsflags=nodocs -y && \n" +
       "      ${package_manager} install --setopt=tsflags=nodocs -y \n" +

distribution/docker/src/docker/bin/docker-entrypoint.sh

@@ -57,6 +57,21 @@ if [[ -f bin/elasticsearch-users ]]; then
   fi
 fi
 
+if [[ -n "$ES_LOG_STYLE" ]]; then
+  case "$ES_LOG_STYLE" in
+    console)
+      # This is the default. Nothing to do.
+      ;;
+    file)
+      # Overwrite the default config with the stack config
+      mv /usr/share/elasticsearch/config/log4j2.file.properties /usr/share/elasticsearch/config/log4j2.properties
+      ;;
+    *)
+      echo "ERROR: ES_LOG_STYLE set to [$ES_LOG_STYLE]. Expected [console] or [file]" >&2
+      exit 1 ;;
+  esac
+fi
+
 # Signal forwarding and child reaping is handled by `tini`, which is the
 # actual entrypoint of the container
 exec /usr/share/elasticsearch/bin/elasticsearch <<<"$KEYSTORE_PASSWORD"

distribution/docker/transform-log4j-config/build.gradle

@@ -0,0 +1,29 @@
+apply plugin: 'elasticsearch.build'
+
+repositories {
+  jcenter()
+}
+
+dependencies {
+  testImplementation "junit:junit:${versions.junit}"
+  testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}"
+}
+
+tasks.named('jar').configure {
+  manifest {
+    attributes 'Main-Class': 'org.elasticsearch.transform.log4j.TransformLog4jConfig'
+  }
+}
+
+// This tests depend on ES core
+disableTasks('forbiddenApisMain', 'forbiddenApisTest')
+
+tasks.named('testingConventions').configure {
+  naming.clear()
+  naming {
+    Tests {
+      baseClass 'junit.framework.TestCase'
+    }
+  }
+}
+