Support _FILE suffixed env vars in Docker entrypoint

distribution/docker/src/docker/bin/docker-entrypoint.sh

@@ -38,6 +38,40 @@ if [[ "$1" != "eswrapper" ]]; then
   fi
 fi
 
+# Allow environment variables to be set by creating a file with the
+# contents, and setting an environment variable with the suffix _FILE to
+# point to it. This can be used to provide secrets to a container, without
+# the values being specified explicitly when running the container.
+for VAR_NAME_FILE in $(env | cut -f1 -d= | grep '_FILE$'); do
+  if [[ -n "$VAR_NAME_FILE" ]]; then
+    VAR_NAME="${VAR_NAME_FILE%_FILE}"
+
+    if env | grep "^${VAR_NAME}="; then
+      echo "ERROR: Both $VAR_NAME_FILE and $VAR_NAME are set. These are mutually exclusive." >&2
+      exit 1
+    fi
+
+    if [[ ! -e "${!VAR_NAME_FILE}" ]]; then
+      echo "ERROR: File ${!VAR_NAME_FILE} from $VAR_NAME_FILE does not exist" >&2
+      exit 1
+    fi
+
+    FILE_PERMS="$(stat -c '%a' ${!VAR_NAME_FILE})"
+
+    if [[ "$FILE_PERMS" != "400" && "$FILE_PERMS" != 600 ]]; then
+        echo "ERROR: File ${!VAR_NAME_FILE} from $VAR_NAME_FILE must have file permissions 400 or 600, but actually has: $FILE_PERMS" >&2
+        exit 1
+    fi
+
+    echo "Setting $VAR_NAME from $VAR_NAME_FILE at ${!VAR_NAME_FILE}" >&2
+    export "$VAR_NAME"="$(cat ${!VAR_NAME_FILE})"
+
+    unset VAR_NAME
+    # Unset the suffixed environment variable
+    unset "$VAR_NAME_FILE"
+  fi
+done
+
 # Parse Docker env vars to customize Elasticsearch
 #
 # e.g. Setting the env var cluster.name=testcluster

docs/reference/setup/install/docker.asciidoc

@@ -290,7 +290,7 @@ https://docs.docker.com/engine/extend/plugins/#volume-plugins[Docker volume plug
 
 ===== Avoid using `loop-lvm` mode
 
-If you are using the devicemapper storage driver, do not use the default `loop-lvm` mode. 
+If you are using the devicemapper storage driver, do not use the default `loop-lvm` mode.
 Configure docker-engine to use 
 https://docs.docker.com/engine/userguide/storagedriver/device-mapper-driver/#configure-docker-with-devicemapper[direct-lvm].
 
@@ -312,7 +312,16 @@ over the configuration files in the image.
 
 You can set individual {es} configuration parameters using Docker environment variables. 
 The <<docker-compose-file, sample compose file>> and the 
-<<docker-cli-run-dev-mode, single-node example>>  use this method. 
+<<docker-cli-run-dev-mode, single-node example>>  use this method.
+
+If you are providing secrets e.g. passwords to {es} and you do not want to pass them
+directly via environment variables, you can instead supply environment variables suffixed with
+"_FILE". The variable value specifies a file, whose contents are used for the secret.
+
+For example, if you specified the environment variable "ELASTIC_PASSWORD_FILE" with value
+"/run/secrets/password.txt", and bind-mounted a file at this location in the container with
+the contents "PleaseChangeMe", then this would be equivalent to specifying the environment
+variable "ELASTIC_PASSWORD" with the value "PleaseChangeMe".
 
 You can also override the default command for the image to pass {es} configuration
 parameters as command line options. For example:

qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java

@@ -25,10 +25,12 @@
 import org.elasticsearch.packaging.util.Installation;
 import org.elasticsearch.packaging.util.ServerUtils;
 import org.elasticsearch.packaging.util.Shell.Result;
+import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 
+import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Map;
@@ -40,22 +42,25 @@
 import static org.elasticsearch.packaging.util.Docker.existsInContainer;
 import static org.elasticsearch.packaging.util.Docker.removeContainer;
 import static org.elasticsearch.packaging.util.Docker.runContainer;
+import static org.elasticsearch.packaging.util.Docker.runContainerExpectingFailure;
 import static org.elasticsearch.packaging.util.Docker.verifyContainerInstallation;
+import static org.elasticsearch.packaging.util.Docker.waitForElasticsearch;
 import static org.elasticsearch.packaging.util.Docker.waitForPathToExist;
+import static org.elasticsearch.packaging.util.FileMatcher.p600;
 import static org.elasticsearch.packaging.util.FileMatcher.p660;
 import static org.elasticsearch.packaging.util.FileUtils.append;
 import static org.elasticsearch.packaging.util.FileUtils.getTempDir;
-import static org.elasticsearch.packaging.util.FileUtils.mkdir;
 import static org.elasticsearch.packaging.util.FileUtils.rm;
 import static org.elasticsearch.packaging.util.ServerUtils.makeRequest;
-import static org.elasticsearch.packaging.util.ServerUtils.waitForElasticsearch;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.Matchers.emptyString;
+import static org.hamcrest.Matchers.equalTo;
 import static org.junit.Assume.assumeTrue;
 
 public class DockerTests extends PackagingTestCase {
     protected DockerShell sh;
+    private Path tempDir;
 
     @BeforeClass
     public static void filterDistros() {
@@ -71,9 +76,15 @@ public static void cleanup() {
     }
 
     @Before
-    public void setupTest() throws Exception {
+    public void setupTest() throws IOException {
         sh = new DockerShell();
         installation = runContainer(distribution());
+        tempDir = Files.createTempDirectory(getTempDir(), DockerTests.class.getSimpleName());
+    }
+
+    @After
+    public void teardownTest() {
+        rm(tempDir);
     }
 
     /**
@@ -143,39 +154,156 @@ public void test60AutoCreateKeystore() throws Exception {
      * Check that the default config can be overridden using a bind mount, and that env vars are respected
      */
     public void test70BindMountCustomPathConfAndJvmOptions() throws Exception {
-        final Path tempConf = getTempDir().resolve("esconf-alternate");
+        copyFromContainer(installation.config("elasticsearch.yml"), tempDir.resolve("elasticsearch.yml"));
+        copyFromContainer(installation.config("log4j2.properties"), tempDir.resolve("log4j2.properties"));
+
+        // we have to disable Log4j from using JMX lest it will hit a security
+        // manager exception before we have configured logging; this will fail
+        // startup since we detect usages of logging before it is configured
+        final String jvmOptions = "-Xms512m\n-Xmx512m\n-Dlog4j2.disable.jmx=true\n";
+        append(tempDir.resolve("jvm.options"), jvmOptions);
+
+        // Make the temp directory and contents accessible when bind-mounted
+        Files.setPosixFilePermissions(tempDir, fromString("rwxrwxrwx"));
+
+        // Restart the container
+        final Map<Path, Path> volumes = Map.of(tempDir, Path.of("/usr/share/elasticsearch/config"));
+        runContainer(distribution(), volumes, Map.of("ES_JAVA_OPTS", "-XX:-UseCompressedOops"));
+
+        waitForElasticsearch(installation);
+
+        final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes"));
+        assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":536870912"));
+        assertThat(nodesResponse, containsString("\"using_compressed_ordinary_object_pointers\":\"false\""));
+    }
+
+    /**
+     * Check that environment variables can be populated by setting variables with the suffix "_FILE",
+     * which point to files that hold the required values.
+     */
+    public void test80SetEnvironmentVariablesUsingFiles() throws Exception {
+        final String optionsFilename = "esJavaOpts.txt";
+
+        // ES_JAVA_OPTS_FILE
+        Files.writeString(tempDir.resolve(optionsFilename), "-XX:-UseCompressedOops\n");
+
+        Map<String, String> envVars = Map.of("ES_JAVA_OPTS_FILE", "/run/secrets/" + optionsFilename);
+
+        // File permissions need to be secured in order for the ES wrapper to accept
+        // them for populating env var values
+        Files.setPosixFilePermissions(tempDir.resolve(optionsFilename), p600);
+
+        final Map<Path, Path> volumes = Map.of(tempDir, Path.of("/run/secrets"));
+
+        // Restart the container
+        runContainer(distribution(), volumes, envVars);
+
+        waitForElasticsearch(installation);
+
+        final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes"));
+
+        assertThat(nodesResponse, containsString("\"using_compressed_ordinary_object_pointers\":\"false\""));
+    }
+
+    /**
+     * Check that the elastic user's password can be configured via a file and the ELASTIC_PASSWORD_FILE environment variable.
+     */
+    public void test81ConfigurePasswordThroughEnvironmentVariableFile() throws Exception {
+        // Test relies on configuring security
+        assumeTrue(distribution.isDefault());
+
+        final String xpackPassword = "hunter2";
+        final String passwordFilename = "password.txt";
 
+        // ELASTIC_PASSWORD_FILE
+        Files.writeString(tempDir.resolve(passwordFilename), xpackPassword + "\n");
+
+        Map<String, String> envVars = Map
+            .of(
+                "ELASTIC_PASSWORD_FILE",
+                "/run/secrets/" + passwordFilename,
+                // Enable security so that we can test that the password has been used
+                "xpack.security.enabled",
+                "true"
+            );
+
+        // File permissions need to be secured in order for the ES wrapper to accept
+        // them for populating env var values
+        Files.setPosixFilePermissions(tempDir.resolve(passwordFilename), p600);
+
+        final Map<Path, Path> volumes = Map.of(tempDir, Path.of("/run/secrets"));
+
+        // Restart the container
+        runContainer(distribution(), volumes, envVars);
+
+        // If we configured security correctly, then this call will only work if we specify the correct credentials.
         try {
-            mkdir(tempConf);
-            copyFromContainer(installation.config("elasticsearch.yml"), tempConf.resolve("elasticsearch.yml"));
-            copyFromContainer(installation.config("log4j2.properties"), tempConf.resolve("log4j2.properties"));
-
-            // we have to disable Log4j from using JMX lest it will hit a security
-            // manager exception before we have configured logging; this will fail
-            // startup since we detect usages of logging before it is configured
-            final String jvmOptions =
-                "-Xms512m\n" +
-                "-Xmx512m\n" +
-                "-Dlog4j2.disable.jmx=true\n";
-            append(tempConf.resolve("jvm.options"), jvmOptions);
-
-            // Make the temp directory and contents accessible when bind-mounted
-            Files.setPosixFilePermissions(tempConf, fromString("rwxrwxrwx"));
-
-            // Restart the container
-            removeContainer();
-            runContainer(distribution(), tempConf, Map.of(
-                "ES_JAVA_OPTS", "-XX:-UseCompressedOops"
-            ));
-
-            waitForElasticsearch(installation);
-
-            final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes"));
-            assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":536870912"));
-            assertThat(nodesResponse, containsString("\"using_compressed_ordinary_object_pointers\":\"false\""));
-        } finally {
-            rm(tempConf);
+            waitForElasticsearch("green", null, installation, "elastic", "hunter2");
+        } catch (Exception e) {
+            throw new AssertionError(
+                "Failed to check whether Elasticsearch had started. This could be because "
+                    + "authentication isn't working properly. Check the container logs",
+                e
+            );
         }
+
+        // Also check that an unauthenticated call fails
+        final int statusCode = Request.Get("http://localhost:9200/_nodes").execute().returnResponse().getStatusLine().getStatusCode();
+        assertThat("Expected server to require authentication", statusCode, equalTo(401));
+    }
+
+    /**
+     * Check that environment variables cannot be used with _FILE environment variables.
+     */
+    public void test81CannotUseEnvVarsAndFiles() throws Exception {
+        final String optionsFilename = "esJavaOpts.txt";
+
+        // ES_JAVA_OPTS_FILE
+        Files.writeString(tempDir.resolve(optionsFilename), "-XX:-UseCompressedOops\n");
+
+        Map<String, String> envVars = Map.of(
+            "ES_JAVA_OPTS", "-XX:+UseCompressedOops",
+            "ES_JAVA_OPTS_FILE", "/run/secrets/" + optionsFilename
+        );
+
+        // File permissions need to be secured in order for the ES wrapper to accept
+        // them for populating env var values
+        Files.setPosixFilePermissions(tempDir.resolve(optionsFilename), p600);
+
+        final Map<Path, Path> volumes = Map.of(tempDir, Path.of("/run/secrets"));
+
+        final Result dockerLogs = runContainerExpectingFailure(distribution, volumes, envVars);
+
+        assertThat(
+            dockerLogs.stderr,
+            containsString("ERROR: Both ES_JAVA_OPTS_FILE and ES_JAVA_OPTS are set. These are mutually exclusive.")
+        );
+    }
+
+    /**
+     * Check that when populating environment variables by setting variables with the suffix "_FILE",
+     * the files' permissions are checked.
+     */
+    public void test82EnvironmentVariablesUsingFilesHaveCorrectPermissions() throws Exception {
+        final String optionsFilename = "esJavaOpts.txt";
+
+        // ES_JAVA_OPTS_FILE
+        Files.writeString(tempDir.resolve(optionsFilename), "-XX:-UseCompressedOops\n");
+
+        Map<String, String> envVars = Map.of("ES_JAVA_OPTS_FILE", "/run/secrets/" + optionsFilename);
+
+        // Set invalid file permissions
+        Files.setPosixFilePermissions(tempDir.resolve(optionsFilename), p660);
+
+        final Map<Path, Path> volumes = Map.of(tempDir, Path.of("/run/secrets"));
+
+        // Restart the container
+        final Result dockerLogs = runContainerExpectingFailure(distribution(), volumes, envVars);
+
+        assertThat(
+            dockerLogs.stderr,
+            containsString("ERROR: File /run/secrets/" + optionsFilename + " from ES_JAVA_OPTS_FILE must have file permissions 400 or 600")
+        );
     }
 
     /**
@@ -219,7 +347,6 @@ public void test92ElasticsearchNodeCliPackaging() {
         final Installation.Executables bin = installation.executables();
 
         final Result result = sh.run(bin.elasticsearchNode + " -h");
-        assertThat(result.stdout,
-            containsString("A CLI tool to do unsafe cluster and index manipulations on current node"));
+        assertThat(result.stdout, containsString("A CLI tool to do unsafe cluster and index manipulations on current node"));
     }
 }