Introduce jvm.options.d for customizing JVM options (elastic#51882)

This commit introduces the ability to override JVM options by adding
custom JVM options files to a jvm.options.d directory. This simplifies
administration of Elasticsearch by not requiring administrators to keep
the root jvm.options file in sync with changes that we make to the root
jvm.options file. Instead, they are not expected to modify this file but
instead supply their own in jvm.options.d. In Docker installations, this
means they can bind mount this directory in. In future versions of
Elasticsearch, we can consider removing the root jvm.options file
(instead, providing all options there as system JVM options).

distribution/archives/build.gradle

@@ -44,6 +44,11 @@ task createPluginsDir(type: EmptyDirTask) {
   dir = "${pluginsDir}"
   dirMode = 0755
 }
+ext.jvmOptionsDir = new File(buildDir, 'jvm-options-hack/jvm.options.d')
+task createJvmOptionsDir(type: EmptyDirTask) {
+  dir = "${jvmOptionsDir}"
+  dirMode = 0750
+}
 
 CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String platform, boolean oss, boolean jdk) {
   return copySpec {
@@ -55,6 +60,10 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla
         dirMode 0750
         fileMode 0660
         with configFiles(distributionType, oss, jdk)
+        from {
+          dirMode 0750
+          jvmOptionsDir.getParent()
+        }
       }
       into('bin') {
         with binFiles(distributionType, oss, jdk)
@@ -94,7 +103,7 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla
 
 // common config across all zip/tar
 tasks.withType(AbstractArchiveTask) {
-  dependsOn createLogsDir, createPluginsDir
+  dependsOn createLogsDir, createPluginsDir, createJvmOptionsDir
   String subdir = it.name.substring('build'.size()).replaceAll(/[A-Z]/) { '-' + it.toLowerCase() }.substring(1)
   destinationDir = file("${subdir}/build/distributions")
   baseName = "elasticsearch${subdir.contains('oss') ? '-oss' : ''}"

distribution/docker/src/docker/Dockerfile

@@ -30,8 +30,8 @@ ${source_elasticsearch}
 RUN tar zxf /opt/${elasticsearch} --strip-components=1
 RUN grep ES_DISTRIBUTION_TYPE=tar /usr/share/elasticsearch/bin/elasticsearch-env \
     && sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' /usr/share/elasticsearch/bin/elasticsearch-env
-RUN mkdir -p config data logs
-RUN chmod 0775 config data logs
+RUN mkdir -p config config/jvm.options.d data logs
+RUN chmod 0775 config config/jvm.options.d data logs
 COPY config/elasticsearch.yml config/log4j2.properties config/
 RUN chmod 0660 config/elasticsearch.yml config/log4j2.properties
 

distribution/packages/build.gradle

@@ -79,8 +79,9 @@ void addProcessFilesTask(String type, boolean oss, boolean jdk) {
       mkdir "${packagingFiles}/var/lib/elasticsearch"
       mkdir "${packagingFiles}/usr/share/elasticsearch/plugins"
 
-      // bare empty dir for /etc/elasticsearch
+      // bare empty dir for /etc/elasticsearch and /etc/elasticsearch/jvm.options.d
       mkdir "${packagingFiles}/elasticsearch"
+      mkdir "${packagingFiles}/elasticsearch/jvm.options.d"
     }
   }
 }
@@ -197,6 +198,7 @@ Closure commonPackageConfig(String type, boolean oss, boolean jdk) {
       includeEmptyDirs true
       createDirectoryEntry true
       include("elasticsearch") // empty dir, just to add directory entry
+      include("elasticsearch/jvm.options.d") // empty dir, just to add directory entry
     }
     from("${packagingFiles}/etc/elasticsearch") {
       into('/etc/elasticsearch')

distribution/packages/src/common/scripts/postrm

@@ -17,6 +17,7 @@ else
 fi
 
 REMOVE_DIRS=false
+REMOVE_JVM_OPTIONS_DIRECTORY=false
 REMOVE_USER_AND_GROUP=false
 
 case "$1" in
@@ -27,6 +28,8 @@ case "$1" in
     ;;
 
     purge)
+        REMOVE_DIRS=true
+        REMOVE_JVM_OPTIONS_DIRECTORY=true
         REMOVE_USER_AND_GROUP=true
     ;;
     failed-upgrade|abort-install|abort-upgrade|disappear|upgrade|disappear)
@@ -80,6 +83,20 @@ if [ "$REMOVE_DIRS" = "true" ]; then
         rmdir --ignore-fail-on-non-empty /var/lib/elasticsearch
     fi
 
+    # delete the jvm.options.d directory if and only if empty
+    if [ -d "${ES_PATH_CONF}/jvm.options.d" ]; then
+        rmdir --ignore-fail-on-non-empty "${ES_PATH_CONF}/jvm.options.d"
+    fi
+
+    # delete the jvm.options.d directory if we are purging
+    if [ "$REMOVE_JVM_OPTIONS_DIRECTORY" = "true" ]; then
+      if [ -d "${ES_PATH_CONF}/jvm.options.d" ]; then
+        echo -n "Deleting jvm.options.d directory..."
+        rm -rf "${ES_PATH_CONF}/jvm.options.d"
+        echo " OK"
+      fi
+    fi
+
     # delete the conf directory if and only if empty
     if [ -d "${ES_PATH_CONF}" ]; then
         rmdir --ignore-fail-on-non-empty "${ES_PATH_CONF}"

distribution/packages/src/deb/lintian/elasticsearch

@@ -16,6 +16,7 @@ missing-dep-on-jarwrapper
 # we prefer to not make our config and log files world readable
 non-standard-file-perm etc/default/elasticsearch 0660 != 0644
 non-standard-dir-perm etc/elasticsearch/ 2750 != 0755
+non-standard-dir-perm etc/elasticsearch/jvm.options.d/ 2750 != 0755
 non-standard-file-perm etc/elasticsearch/*
 non-standard-dir-perm var/lib/elasticsearch/ 2750 != 0755
 non-standard-dir-perm var/log/elasticsearch/ 2750 != 0755

distribution/src/bin/elasticsearch

@@ -46,8 +46,7 @@ then
   fi
 fi
 
-ES_JVM_OPTIONS="$ES_PATH_CONF"/jvm.options
-ES_JAVA_OPTS=`export ES_TMPDIR; "$JAVA" -cp "$ES_CLASSPATH" org.elasticsearch.tools.launchers.JvmOptionsParser "$ES_JVM_OPTIONS"`
+ES_JAVA_OPTS=`export ES_TMPDIR; "$JAVA" -cp "$ES_CLASSPATH" org.elasticsearch.tools.launchers.JvmOptionsParser "$ES_PATH_CONF"`
 
 # manual parsing to find out, if process should be detached
 if [[ $DAEMONIZE = false ]]; then

distribution/src/bin/elasticsearch-service.bat

@@ -115,7 +115,7 @@ set ES_JVM_OPTIONS=%ES_PATH_CONF%\jvm.options
 if not "%ES_JAVA_OPTS%" == "" set ES_JAVA_OPTS=%ES_JAVA_OPTS: =;%
 
 @setlocal
-for /F "usebackq delims=" %%a in (`"%JAVA% -cp "!ES_CLASSPATH!" "org.elasticsearch.tools.launchers.JvmOptionsParser" "!ES_JVM_OPTIONS!" || echo jvm_options_parser_failed"`) do set ES_JAVA_OPTS=%%a
+for /F "usebackq delims=" %%a in (`"%JAVA% -cp "!ES_CLASSPATH!" "org.elasticsearch.tools.launchers.JvmOptionsParser" "!ES_PATH_CONF!" || echo jvm_options_parser_failed"`) do set ES_JAVA_OPTS=%%a
 @endlocal & set "MAYBE_JVM_OPTIONS_PARSER_FAILED=%ES_JAVA_OPTS%" & set ES_JAVA_OPTS=%ES_JAVA_OPTS%
 
 if "%MAYBE_JVM_OPTIONS_PARSER_FAILED%" == "jvm_options_parser_failed" (
@@ -167,15 +167,15 @@ for %%a in ("%ES_JAVA_OPTS:;=","%") do (
 @endlocal & set JVM_MS=%JVM_MS% & set JVM_MX=%JVM_MX% & set JVM_SS=%JVM_SS% & set OTHER_JAVA_OPTS=%OTHER_JAVA_OPTS%
 
 if "%JVM_MS%" == "" (
-  echo minimum heap size not set; configure using -Xms via "%ES_JVM_OPTIONS%" or ES_JAVA_OPTS
+  echo minimum heap size not set; configure using -Xms via "%ES_PATH_CONF%/jvm.options.d", or ES_JAVA_OPTS
   goto:eof
 )
 if "%JVM_MX%" == "" (
-  echo maximum heap size not set; configure using -Xmx via "%ES_JVM_OPTIONS%" or ES_JAVA_OPTS
+  echo maximum heap size not set; configure using -Xmx via "%ES_PATH_CONF%/jvm.options.d", or ES_JAVA_OPTS
   goto:eof
 )
 if "%JVM_SS%" == "" (
-  echo thread stack size not set; configure using -Xss via "%ES_JVM_OPTIONS%" or ES_JAVA_OPTS
+  echo thread stack size not set; configure using -Xss via "%ES_PATH_CONF%/jvm.options.d", or ES_JAVA_OPTS
   goto:eof
 )
 set OTHER_JAVA_OPTS=%OTHER_JAVA_OPTS:"=%

distribution/src/bin/elasticsearch.bat

@@ -72,9 +72,8 @@ if not defined ES_TMPDIR (
   for /f "tokens=* usebackq" %%a in (`CALL %JAVA% -cp "!ES_CLASSPATH!" "org.elasticsearch.tools.launchers.TempDirectory"`) do set  ES_TMPDIR=%%a
 )
 
-set ES_JVM_OPTIONS=%ES_PATH_CONF%\jvm.options
 @setlocal
-for /F "usebackq delims=" %%a in (`CALL %JAVA% -cp "!ES_CLASSPATH!" "org.elasticsearch.tools.launchers.JvmOptionsParser" "!ES_JVM_OPTIONS!" ^|^| echo jvm_options_parser_failed`) do set ES_JAVA_OPTS=%%a
+for /F "usebackq delims=" %%a in (`CALL %JAVA% -cp "!ES_CLASSPATH!" "org.elasticsearch.tools.launchers.JvmOptionsParser" "!ES_PATH_CONF!" ^|^| echo jvm_options_parser_failed`) do set ES_JAVA_OPTS=%%a
 @endlocal & set "MAYBE_JVM_OPTIONS_PARSER_FAILED=%ES_JAVA_OPTS%" & set ES_JAVA_OPTS=%ES_JAVA_OPTS%
 
 if "%MAYBE_JVM_OPTIONS_PARSER_FAILED%" == "jvm_options_parser_failed" (

distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java

@@ -27,7 +27,9 @@
 import java.io.InputStreamReader;
 import java.io.Reader;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
+import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -43,6 +45,7 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
 
 /**
  * Parses JVM options from a file and prints a single line with all JVM options to standard output.
@@ -53,82 +56,90 @@ final class JvmOptionsParser {
      * The main entry point. The exit code is 0 if the JVM options were successfully parsed, otherwise the exit code is 1. If an improperly
      * formatted line is discovered, the line is output to standard error.
      *
-     * @param args the args to the program which should consist of a single option, the path to the JVM options
+     * @param args the args to the program which should consist of a single option, the path to ES_PATH_CONF
      */
     public static void main(final String[] args) throws InterruptedException, IOException {
         if (args.length != 1) {
-            throw new IllegalArgumentException("expected one argument specifying path to jvm.options but was " + Arrays.toString(args));
-        }
-        final List<String> jvmOptions = new ArrayList<>();
-        final SortedMap<Integer, String> invalidLines = new TreeMap<>();
-        try (
-            InputStream is = Files.newInputStream(Paths.get(args[0]));
-            Reader reader = new InputStreamReader(is, StandardCharsets.UTF_8);
-            BufferedReader br = new BufferedReader(reader)
-        ) {
-            parse(JavaVersion.majorVersion(JavaVersion.CURRENT), br, new JvmOptionConsumer() {
-                @Override
-                public void accept(final String jvmOption) {
-                    jvmOptions.add(jvmOption);
-                }
-            }, new InvalidLineConsumer() {
-                @Override
-                public void accept(final int lineNumber, final String line) {
-                    invalidLines.put(lineNumber, line);
-                }
-            });
+            throw new IllegalArgumentException("expected one argument specifying path to ES_PATH_CONF but was " + Arrays.toString(args));
         }
 
-        if (invalidLines.isEmpty()) {
-            // now append the JVM options from ES_JAVA_OPTS
-            final String environmentJvmOptions = System.getenv("ES_JAVA_OPTS");
-            if (environmentJvmOptions != null) {
-                jvmOptions.addAll(
-                    Arrays.stream(environmentJvmOptions.split("\\s+"))
-                        .filter(Predicate.not(String::isBlank))
-                        .collect(Collectors.toUnmodifiableList())
-                );
+        final ArrayList<Path> jvmOptionsFiles = new ArrayList<>();
+        jvmOptionsFiles.add(Paths.get(args[0], "jvm.options"));
+
+        final Path jvmOptionsDirectory = Paths.get(args[0], "jvm.options.d");
+
+        if (Files.isDirectory(jvmOptionsDirectory)) {
+            try (
+                DirectoryStream<Path> jvmOptionsDirectoryStream = Files.newDirectoryStream(Paths.get(args[0], "jvm.options.d"), "*.options")
+            ) {
+                // collect the matching JVM options files after sorting them by Path::compareTo
+                StreamSupport.stream(jvmOptionsDirectoryStream.spliterator(), false).sorted().forEach(jvmOptionsFiles::add);
             }
-            final Map<String, String> substitutions = new HashMap<>();
-            substitutions.put("ES_TMPDIR", System.getenv("ES_TMPDIR"));
-            if (null != System.getenv("ES_PATH_CONF")) {
-                substitutions.put("ES_PATH_CONF", System.getenv("ES_PATH_CONF"));
+        }
+
+        final List<String> jvmOptions = new ArrayList<>();
+
+        for (final Path jvmOptionsFile : jvmOptionsFiles) {
+            final SortedMap<Integer, String> invalidLines = new TreeMap<>();
+            try (
+                InputStream is = Files.newInputStream(jvmOptionsFile);
+                Reader reader = new InputStreamReader(is, StandardCharsets.UTF_8);
+                BufferedReader br = new BufferedReader(reader)
+            ) {
+                parse(JavaVersion.majorVersion(JavaVersion.CURRENT), br, jvmOptions::add, invalidLines::put);
             }
-            final List<String> substitutedJvmOptions = substitutePlaceholders(jvmOptions, Collections.unmodifiableMap(substitutions));
-            final List<String> ergonomicJvmOptions = JvmErgonomics.choose(substitutedJvmOptions);
-            final List<String> systemJvmOptions = SystemJvmOptions.systemJvmOptions();
-            final List<String> finalJvmOptions = new ArrayList<>(
-                systemJvmOptions.size() + substitutedJvmOptions.size() + ergonomicJvmOptions.size()
-            );
-            finalJvmOptions.addAll(systemJvmOptions); // add the system JVM options first so that they can be overridden
-            finalJvmOptions.addAll(substitutedJvmOptions);
-            finalJvmOptions.addAll(ergonomicJvmOptions);
-            final String spaceDelimitedJvmOptions = spaceDelimitJvmOptions(finalJvmOptions);
-            Launchers.outPrintln(spaceDelimitedJvmOptions);
-            Launchers.exit(0);
-        } else {
-            final String errorMessage = String.format(
-                Locale.ROOT,
-                "encountered [%d] error%s parsing [%s]",
-                invalidLines.size(),
-                invalidLines.size() == 1 ? "" : "s",
-                args[0]
-            );
-            Launchers.errPrintln(errorMessage);
-            int count = 0;
-            for (final Map.Entry<Integer, String> entry : invalidLines.entrySet()) {
-                count++;
-                final String message = String.format(
+            if (invalidLines.isEmpty() == false) {
+                final String errorMessage = String.format(
                     Locale.ROOT,
-                    "[%d]: encountered improperly formatted JVM option line [%s] on line number [%d]",
-                    count,
-                    entry.getValue(),
-                    entry.getKey()
+                    "encountered [%d] error%s parsing [%s]",
+                    invalidLines.size(),
+                    invalidLines.size() == 1 ? "" : "s",
+                    jvmOptionsFile
                 );
-                Launchers.errPrintln(message);
+                Launchers.errPrintln(errorMessage);
+                int count = 0;
+                for (final Map.Entry<Integer, String> entry : invalidLines.entrySet()) {
+                    count++;
+                    final String message = String.format(
+                        Locale.ROOT,
+                        "[%d]: encountered improperly formatted JVM option in [%s] on line number [%d]: [%s]",
+                        count,
+                        jvmOptionsFile,
+                        entry.getKey(),
+                        entry.getValue()
+                    );
+                    Launchers.errPrintln(message);
+                }
+                Launchers.exit(1);
             }
-            Launchers.exit(1);
         }
+
+        // now append the JVM options from ES_JAVA_OPTS
+        final String environmentJvmOptions = System.getenv("ES_JAVA_OPTS");
+        if (environmentJvmOptions != null) {
+            jvmOptions.addAll(
+                Arrays.stream(environmentJvmOptions.split("\\s+"))
+                    .filter(Predicate.not(String::isBlank))
+                    .collect(Collectors.toUnmodifiableList())
+            );
+        }
+        final Map<String, String> substitutions = new HashMap<>();
+        substitutions.put("ES_TMPDIR", System.getenv("ES_TMPDIR"));
+        if (null != System.getenv("ES_PATH_CONF")) {
+            substitutions.put("ES_PATH_CONF", System.getenv("ES_PATH_CONF"));
+        }
+        final List<String> substitutedJvmOptions = substitutePlaceholders(jvmOptions, Collections.unmodifiableMap(substitutions));
+        final List<String> ergonomicJvmOptions = JvmErgonomics.choose(substitutedJvmOptions);
+        final List<String> systemJvmOptions = SystemJvmOptions.systemJvmOptions();
+        final List<String> finalJvmOptions = new ArrayList<>(
+            systemJvmOptions.size() + substitutedJvmOptions.size() + ergonomicJvmOptions.size()
+        );
+        finalJvmOptions.addAll(systemJvmOptions); // add the system JVM options first so that they can be overridden
+        finalJvmOptions.addAll(substitutedJvmOptions);
+        finalJvmOptions.addAll(ergonomicJvmOptions);
+        final String spaceDelimitedJvmOptions = spaceDelimitJvmOptions(finalJvmOptions);
+        Launchers.outPrintln(spaceDelimitedJvmOptions);
+        Launchers.exit(0);
     }
 
     static List<String> substitutePlaceholders(final List<String> jvmOptions, final Map<String, String> substitutions) {

docs/reference/setup/important-settings/heap-size.asciidoc

@@ -49,7 +49,7 @@ caches, but the less memory it leaves available for the operating system to use
 for the filesystem cache. Also, larger heaps can cause longer garbage
 collection pauses.
 
-Here are examples of how to set the heap size via the jvm.options file:
+Here is an example of how to set the heap size via a `jvm.options.d/` file:
 
 [source,txt]
 ------------------
@@ -60,8 +60,7 @@ Here are examples of how to set the heap size via the jvm.options file:
 <2> Set the maximum heap size to 2g.
 
 It is also possible to set the heap size via an environment variable. This can
-be done by commenting out the `Xms` and `Xmx` settings in the
-<<jvm-options,`jvm.options`>> file and setting these values via `ES_JAVA_OPTS`:
+be done by setting these values via `ES_JAVA_OPTS`:
 
 [source,sh]
 ------------------

docs/reference/setup/install/docker.asciidoc

@@ -266,22 +266,23 @@ unless you are pinning one container per host.
 [[docker-set-heap-size]]
 ===== Set the heap size
 
-Use the `ES_JAVA_OPTS` environment variable to set the heap size.
-For example, to use 16GB, specify `-e ES_JAVA_OPTS="-Xms16g -Xmx16g"` with
-`docker run`. Note that while the default configuration file `jvm.options`
-sets a default heap of 1GB, any value you set in `ES_JAVA_OPTS` will
-override it.
+To configure the heap size, you can bind mount a <<jvm-options,JVM options>>
+file under `/usr/share/elasticsearch/config/jvm.options.d` that includes your
+desired <<heap-size,heap size>> settings. Note that while the default root
+`jvm.options` file sets a default heap of 1 GB, any value you set in a
+bind-mounted JVM options file will override it.
+
+While setting the heap size via bind-mounted JVM options is the recommended
+method, you can also configure this by using the `ES_JAVA_OPTS` environment
+variable to set the heap size. For example, to use 16 GB, specify
+`-e ES_JAVA_OPTS="-Xms16g -Xmx16g"` with `docker run`. Note that while the
+default root `jvm.options` file sets a default heap of 1 GB, any value you set
+in `ES_JAVA_OPTS` will override it.
 
 IMPORTANT: You must <<heap-size,configure the heap size>> even if you are
 https://docs.docker.com/config/containers/resource_constraints/#limit-a-containers-access-to-memory[limiting
 memory access] to the container.
 
-While setting the heap size via an environment variable is the recommended
-method, you can also configure this by bind-mounting your own `jvm.options`
-file under `/usr/share/elasticsearch/config/`. The file that {es} provides
-contains some important settings, so you should start by taking a copy of
-`jvm.options` from an {es} container and editing it as you require.
-
 ===== Pin deployments to a specific image version
 
 Pin your deployments to a specific version of the {es} Docker image. For