Autodetermine heap settings based on node roles and total system memory

distribution/tools/launchers/build.gradle

@@ -22,6 +22,7 @@ apply plugin: 'elasticsearch.build'
 
 dependencies {
   compileOnly project(':distribution:tools:java-version-checker')
+  compileOnly "org.yaml:snakeyaml:${versions.snakeyaml}"
   testImplementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testImplementation "junit:junit:${versions.junit}"
   testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}"
@@ -44,4 +45,4 @@ tasks.named("testingConventions").configure {
 
 ["javadoc", "loggerUsageCheck", "jarHell"].each { tsk ->
   tasks.named(tsk).configure { enabled = false }
-}
+}

distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/DefaultSystemMemoryInfo.java

@@ -0,0 +1,51 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.tools.launchers;
+
+import com.sun.management.OperatingSystemMXBean;
+import org.elasticsearch.tools.java_version_checker.JavaVersion;
+import org.elasticsearch.tools.java_version_checker.SuppressForbidden;
+
+import java.lang.management.ManagementFactory;
+
+/**
+ * A {@link SystemMemoryInfo} which delegates to {@link OperatingSystemMXBean}.
+ *
+ * <p>Prior to JDK 14 {@link OperatingSystemMXBean} did not take into consideration container memory limits when reporting total system
+ * memory. Therefore attempts to use this implementation on earlier JDKs will result in an {@link SystemMemoryInfoException}.
+ */
+@SuppressForbidden(reason = "Using com.sun internals is the only way to query total system memory")
+public final class DefaultSystemMemoryInfo implements SystemMemoryInfo {
+    private final OperatingSystemMXBean operatingSystemMXBean;
+
+    public DefaultSystemMemoryInfo() {
+        this.operatingSystemMXBean = (OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();
+    }
+
+    @Override
+    @SuppressWarnings("deprecation")
+    public long availableSystemMemory() throws SystemMemoryInfoException {
+        if (JavaVersion.majorVersion(JavaVersion.CURRENT) < 14) {
+            throw new SystemMemoryInfoException("The minimum required Java version is 14 to use " + this.getClass().getName());
+        }
+
+        return operatingSystemMXBean.getTotalPhysicalMemorySize();
+    }
+}

distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmErgonomics.java

@@ -19,22 +19,13 @@
 
 package org.elasticsearch.tools.launchers;
 
-import java.io.BufferedReader;
 import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
-import java.util.Optional;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 /**
  * Tunes Elasticsearch JVM settings based on inspection of provided JVM options.
@@ -53,9 +44,9 @@ private JvmErgonomics() {
      */
     static List<String> choose(final List<String> userDefinedJvmOptions) throws InterruptedException, IOException {
         final List<String> ergonomicChoices = new ArrayList<>();
-        final Map<String, JvmOption> finalJvmOptions = finalJvmOptions(userDefinedJvmOptions);
-        final long heapSize = extractHeapSize(finalJvmOptions);
-        final long maxDirectMemorySize = extractMaxDirectMemorySize(finalJvmOptions);
+        final Map<String, JvmOption> finalJvmOptions = JvmOption.findFinalOptions(userDefinedJvmOptions);
+        final long heapSize = JvmOption.extractMaxHeapSize(finalJvmOptions);
+        final long maxDirectMemorySize = JvmOption.extractMaxDirectMemorySize(finalJvmOptions);
         if (maxDirectMemorySize == 0) {
             ergonomicChoices.add("-XX:MaxDirectMemorySize=" + heapSize / 2);
         }
@@ -78,89 +69,6 @@ static List<String> choose(final List<String> userDefinedJvmOptions) throws Inte
         return ergonomicChoices;
     }
 
-    private static final Pattern OPTION = Pattern.compile(
-        "^\\s*\\S+\\s+(?<flag>\\S+)\\s+:?=\\s+(?<value>\\S+)?\\s+\\{[^}]+?\\}\\s+\\{(?<origin>[^}]+)}"
-    );
-
-    private static class JvmOption {
-        private final String value;
-        private final String origin;
-
-        JvmOption(String value, String origin) {
-            this.value = value;
-            this.origin = origin;
-        }
-
-        public Optional<String> getValue() {
-            return Optional.ofNullable(value);
-        }
-
-        public String getMandatoryValue() {
-            return value;
-        }
-
-        public boolean isCommandLineOrigin() {
-            return "command line".equals(this.origin);
-        }
-    }
-
-    static Map<String, JvmOption> finalJvmOptions(final List<String> userDefinedJvmOptions) throws InterruptedException, IOException {
-        return flagsFinal(userDefinedJvmOptions).stream()
-            .map(OPTION::matcher)
-            .filter(Matcher::matches)
-            .collect(Collectors.toUnmodifiableMap(m -> m.group("flag"), m -> new JvmOption(m.group("value"), m.group("origin"))));
-    }
-
-    private static List<String> flagsFinal(final List<String> userDefinedJvmOptions) throws InterruptedException, IOException {
-        /*
-         * To deduce the final set of JVM options that Elasticsearch is going to start with, we start a separate Java process with the JVM
-         * options that we would pass on the command line. For this Java process we will add two additional flags, -XX:+PrintFlagsFinal and
-         * -version. This causes the Java process that we start to parse the JVM options into their final values, display them on standard
-         * output, print the version to standard error, and then exit. The JVM itself never bootstraps, and therefore this process is
-         * lightweight. By doing this, we get the JVM options parsed exactly as the JVM that we are going to execute would parse them
-         * without having to implement our own JVM option parsing logic.
-         */
-        final String java = Path.of(System.getProperty("java.home"), "bin", "java").toString();
-        final List<String> command = Stream.of(
-            Stream.of(java),
-            userDefinedJvmOptions.stream(),
-            Stream.of("-Xshare:off"),
-            Stream.of("-XX:+PrintFlagsFinal"),
-            Stream.of("-version")
-        ).reduce(Stream::concat).get().collect(Collectors.toUnmodifiableList());
-        final Process process = new ProcessBuilder().command(command).start();
-        final List<String> output = readLinesFromInputStream(process.getInputStream());
-        final List<String> error = readLinesFromInputStream(process.getErrorStream());
-        final int status = process.waitFor();
-        if (status != 0) {
-            final String message = String.format(
-                Locale.ROOT,
-                "starting java failed with [%d]\noutput:\n%s\nerror:\n%s",
-                status,
-                String.join("\n", output),
-                String.join("\n", error)
-            );
-            throw new RuntimeException(message);
-        } else {
-            return output;
-        }
-    }
-
-    private static List<String> readLinesFromInputStream(final InputStream is) throws IOException {
-        try (InputStreamReader isr = new InputStreamReader(is, StandardCharsets.UTF_8); BufferedReader br = new BufferedReader(isr)) {
-            return br.lines().collect(Collectors.toUnmodifiableList());
-        }
-    }
-
-    // package private for testing
-    static Long extractHeapSize(final Map<String, JvmOption> finalJvmOptions) {
-        return Long.parseLong(finalJvmOptions.get("MaxHeapSize").getMandatoryValue());
-    }
-
-    static long extractMaxDirectMemorySize(final Map<String, JvmOption> finalJvmOptions) {
-        return Long.parseLong(finalJvmOptions.get("MaxDirectMemorySize").getMandatoryValue());
-    }
-
     // Tune G1GC options for heaps < 8GB
     static boolean tuneG1GCForSmallHeap(final long heapSize) {
         return heapSize < 8L << 30;

distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOption.java

@@ -0,0 +1,129 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.tools.launchers;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+class JvmOption {
+    private final String value;
+    private final String origin;
+
+    JvmOption(String value, String origin) {
+        this.value = value;
+        this.origin = origin;
+    }
+
+    public Optional<String> getValue() {
+        return Optional.ofNullable(value);
+    }
+
+    public String getMandatoryValue() {
+        return value;
+    }
+
+    public boolean isCommandLineOrigin() {
+        return "command line".equals(this.origin);
+    }
+
+    private static final Pattern OPTION = Pattern.compile(
+        "^\\s*\\S+\\s+(?<flag>\\S+)\\s+:?=\\s+(?<value>\\S+)?\\s+\\{[^}]+?\\}\\s+\\{(?<origin>[^}]+)}"
+    );
+
+    public static Long extractMaxHeapSize(final Map<String, JvmOption> finalJvmOptions) {
+        return Long.parseLong(finalJvmOptions.get("MaxHeapSize").getMandatoryValue());
+    }
+
+    public static boolean isMaxHeapSpecified(final Map<String, JvmOption> finalJvmOptions) {
+        return finalJvmOptions.get("MaxHeapSize").isCommandLineOrigin();
+    }
+
+    public static boolean isMinHeapSpecified(final Map<String, JvmOption> finalJvmOptions) {
+        return finalJvmOptions.get("MinHeapSize").isCommandLineOrigin();
+    }
+
+    public static long extractMaxDirectMemorySize(final Map<String, JvmOption> finalJvmOptions) {
+        return Long.parseLong(finalJvmOptions.get("MaxDirectMemorySize").getMandatoryValue());
+    }
+
+    /**
+     * Determine the options present when invoking a JVM with the given user defined options.
+     */
+    public static Map<String, JvmOption> findFinalOptions(final List<String> userDefinedJvmOptions)
+            throws InterruptedException, IOException {
+        return flagsFinal(userDefinedJvmOptions).stream()
+            .map(OPTION::matcher)
+            .filter(Matcher::matches)
+            .collect(Collectors.toUnmodifiableMap(m -> m.group("flag"), m -> new JvmOption(m.group("value"), m.group("origin"))));
+    }
+
+    private static List<String> flagsFinal(final List<String> userDefinedJvmOptions) throws InterruptedException, IOException {
+        /*
+         * To deduce the final set of JVM options that Elasticsearch is going to start with, we start a separate Java process with the JVM
+         * options that we would pass on the command line. For this Java process we will add two additional flags, -XX:+PrintFlagsFinal and
+         * -version. This causes the Java process that we start to parse the JVM options into their final values, display them on standard
+         * output, print the version to standard error, and then exit. The JVM itself never bootstraps, and therefore this process is
+         * lightweight. By doing this, we get the JVM options parsed exactly as the JVM that we are going to execute would parse them
+         * without having to implement our own JVM option parsing logic.
+         */
+        final String java = Path.of(System.getProperty("java.home"), "bin", "java").toString();
+        final List<String> command = Stream.of(
+            Stream.of(java),
+            userDefinedJvmOptions.stream(),
+            Stream.of("-Xshare:off"),
+            Stream.of("-XX:+PrintFlagsFinal"),
+            Stream.of("-version")
+        ).reduce(Stream::concat).get().collect(Collectors.toUnmodifiableList());
+        final Process process = new ProcessBuilder().command(command).start();
+        final List<String> output = readLinesFromInputStream(process.getInputStream());
+        final List<String> error = readLinesFromInputStream(process.getErrorStream());
+        final int status = process.waitFor();
+        if (status != 0) {
+            final String message = String.format(
+                Locale.ROOT,
+                "starting java failed with [%d]\noutput:\n%s\nerror:\n%s",
+                status,
+                String.join("\n", output),
+                String.join("\n", error)
+            );
+            throw new RuntimeException(message);
+        } else {
+            return output;
+        }
+    }
+
+    private static List<String> readLinesFromInputStream(final InputStream is) throws IOException {
+        try (InputStreamReader isr = new InputStreamReader(is, StandardCharsets.UTF_8); BufferedReader br = new BufferedReader(isr)) {
+            return br.lines().collect(Collectors.toUnmodifiableList());
+        }
+    }
+}

distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java

@@ -134,6 +134,7 @@ private List<String> jvmOptions(final Path config, Path plugins, final String es
         throws InterruptedException, IOException, JvmOptionsFileParserException {
 
         final List<String> jvmOptions = readJvmOptionsFiles(config);
+        final MachineDependentHeap machineDependentHeap = new MachineDependentHeap(new DefaultSystemMemoryInfo());
 
         if (esJavaOpts != null) {
             jvmOptions.addAll(
@@ -142,6 +143,7 @@ private List<String> jvmOptions(final Path config, Path plugins, final String es
         }
 
         final List<String> substitutedJvmOptions = substitutePlaceholders(jvmOptions, Collections.unmodifiableMap(substitutions));
+        substitutedJvmOptions.addAll(machineDependentHeap.determineHeapSettings(config, substitutedJvmOptions));
         final List<String> ergonomicJvmOptions = JvmErgonomics.choose(substitutedJvmOptions);
         final List<String> systemJvmOptions = SystemJvmOptions.systemJvmOptions();
         final List<String> bootstrapOptions = BootstrapJvmOptions.bootstrapJvmOptions(plugins);