Autodetermine heap settings based on node roles and total system memory

distribution/src/config/jvm.options

@@ -36,8 +36,8 @@
 # Xms represents the initial size of the JVM heap
 # Xmx represents the maximum size of the JVM heap
 
--Xms${heap.min}
--Xmx${heap.max}
+# -Xms${heap.min}
+# -Xmx${heap.max}
 
 
 

distribution/tools/launchers/build.gradle

@@ -22,6 +22,7 @@ apply plugin: 'elasticsearch.build'
 
 dependencies {
   compileOnly project(':distribution:tools:java-version-checker')
+  compileOnly "org.yaml:snakeyaml:${versions.snakeyaml}"
   testImplementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testImplementation "junit:junit:${versions.junit}"
   testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}"
@@ -44,4 +45,4 @@ tasks.named("testingConventions").configure {
 
 ["javadoc", "loggerUsageCheck", "jarHell"].each { tsk ->
   tasks.named(tsk).configure { enabled = false }
-}
+}

distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/DefaultSystemMemoryInfo.java

@@ -0,0 +1,51 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.tools.launchers;
+
+import com.sun.management.OperatingSystemMXBean;
+import org.elasticsearch.tools.java_version_checker.JavaVersion;
+import org.elasticsearch.tools.java_version_checker.SuppressForbidden;
+
+import java.lang.management.ManagementFactory;
+
+/**
+ * A {@link SystemMemoryInfo} which delegates to {@link OperatingSystemMXBean}.
+ *
+ * <p>Prior to JDK 14 {@link OperatingSystemMXBean} did not take into consideration container memory limits when reporting total system
+ * memory. Therefore attempts to use this implementation on earlier JDKs will result in an {@link SystemMemoryInfoException}.
+ */
+@SuppressForbidden(reason = "Using com.sun internals is the only way to query total system memory")
+public final class DefaultSystemMemoryInfo implements SystemMemoryInfo {
+    private final OperatingSystemMXBean operatingSystemMXBean;
+
+    public DefaultSystemMemoryInfo() {
+        this.operatingSystemMXBean = (OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();
+    }
+
+    @Override
+    @SuppressWarnings("deprecation")
+    public long availableSystemMemory() throws SystemMemoryInfoException {
+        if (JavaVersion.majorVersion(JavaVersion.CURRENT) < 14) {
+            throw new SystemMemoryInfoException("The minimum required Java version is 14 to use " + this.getClass().getName());
+        }
+
+        return operatingSystemMXBean.getTotalPhysicalMemorySize();
+    }
+}

distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java

@@ -134,6 +134,7 @@ private List<String> jvmOptions(final Path config, Path plugins, final String es
         throws InterruptedException, IOException, JvmOptionsFileParserException {
 
         final List<String> jvmOptions = readJvmOptionsFiles(config);
+        final MachineDependentHeap machineDependentHeap = new MachineDependentHeap(new DefaultSystemMemoryInfo());
 
         if (esJavaOpts != null) {
             jvmOptions.addAll(
@@ -142,6 +143,7 @@ private List<String> jvmOptions(final Path config, Path plugins, final String es
         }
 
         final List<String> substitutedJvmOptions = substitutePlaceholders(jvmOptions, Collections.unmodifiableMap(substitutions));
+        substitutedJvmOptions.addAll(machineDependentHeap.determineHeapSettings(config, substitutedJvmOptions));
         final List<String> ergonomicJvmOptions = JvmErgonomics.choose(substitutedJvmOptions);
         final List<String> systemJvmOptions = SystemJvmOptions.systemJvmOptions();
         final List<String> bootstrapOptions = BootstrapJvmOptions.bootstrapJvmOptions(plugins);

distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/MachineDependentHeap.java

@@ -0,0 +1,247 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.tools.launchers;
+
+import org.yaml.snakeyaml.Yaml;
+import org.yaml.snakeyaml.error.YAMLException;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
+
+import static java.lang.Math.max;
+import static java.lang.Math.min;
+
+/**
+ * Determines optimal default heap settings based on available system memory and assigned node roles.
+ */
+public final class MachineDependentHeap {
+    private static final long GB = 1024L * 1024L * 1024L; // 1GB
+    private static final long MAX_HEAP_SIZE = GB * 31; // 31GB
+    private static final long MAX_ML_HEAP_SIZE = GB * 2; // 2GB
+    private static final long MIN_HEAP_SIZE = 1024 * 1024 * 128; // 128MB
+    private static final int DEFAULT_HEAP_SIZE_MB = 1024;
+    private static final String ELASTICSEARCH_YML = "elasticsearch.yml";
+
+    private final SystemMemoryInfo systemMemoryInfo;
+
+    public MachineDependentHeap(SystemMemoryInfo systemMemoryInfo) {
+        this.systemMemoryInfo = systemMemoryInfo;
+    }
+
+    /**
+     * Calculate heap options.
+     *
+     * @param configDir path to config directory
+     * @param userDefinedJvmOptions JVM arguments provided by the user
+     * @return final heap options, or an empty collection if user provided heap options are to be used
+     * @throws IOException if unable to load elasticsearch.yml
+     */
+    public List<String> determineHeapSettings(Path configDir, List<String> userDefinedJvmOptions) throws IOException {
+        if (userDefinedJvmOptions.stream().anyMatch(s -> s.startsWith("-Xms") || s.startsWith("-Xmx"))) {
+            // User has explicitly set memory settings so we use those
+            return Collections.emptyList();
+        }
+
+        Path config = configDir.resolve(ELASTICSEARCH_YML);
+        try (InputStream in = Files.newInputStream(config)) {
+            return determineHeapSettings(in);
+        }
+    }
+
+    List<String> determineHeapSettings(InputStream config) {
+        MachineNodeRole nodeRole = NodeRoleParser.parse(config);
+
+        try {
+            long availableSystemMemory = systemMemoryInfo.availableSystemMemory();
+            return options(nodeRole.heap(availableSystemMemory));
+        } catch (SystemMemoryInfo.SystemMemoryInfoException e) {
+            // If unable to determine system memory (ex: incompatible jdk version) fallback to defaults
+            return options(DEFAULT_HEAP_SIZE_MB);
+        }
+    }
+
+    private static List<String> options(int heapSize) {
+        return List.of("-Xms" + heapSize + "m", "-Xmx" + heapSize + "m");
+    }
+
+    /**
+     * Parses role information from elasticsearch.yml and determines machine node role.
+     */
+    static class NodeRoleParser {
+        private static final Set<String> LEGACY_ROLE_SETTINGS = Set.of(
+            "node.master",
+            "node.ingest",
+            "node.data",
+            "node.voting_only",
+            "node.ml",
+            "node.transform",
+            "node.remote_cluster_client"
+        );
+
+        @SuppressWarnings("unchecked")
+        public static MachineNodeRole parse(InputStream config) {
+            Yaml yaml = new Yaml();
+            Map<String, Object> root;
+            try {
+                root = yaml.load(config);
+            } catch (ClassCastException ex) {
+                // Strangely formatted config, so just return defaults and let startup settings validation catch the problem
+                return MachineNodeRole.UNKNOWN;
+            } catch (YAMLException ex) {
+                throw new IllegalStateException("Unable to parse elasticsearch.yml:", ex);
+            }
+
+            if (root != null) {
+                Map<String, Object> map = flatten(root, null);
+
+                if (hasLegacySettings(map.keySet())) {
+                    // We don't attempt to auto-determine heap if legacy role settings are used
+                    return MachineNodeRole.UNKNOWN;
+                } else {
+                    List<String> roles = null;
+                    try {
+                        if (map.containsKey("node.roles")) {
+                            roles = (List<String>) map.get("node.roles");
+                        }
+                    } catch (ClassCastException ex) {
+                        throw new IllegalStateException("Unable to parse elasticsearch.yml. Expected 'node.roles' to be a list.");
+                    }
+
+                    if (roles == null || roles.isEmpty()) {
+                        // If roles are missing or empty (coordinating node) assume defaults and consider this a data node
+                        return MachineNodeRole.DATA;
+                    } else if (containsOnly(roles, "master", "voting_only")) {
+                        return MachineNodeRole.MASTER_ONLY;
+                    } else if (containsOnly(roles, "ml", "voting_only")) {
+                        return MachineNodeRole.ML_ONLY;
+                    } else {
+                        return MachineNodeRole.DATA;
+                    }
+                }
+            } else { // if the config is completely empty, then assume defaults and consider this a data node
+                return MachineNodeRole.DATA;
+            }
+        }
+
+        /**
+         * Flattens a nested configuration structure. This creates a consistent way of referencing settings from a config file that uses
+         * a mix of object and flat setting notation. The returned map is a single-level deep structure of dot-notation property names
+         * to values.
+         *
+         * <p>No attempt is made to deterministically deal with duplicate settings, nor are they explicitly disallowed.
+         *
+         * @param config nested configuration map
+         * @param parentPath parent node path or {@code null} if parsing the root node
+         * @return flattened configuration map
+         */
+        @SuppressWarnings("unchecked")
+        private static Map<String, Object> flatten(Map<String, Object> config, String parentPath) {
+            Map<String, Object> flatMap = new HashMap<>();
+            String prefix = parentPath != null ? parentPath + "." : "";
+
+            for (Map.Entry<String, Object> entry : config.entrySet()) {
+                if (entry.getValue() instanceof Map) {
+                    flatMap.putAll(flatten((Map<String, Object>) entry.getValue(), prefix + entry.getKey()));
+                } else {
+                    flatMap.put(prefix + entry.getKey(), entry.getValue());
+                }
+            }
+
+            return flatMap;
+        }
+
+        @SuppressWarnings("unchecked")
+        private static <T> boolean containsOnly(Collection<T> collection, T... items) {
+            return Arrays.asList(items).containsAll(collection);
+        }
+
+        private static boolean hasLegacySettings(Set<String> keys) {
+            return LEGACY_ROLE_SETTINGS.stream().anyMatch(keys::contains);
+        }
+    }
+
+    enum MachineNodeRole {
+        /**
+         * Master-only node.
+         *
+         * <p>Heap is computed as 60% of total system memory up to a maximum of 31 gigabytes.
+         */
+        MASTER_ONLY(m -> mb(min((long) (m * .6), MAX_HEAP_SIZE))),
+
+        /**
+         * Machine learning only node.
+         *
+         * <p>Heap is computed as:
+         * <ul>
+         *     <li>40% of total system memory when less than 2 gigabytes.</li>
+         *     <li>25% of total system memory when greater than 2 gigabytes up to a maximum of 2 gigabytes.</li>
+         * </ul>
+         */
+        ML_ONLY(m -> mb(m < (GB * 2) ? (long) (m * .4) : (long) min(m * .25, MAX_ML_HEAP_SIZE))),
+
+        /**
+         * Data node. Essentially any node that isn't a master or ML only node.
+         *
+         * <p>Heap is computed as:
+         * <ul>
+         *     <li>40% of total system memory when less than 1 gigabyte with a minimum of 128 megabytes.</li>
+         *     <li>50% of total system memory when greater than 1 gigabyte up to a maximum of 31 gigabytes.</li>
+         * </ul>
+         */
+        DATA(m -> mb(m < GB ? max((long) (m * .4), MIN_HEAP_SIZE) : min((long) (m * .5), MAX_HEAP_SIZE))),
+
+        /**
+         * Unknown role node.
+         *
+         * <p>Hard-code heap to a default of 1 gigabyte.
+         */
+        UNKNOWN(m -> DEFAULT_HEAP_SIZE_MB);
+
+        private final Function<Long, Integer> formula;
+
+        MachineNodeRole(Function<Long, Integer> formula) {
+            this.formula = formula;
+        }
+
+        /**
+         * Determine the appropriate heap size for the given role and available system memory.
+         *
+         * @param systemMemory total available system memory in bytes
+         * @return recommended heap size in megabytes
+         */
+        public int heap(long systemMemory) {
+            return formula.apply(systemMemory);
+        }
+
+        private static int mb(long bytes) {
+            return (int) (bytes / (1024 * 1024));
+        }
+    }
+}

distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/SystemMemoryInfo.java

@@ -0,0 +1,42 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.tools.launchers;
+
+/**
+ * Determines available system memory that could be allocated for Elasticsearch, to include JVM heap and other native processes.
+ * The "available system memory" is defined as the total system memory which is visible to the Elasticsearch process. For instances
+ * in which Elasticsearch is running in a containerized environment (i.e. Docker) this is expected to be the limits set for the container,
+ * not the host system.
+ */
+public interface SystemMemoryInfo {
+
+    /**
+     *
+     * @return total system memory available to heap or native process allocation in bytes
+     * @throws SystemMemoryInfoException if unable to determine available system memory
+     */
+    long availableSystemMemory() throws SystemMemoryInfoException;
+
+    class SystemMemoryInfoException extends Exception {
+        public SystemMemoryInfoException(String message) {
+            super(message);
+        }
+    }
+}