Allow total memory to be overridden

distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java

@@ -123,7 +123,6 @@ private List<String> jvmOptions(final Path config, Path plugins, final String es
         throws InterruptedException, IOException, JvmOptionsFileParserException {
 
         final List<String> jvmOptions = readJvmOptionsFiles(config);
-        final MachineDependentHeap machineDependentHeap = new MachineDependentHeap(new DefaultSystemMemoryInfo());
 
         if (esJavaOpts != null) {
             jvmOptions.addAll(
@@ -132,6 +131,9 @@ private List<String> jvmOptions(final Path config, Path plugins, final String es
         }
 
         final List<String> substitutedJvmOptions = substitutePlaceholders(jvmOptions, Collections.unmodifiableMap(substitutions));
+        final MachineDependentHeap machineDependentHeap = new MachineDependentHeap(
+            new OverridableSystemMemoryInfo(substitutedJvmOptions, new DefaultSystemMemoryInfo())
+        );
         substitutedJvmOptions.addAll(machineDependentHeap.determineHeapSettings(config, substitutedJvmOptions));
         final List<String> ergonomicJvmOptions = JvmErgonomics.choose(substitutedJvmOptions);
         final List<String> systemJvmOptions = SystemJvmOptions.systemJvmOptions();

distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/OverridableSystemMemoryInfo.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.tools.launchers;
+
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * A {@link SystemMemoryInfo} which returns a user-overridden memory size if one
+ * has been specified using the {@code es.total_memory_bytes} system property, or
+ * else returns the value provided by a fallback provider.
+ */
+public final class OverridableSystemMemoryInfo implements SystemMemoryInfo {
+
+    private final List<String> userDefinedJvmOptions;
+    private final SystemMemoryInfo fallbackSystemMemoryInfo;
+
+    public OverridableSystemMemoryInfo(final List<String> userDefinedJvmOptions, SystemMemoryInfo fallbackSystemMemoryInfo) {
+        this.userDefinedJvmOptions = Objects.requireNonNull(userDefinedJvmOptions);
+        this.fallbackSystemMemoryInfo = Objects.requireNonNull(fallbackSystemMemoryInfo);
+    }
+
+    @Override
+    public long availableSystemMemory() throws SystemMemoryInfoException {
+
+        return userDefinedJvmOptions.stream()
+            .filter(option -> option.startsWith("-Des.total_memory_bytes="))
+            .map(totalMemoryBytesOption -> {
+                try {
+                    long bytes = Long.parseLong(totalMemoryBytesOption.split("=", 2)[1]);
+                    if (bytes < 0) {
+                        throw new IllegalArgumentException("Negative memory size specified in [" + totalMemoryBytesOption + "]");
+                    }
+                    return bytes;
+                } catch (NumberFormatException e) {
+                    throw new IllegalArgumentException("Unable to parse number of bytes from [" + totalMemoryBytesOption + "]", e);
+                }
+            })
+            .reduce((previous, current) -> current) // this is effectively findLast(), so that ES_JAVA_OPTS overrides jvm.options
+            .orElse(fallbackSystemMemoryInfo.availableSystemMemory());
+    }
+}

distribution/tools/launchers/src/test/java/org/elasticsearch/tools/launchers/OverridableSystemMemoryInfoTests.java

@@ -0,0 +1,86 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.tools.launchers;
+
+import org.elasticsearch.tools.launchers.SystemMemoryInfo.SystemMemoryInfoException;
+
+import java.util.List;
+
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.fail;
+
+public class OverridableSystemMemoryInfoTests extends LaunchersTestCase {
+
+    private static final long FALLBACK = -1L;
+
+    public void testNoOptions() throws SystemMemoryInfoException {
+        final SystemMemoryInfo memoryInfo = new OverridableSystemMemoryInfo(List.of(), fallbackSystemMemoryInfo());
+        assertThat(memoryInfo.availableSystemMemory(), is(FALLBACK));
+    }
+
+    public void testNoOverrides() throws SystemMemoryInfoException {
+        final SystemMemoryInfo memoryInfo = new OverridableSystemMemoryInfo(List.of("-Da=b", "-Dx=y"), fallbackSystemMemoryInfo());
+        assertThat(memoryInfo.availableSystemMemory(), is(FALLBACK));
+    }
+
+    public void testValidSingleOverride() throws SystemMemoryInfoException {
+        final SystemMemoryInfo memoryInfo = new OverridableSystemMemoryInfo(
+            List.of("-Des.total_memory_bytes=123456789"),
+            fallbackSystemMemoryInfo()
+        );
+        assertThat(memoryInfo.availableSystemMemory(), is(123456789L));
+    }
+
+    public void testValidOverrideInList() throws SystemMemoryInfoException {
+        final SystemMemoryInfo memoryInfo = new OverridableSystemMemoryInfo(
+            List.of("-Da=b", "-Des.total_memory_bytes=987654321", "-Dx=y"),
+            fallbackSystemMemoryInfo()
+        );
+        assertThat(memoryInfo.availableSystemMemory(), is(987654321L));
+    }
+
+    public void testMultipleValidOverridesInList() throws SystemMemoryInfoException {
+        final SystemMemoryInfo memoryInfo = new OverridableSystemMemoryInfo(
+            List.of("-Des.total_memory_bytes=123456789", "-Da=b", "-Des.total_memory_bytes=987654321", "-Dx=y"),
+            fallbackSystemMemoryInfo()
+        );
+        assertThat(memoryInfo.availableSystemMemory(), is(987654321L));
+    }
+
+    public void testNegativeOverride() throws SystemMemoryInfoException {
+        final SystemMemoryInfo memoryInfo = new OverridableSystemMemoryInfo(
+            List.of("-Da=b", "-Des.total_memory_bytes=-123", "-Dx=y"),
+            fallbackSystemMemoryInfo()
+        );
+        try {
+            memoryInfo.availableSystemMemory();
+            fail("expected to fail");
+        } catch (IllegalArgumentException e) {
+            assertThat(e.getMessage(), is("Negative memory size specified in [-Des.total_memory_bytes=-123]"));
+        }
+    }
+
+    public void testUnparsableOverride() throws SystemMemoryInfoException {
+        final SystemMemoryInfo memoryInfo = new OverridableSystemMemoryInfo(
+            List.of("-Da=b", "-Des.total_memory_bytes=invalid", "-Dx=y"),
+            fallbackSystemMemoryInfo()
+        );
+        try {
+            memoryInfo.availableSystemMemory();
+            fail("expected to fail");
+        } catch (IllegalArgumentException e) {
+            assertThat(e.getMessage(), is("Unable to parse number of bytes from [-Des.total_memory_bytes=invalid]"));
+        }
+    }
+
+    private static SystemMemoryInfo fallbackSystemMemoryInfo() {
+        return () -> FALLBACK;
+    }
+}

docs/changelog/78750.yaml

@@ -0,0 +1,6 @@
+pr: 78750
+summary: Allow total memory to be overridden
+area: Packaging
+type: enhancement
+issues:
+ - 65905

docs/reference/cluster/nodes-stats.asciidoc

@@ -1036,6 +1036,17 @@ Total amount of physical memory.
 (integer)
 Total amount of physical memory in bytes.
 
+`total_override`::
+(<<byte-units,byte value>>)
+If the amount of physical memory has been overridden using the `es.total_memory_bytes`
+system property then this reports the overridden value. Otherwise it is not present.
+
+`total_override_in_bytes`::
+(integer)
+If the amount of physical memory has been overridden using the `es.total_memory_bytes`
+system property then this reports the overridden value in bytes. Otherwise it is not
+present.
+
 `free`::
 (<<byte-units,byte value>>)
 Amount of free physical memory.

docs/reference/cluster/stats.asciidoc

@@ -916,6 +916,18 @@ Total amount of physical memory across all selected nodes.
 (integer)
 Total amount, in bytes, of physical memory across all selected nodes.
 
+`total_override`::
+(<<byte-units,byte value>>)
+If the amount of physical memory has been overridden using the `es.total_memory_bytes`
+system property on all selected nodes then this reports the sum of the overridden
+values. Otherwise it is not present.
+
+`total_override_in_bytes`::
+(integer)
+If the amount of physical memory has been overridden using the `es.total_memory_bytes`
+system property on all selected nodes then this reports the sum of the overridden
+values in bytes. Otherwise it is not present.
+
 `free`::
 (<<byte-units, byte units>>)
 Amount of free physical memory across all selected nodes.

qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java

@@ -346,6 +346,27 @@ public void test73CustomJvmOptionsDirectoryFilesWithoutOptionsExtensionIgnored()
         }
     }
 
+    public void test74CustomJvmOptionsTotalMemoryOverride() throws Exception {
+        final Path heapOptions = installation.config(Paths.get("jvm.options.d", "total_memory.options"));
+        try {
+            setHeap(null); // delete default options
+            // Work as though total system memory is 850MB
+            append(heapOptions, "-Des.total_memory_bytes=891289600\n");
+
+            startElasticsearch();
+
+            final String nodesStatsResponse = makeRequest(Request.Get("http://localhost:9200/_nodes/stats"));
+            assertThat(nodesStatsResponse, containsString("\"total_override_in_bytes\":891289600"));
+            final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes"));
+            // 40% of 850MB
+            assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":356515840"));
+
+            stopElasticsearch();
+        } finally {
+            rm(heapOptions);
+        }
+    }
+
     public void test80RelativePathConf() throws Exception {
 
         withCustomConfig(tempConf -> {

qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java

@@ -868,22 +868,45 @@ public void test140CgroupOsStatsAreAvailable() throws Exception {
      * logic sets the correct heap size, based on the container limits.
      */
     public void test150MachineDependentHeap() throws Exception {
+        final List<String> xArgs = machineDependentHeapTest("942m", List.of());
+
+        // This is roughly 0.4 * 942
+        assertThat(xArgs, hasItems("-Xms376m", "-Xmx376m"));
+    }
+
+    /**
+     * Check that when available system memory is constrained by a total memory override as well as Docker,
+     * the machine-dependant heap sizing logic sets the correct heap size, preferring the override to the
+     * container limits.
+     */
+    public void test151MachineDependentHeapWithSizeOverride() throws Exception {
+        final List<String> xArgs = machineDependentHeapTest(
+            "942m",
+            // 799014912 = 762m
+            List.of("-Des.total_memory_bytes=799014912")
+        );
+
+        // This is roughly 0.4 * 762, in particular it's NOT 0.4 * 942
+        assertThat(xArgs, hasItems("-Xms304m", "-Xmx304m"));
+    }
+
+    private List<String> machineDependentHeapTest(final String containerMemory, final List<String> extraJvmOptions) throws Exception {
         // Start by ensuring `jvm.options` doesn't define any heap options
         final Path jvmOptionsPath = tempDir.resolve("jvm.options");
         final Path containerJvmOptionsPath = installation.config("jvm.options");
         copyFromContainer(containerJvmOptionsPath, jvmOptionsPath);
 
-        final List<String> jvmOptions = Files.readAllLines(jvmOptionsPath)
-            .stream()
-            .filter(line -> (line.startsWith("-Xms") || line.startsWith("-Xmx")) == false)
-            .collect(Collectors.toList());
+        final List<String> jvmOptions = Stream.concat(
+            Files.readAllLines(jvmOptionsPath).stream().filter(line -> (line.startsWith("-Xms") || line.startsWith("-Xmx")) == false),
+            extraJvmOptions.stream()
+        ).collect(Collectors.toList());
 
         Files.writeString(jvmOptionsPath, String.join("\n", jvmOptions));
 
         // Now run the container, being explicit about the available memory
         runContainer(
             distribution(),
-            builder().memory("942m").volume(jvmOptionsPath, containerJvmOptionsPath).envVar("ELASTIC_PASSWORD", PASSWORD)
+            builder().memory(containerMemory).volume(jvmOptionsPath, containerJvmOptionsPath).envVar("ELASTIC_PASSWORD", PASSWORD)
         );
         waitForElasticsearch(installation, USERNAME, PASSWORD);
 
@@ -897,12 +920,9 @@ public void test150MachineDependentHeap() throws Exception {
         final JsonNode jsonNode = new ObjectMapper().readTree(jvmArgumentsLine.get());
 
         final String argsStr = jsonNode.get("message").textValue();
-        final List<String> xArgs = Arrays.stream(argsStr.substring(1, argsStr.length() - 1).split(",\\s*"))
+        return Arrays.stream(argsStr.substring(1, argsStr.length() - 1).split(",\\s*"))
             .filter(arg -> arg.startsWith("-X"))
             .collect(Collectors.toList());
-
-        // This is roughly 0.4 * 942
-        assertThat(xArgs, hasItems("-Xms376m", "-Xmx376m"));
     }
 
     /**

qa/os/src/test/java/org/elasticsearch/packaging/test/PackageTests.java

@@ -298,6 +298,32 @@ public void test81CustomPathConfAndJvmOptions() throws Exception {
         cleanup();
     }
 
+    public void test82JvmOptionsTotalMemoryOverride() throws Exception {
+        assumeTrue(isSystemd());
+
+        install();
+
+        assertPathsExist(installation.envFile);
+        stopElasticsearch();
+
+        withCustomConfig(tempConf -> {
+            // Work as though total system memory is 850MB
+            append(installation.envFile, "ES_JAVA_OPTS=\"-Des.total_memory_bytes=891289600\"");
+
+            startElasticsearch();
+
+            final String nodesStatsResponse = makeRequest(Request.Get("http://localhost:9200/_nodes/stats"));
+            assertThat(nodesStatsResponse, containsString("\"total_override_in_bytes\":891289600"));
+
+            // 40% of 850MB
+            assertThat(sh.run("ps auwwx").stdout, containsString("-Xmx340m -Xms340m"));
+
+            stopElasticsearch();
+        });
+
+        cleanup();
+    }
+
     public void test83SystemdMask() throws Exception {
         try {
             assumeTrue(isSystemd());

server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java

@@ -266,20 +266,33 @@ private OsStats(List<NodeInfo> nodeInfos, List<NodeStats> nodeStatsList) {
             this.allocatedProcessors = allocatedProcessors;
 
             long totalMemory = 0;
+            Long totalMemoryOverride = 0L;
             long freeMemory = 0;
             for (NodeStats nodeStats : nodeStatsList) {
                 if (nodeStats.getOs() != null) {
-                    long total = nodeStats.getOs().getMem().getTotal().getBytes();
+                    org.elasticsearch.monitor.os.OsStats.Mem mem = nodeStats.getOs().getMem();
+                    long total = mem.getTotal().getBytes();
                     if (total > 0) {
                         totalMemory += total;
                     }
+                    // Only report a total memory override for the whole cluster if every node has overridden total memory
+                    if (totalMemoryOverride != null) {
+                        if (mem.getTotalOverride() != null) {
+                            long totalOverride = mem.getTotalOverride().getBytes();
+                            if (totalOverride > 0) {
+                                totalMemoryOverride += totalOverride;
+                            }
+                        } else {
+                            totalMemoryOverride = null;
+                        }
+                    }
                     long free = nodeStats.getOs().getMem().getFree().getBytes();
                     if (free > 0) {
                         freeMemory += free;
                     }
                 }
             }
-            this.mem = new org.elasticsearch.monitor.os.OsStats.Mem(totalMemory, freeMemory);
+            this.mem = new org.elasticsearch.monitor.os.OsStats.Mem(totalMemory, totalMemoryOverride, freeMemory);
         }
 
         public int getAvailableProcessors() {