diff --git a/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java index bb6da0c6a4eb5..4f33a1ba5b02c 100644 --- a/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java +++ b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java @@ -123,7 +123,6 @@ private List jvmOptions(final Path config, Path plugins, final String es throws InterruptedException, IOException, JvmOptionsFileParserException { final List jvmOptions = readJvmOptionsFiles(config); - final MachineDependentHeap machineDependentHeap = new MachineDependentHeap(new DefaultSystemMemoryInfo()); if (esJavaOpts != null) { jvmOptions.addAll( @@ -132,6 +131,9 @@ private List jvmOptions(final Path config, Path plugins, final String es } final List substitutedJvmOptions = substitutePlaceholders(jvmOptions, Collections.unmodifiableMap(substitutions)); + final MachineDependentHeap machineDependentHeap = new MachineDependentHeap( + new OverridableSystemMemoryInfo(substitutedJvmOptions, new DefaultSystemMemoryInfo()) + ); substitutedJvmOptions.addAll(machineDependentHeap.determineHeapSettings(config, substitutedJvmOptions)); final List ergonomicJvmOptions = JvmErgonomics.choose(substitutedJvmOptions); final List systemJvmOptions = SystemJvmOptions.systemJvmOptions(); diff --git a/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/OverridableSystemMemoryInfo.java b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/OverridableSystemMemoryInfo.java new file mode 100644 index 0000000000000..118c68b2111b6 --- /dev/null +++ b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/OverridableSystemMemoryInfo.java @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.tools.launchers; + +import java.util.List; +import java.util.Objects; + +/** + * A {@link SystemMemoryInfo} which returns a user-overridden memory size if one + * has been specified using the {@code es.total_memory_bytes} system property, or + * else returns the value provided by a fallback provider. + */ +public final class OverridableSystemMemoryInfo implements SystemMemoryInfo { + + private final List userDefinedJvmOptions; + private final SystemMemoryInfo fallbackSystemMemoryInfo; + + public OverridableSystemMemoryInfo(final List userDefinedJvmOptions, SystemMemoryInfo fallbackSystemMemoryInfo) { + this.userDefinedJvmOptions = Objects.requireNonNull(userDefinedJvmOptions); + this.fallbackSystemMemoryInfo = Objects.requireNonNull(fallbackSystemMemoryInfo); + } + + @Override + public long availableSystemMemory() throws SystemMemoryInfoException { + + return userDefinedJvmOptions.stream() + .filter(option -> option.startsWith("-Des.total_memory_bytes=")) + .map(totalMemoryBytesOption -> { + try { + long bytes = Long.parseLong(totalMemoryBytesOption.split("=", 2)[1]); + if (bytes < 0) { + throw new IllegalArgumentException("Negative memory size specified in [" + totalMemoryBytesOption + "]"); + } + return bytes; + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Unable to parse number of bytes from [" + totalMemoryBytesOption + "]", e); + } + }) + .reduce((previous, current) -> current) // this is effectively findLast(), so that ES_JAVA_OPTS overrides jvm.options + .orElse(fallbackSystemMemoryInfo.availableSystemMemory()); + } +} diff --git a/distribution/tools/launchers/src/test/java/org/elasticsearch/tools/launchers/OverridableSystemMemoryInfoTests.java b/distribution/tools/launchers/src/test/java/org/elasticsearch/tools/launchers/OverridableSystemMemoryInfoTests.java new file mode 100644 index 0000000000000..f56db17422578 --- /dev/null +++ b/distribution/tools/launchers/src/test/java/org/elasticsearch/tools/launchers/OverridableSystemMemoryInfoTests.java @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.tools.launchers; + +import org.elasticsearch.tools.launchers.SystemMemoryInfo.SystemMemoryInfoException; + +import java.util.List; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; + +public class OverridableSystemMemoryInfoTests extends LaunchersTestCase { + + private static final long FALLBACK = -1L; + + public void testNoOptions() throws SystemMemoryInfoException { + final SystemMemoryInfo memoryInfo = new OverridableSystemMemoryInfo(List.of(), fallbackSystemMemoryInfo()); + assertThat(memoryInfo.availableSystemMemory(), is(FALLBACK)); + } + + public void testNoOverrides() throws SystemMemoryInfoException { + final SystemMemoryInfo memoryInfo = new OverridableSystemMemoryInfo(List.of("-Da=b", "-Dx=y"), fallbackSystemMemoryInfo()); + assertThat(memoryInfo.availableSystemMemory(), is(FALLBACK)); + } + + public void testValidSingleOverride() throws SystemMemoryInfoException { + final SystemMemoryInfo memoryInfo = new OverridableSystemMemoryInfo( + List.of("-Des.total_memory_bytes=123456789"), + fallbackSystemMemoryInfo() + ); + assertThat(memoryInfo.availableSystemMemory(), is(123456789L)); + } + + public void testValidOverrideInList() throws SystemMemoryInfoException { + final SystemMemoryInfo memoryInfo = new OverridableSystemMemoryInfo( + List.of("-Da=b", "-Des.total_memory_bytes=987654321", "-Dx=y"), + fallbackSystemMemoryInfo() + ); + assertThat(memoryInfo.availableSystemMemory(), is(987654321L)); + } + + public void testMultipleValidOverridesInList() throws SystemMemoryInfoException { + final SystemMemoryInfo memoryInfo = new OverridableSystemMemoryInfo( + List.of("-Des.total_memory_bytes=123456789", "-Da=b", "-Des.total_memory_bytes=987654321", "-Dx=y"), + fallbackSystemMemoryInfo() + ); + assertThat(memoryInfo.availableSystemMemory(), is(987654321L)); + } + + public void testNegativeOverride() throws SystemMemoryInfoException { + final SystemMemoryInfo memoryInfo = new OverridableSystemMemoryInfo( + List.of("-Da=b", "-Des.total_memory_bytes=-123", "-Dx=y"), + fallbackSystemMemoryInfo() + ); + try { + memoryInfo.availableSystemMemory(); + fail("expected to fail"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), is("Negative memory size specified in [-Des.total_memory_bytes=-123]")); + } + } + + public void testUnparsableOverride() throws SystemMemoryInfoException { + final SystemMemoryInfo memoryInfo = new OverridableSystemMemoryInfo( + List.of("-Da=b", "-Des.total_memory_bytes=invalid", "-Dx=y"), + fallbackSystemMemoryInfo() + ); + try { + memoryInfo.availableSystemMemory(); + fail("expected to fail"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), is("Unable to parse number of bytes from [-Des.total_memory_bytes=invalid]")); + } + } + + private static SystemMemoryInfo fallbackSystemMemoryInfo() { + return () -> FALLBACK; + } +} diff --git a/docs/changelog/78750.yaml b/docs/changelog/78750.yaml new file mode 100644 index 0000000000000..c5f06c8783460 --- /dev/null +++ b/docs/changelog/78750.yaml @@ -0,0 +1,6 @@ +pr: 78750 +summary: Allow total memory to be overridden +area: Packaging +type: enhancement +issues: + - 65905 diff --git a/docs/reference/cluster/nodes-stats.asciidoc b/docs/reference/cluster/nodes-stats.asciidoc index a2ba22578275f..7e15acb99d4bd 100644 --- a/docs/reference/cluster/nodes-stats.asciidoc +++ b/docs/reference/cluster/nodes-stats.asciidoc @@ -1036,6 +1036,18 @@ Total amount of physical memory. (integer) Total amount of physical memory in bytes. +`adjusted_total`:: +(<>) +If the amount of physical memory has been overridden using the `es.total_memory_bytes` +system property then this reports the overridden value. Otherwise it reports the same +value as `total`. + +`adjusted_total_in_bytes`:: +(integer) +If the amount of physical memory has been overridden using the `es.total_memory_bytes` +system property then this reports the overridden value in bytes. Otherwise it reports +the same value as `total_in_bytes`. + `free`:: (<>) Amount of free physical memory. diff --git a/docs/reference/cluster/stats.asciidoc b/docs/reference/cluster/stats.asciidoc index e9066542b0157..209c40a3e4070 100644 --- a/docs/reference/cluster/stats.asciidoc +++ b/docs/reference/cluster/stats.asciidoc @@ -916,6 +916,18 @@ Total amount of physical memory across all selected nodes. (integer) Total amount, in bytes, of physical memory across all selected nodes. +`adjusted_total`:: +(<>) +Total amount of memory across all selected nodes, but using the value specified +using the `es.total_memory_bytes` system property instead of measured total +memory for those nodes where that system property was set. + +`adjusted_total_in_bytes`:: +(integer) +Total amount, in bytes, of memory across all selected nodes, but using the +value specified using the `es.total_memory_bytes` system property instead +of measured total memory for those nodes where that system property was set. + `free`:: (<>) Amount of free physical memory across all selected nodes. @@ -1399,6 +1411,8 @@ The API returns the following response: "mem" : { "total" : "16gb", "total_in_bytes" : 17179869184, + "adjusted_total" : "16gb", + "adjusted_total_in_bytes" : 17179869184, "free" : "78.1mb", "free_in_bytes" : 81960960, "used" : "15.9gb", diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java index 2708e4bc480c8..72f4a37ce1853 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java @@ -511,6 +511,27 @@ public void test73CustomJvmOptionsDirectoryFilesWithoutOptionsExtensionIgnored() } } + public void test74CustomJvmOptionsTotalMemoryOverride() throws Exception { + final Path heapOptions = installation.config(Paths.get("jvm.options.d", "total_memory.options")); + try { + setHeap(null); // delete default options + // Work as though total system memory is 850MB + append(heapOptions, "-Des.total_memory_bytes=891289600\n"); + + startElasticsearch(); + + final String nodesStatsResponse = makeRequest("https://localhost:9200/_nodes/stats"); + assertThat(nodesStatsResponse, containsString("\"adjusted_total_in_bytes\":891289600")); + final String nodesResponse = makeRequest("https://localhost:9200/_nodes"); + // 40% of 850MB + assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":356515840")); + + stopElasticsearch(); + } finally { + rm(heapOptions); + } + } + public void test80RelativePathConf() throws Exception { withCustomConfig(tempConf -> { append(tempConf.resolve("elasticsearch.yml"), "node.name: relative"); diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java index 6ef7a700e8ff2..7c0ab7f9a22e5 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java @@ -883,22 +883,45 @@ public void test140CgroupOsStatsAreAvailable() throws Exception { * logic sets the correct heap size, based on the container limits. */ public void test150MachineDependentHeap() throws Exception { + final List xArgs = machineDependentHeapTest("942m", List.of()); + + // This is roughly 0.4 * 942 + assertThat(xArgs, hasItems("-Xms376m", "-Xmx376m")); + } + + /** + * Check that when available system memory is constrained by a total memory override as well as Docker, + * the machine-dependant heap sizing logic sets the correct heap size, preferring the override to the + * container limits. + */ + public void test151MachineDependentHeapWithSizeOverride() throws Exception { + final List xArgs = machineDependentHeapTest( + "942m", + // 799014912 = 762m + List.of("-Des.total_memory_bytes=799014912") + ); + + // This is roughly 0.4 * 762, in particular it's NOT 0.4 * 942 + assertThat(xArgs, hasItems("-Xms304m", "-Xmx304m")); + } + + private List machineDependentHeapTest(final String containerMemory, final List extraJvmOptions) throws Exception { // Start by ensuring `jvm.options` doesn't define any heap options final Path jvmOptionsPath = tempDir.resolve("jvm.options"); final Path containerJvmOptionsPath = installation.config("jvm.options"); copyFromContainer(containerJvmOptionsPath, jvmOptionsPath); - final List jvmOptions = Files.readAllLines(jvmOptionsPath) - .stream() - .filter(line -> (line.startsWith("-Xms") || line.startsWith("-Xmx")) == false) - .collect(Collectors.toList()); + final List jvmOptions = Stream.concat( + Files.readAllLines(jvmOptionsPath).stream().filter(line -> (line.startsWith("-Xms") || line.startsWith("-Xmx")) == false), + extraJvmOptions.stream() + ).collect(Collectors.toList()); Files.writeString(jvmOptionsPath, String.join("\n", jvmOptions)); // Now run the container, being explicit about the available memory runContainer( distribution(), - builder().memory("942m").volume(jvmOptionsPath, containerJvmOptionsPath).envVar("ELASTIC_PASSWORD", PASSWORD) + builder().memory(containerMemory).volume(jvmOptionsPath, containerJvmOptionsPath).envVar("ELASTIC_PASSWORD", PASSWORD) ); waitForElasticsearch(installation, "elastic", PASSWORD); @@ -913,12 +936,9 @@ public void test150MachineDependentHeap() throws Exception { final JsonNode jsonNode = new ObjectMapper().readTree(jvmArgumentsLine.get()); final String argsStr = jsonNode.get("message").textValue(); - final List xArgs = Arrays.stream(argsStr.substring(1, argsStr.length() - 1).split(",\\s*")) + return Arrays.stream(argsStr.substring(1, argsStr.length() - 1).split(",\\s*")) .filter(arg -> arg.startsWith("-X")) .collect(Collectors.toList()); - - // This is roughly 0.4 * 942 - assertThat(xArgs, hasItems("-Xms376m", "-Xmx376m")); } /** diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/PackageTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/PackageTests.java index 096e01b1f737f..4047277f0b1ca 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/PackageTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/PackageTests.java @@ -237,6 +237,34 @@ public void test70RestartServer() throws Exception { } } + public void test71JvmOptionsTotalMemoryOverride() throws Exception { + try { + install(); + assertPathsExist(installation.envFile); + setHeap(null); + + // Recreate file realm users that have been deleted in earlier tests + setFileSuperuser("test_superuser", "test_superuser_password"); + + withCustomConfig(tempConf -> { + // Work as though total system memory is 850MB + append(installation.envFile, "ES_JAVA_OPTS=\"-Des.total_memory_bytes=891289600\""); + + startElasticsearch(); + + final String nodesStatsResponse = makeRequest("https://localhost:9200/_nodes/stats"); + assertThat(nodesStatsResponse, containsString("\"adjusted_total_in_bytes\":891289600")); + + // 40% of 850MB + assertThat(sh.run("ps auwwx").stdout, containsString("-Xms340m -Xmx340m")); + + stopElasticsearch(); + }); + } finally { + cleanup(); + } + } + public void test72TestRuntimeDirectory() throws Exception { try { install(); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java index 29def459a3724..31b009a08b37b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java @@ -266,20 +266,26 @@ private OsStats(List nodeInfos, List nodeStatsList) { this.allocatedProcessors = allocatedProcessors; long totalMemory = 0; + long adjustedTotalMemory = 0; long freeMemory = 0; for (NodeStats nodeStats : nodeStatsList) { if (nodeStats.getOs() != null) { - long total = nodeStats.getOs().getMem().getTotal().getBytes(); + org.elasticsearch.monitor.os.OsStats.Mem mem = nodeStats.getOs().getMem(); + long total = mem.getTotal().getBytes(); if (total > 0) { totalMemory += total; } - long free = nodeStats.getOs().getMem().getFree().getBytes(); + long adjustedTotal = mem.getAdjustedTotal().getBytes(); + if (adjustedTotal > 0) { + adjustedTotalMemory += adjustedTotal; + } + long free = mem.getFree().getBytes(); if (free > 0) { freeMemory += free; } } } - this.mem = new org.elasticsearch.monitor.os.OsStats.Mem(totalMemory, freeMemory); + this.mem = new org.elasticsearch.monitor.os.OsStats.Mem(totalMemory, adjustedTotalMemory, freeMemory); } public int getAvailableProcessors() { diff --git a/server/src/main/java/org/elasticsearch/monitor/os/OsProbe.java b/server/src/main/java/org/elasticsearch/monitor/os/OsProbe.java index 8ee5a945d490d..4f5e5e419f182 100644 --- a/server/src/main/java/org/elasticsearch/monitor/os/OsProbe.java +++ b/server/src/main/java/org/elasticsearch/monitor/os/OsProbe.java @@ -60,6 +60,10 @@ public class OsProbe { private static final OperatingSystemMXBean osMxBean = ManagementFactory.getOperatingSystemMXBean(); + // This property is specified without units because it also needs to be parsed by the launcher + // code, which does not have access to all the utility classes of the Elasticsearch server. + private static final String memoryOverrideProperty = System.getProperty("es.total_memory_bytes"); + private static final Method getFreePhysicalMemorySize; private static final Method getTotalPhysicalMemorySize; private static final Method getFreeSwapSpaceSize; @@ -123,6 +127,34 @@ public long getTotalPhysicalMemorySize() { } } + /** + * Returns the adjusted total amount of physical memory in bytes. + * Total memory may be overridden when some other process is running + * that is known to consume a non-negligible amount of memory. This + * is read from the "es.total_memory_bytes" system property. When + * there is no override this method returns the same value as + * {@link #getTotalPhysicalMemorySize}. + */ + public long getAdjustedTotalMemorySize() { + return Optional.ofNullable(getTotalMemoryOverride(memoryOverrideProperty)).orElse(getTotalPhysicalMemorySize()); + } + + static Long getTotalMemoryOverride(String memoryOverrideProperty) { + if (memoryOverrideProperty == null) { + return null; + } + try { + long memoryOverride = Long.parseLong(memoryOverrideProperty); + if (memoryOverride < 0) { + throw new IllegalArgumentException("Negative memory size specified in [es.total_memory_bytes]: [" + + memoryOverrideProperty + "]"); + } + return memoryOverride; + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid value for [es.total_memory_bytes]: [" + memoryOverrideProperty + "]", e); + } + } + /** * Returns the amount of free swap space in bytes. */ @@ -859,7 +891,7 @@ OsStats.Cgroup getCgroup(boolean isLinux) { public OsStats osStats() { final OsStats.Cpu cpu = new OsStats.Cpu(getSystemCpuPercent(), getSystemLoadAverage()); - final OsStats.Mem mem = new OsStats.Mem(getTotalPhysicalMemorySize(), getFreePhysicalMemorySize()); + final OsStats.Mem mem = new OsStats.Mem(getTotalPhysicalMemorySize(), getAdjustedTotalMemorySize(), getFreePhysicalMemorySize()); final OsStats.Swap swap = new OsStats.Swap(getTotalSwapSpaceSize(), getFreeSwapSpaceSize()); final OsStats.Cgroup cgroup = getCgroup(Constants.LINUX); return new OsStats(System.currentTimeMillis(), cpu, mem, swap, cgroup); diff --git a/server/src/main/java/org/elasticsearch/monitor/os/OsStats.java b/server/src/main/java/org/elasticsearch/monitor/os/OsStats.java index 227751b0db586..d5913c89fd165 100644 --- a/server/src/main/java/org/elasticsearch/monitor/os/OsStats.java +++ b/server/src/main/java/org/elasticsearch/monitor/os/OsStats.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -90,6 +91,8 @@ static final class Fields { static final String USED_IN_BYTES = "used_in_bytes"; static final String TOTAL = "total"; static final String TOTAL_IN_BYTES = "total_in_bytes"; + static final String ADJUSTED_TOTAL = "adjusted_total"; + static final String ADJUSTED_TOTAL_IN_BYTES = "adjusted_total_in_bytes"; static final String FREE_PERCENT = "free_percent"; static final String USED_PERCENT = "used_percent"; @@ -237,25 +240,66 @@ public static class Mem implements Writeable, ToXContentFragment { private static final Logger logger = LogManager.getLogger(Mem.class); private final long total; + private final long adjustedTotal; private final long free; - public Mem(long total, long free) { + public Mem(long total, long adjustedTotal, long free) { assert total >= 0 : "expected total memory to be positive, got: " + total; - assert free >= 0 : "expected free memory to be positive, got: " + total; + assert adjustedTotal >= 0 : "expected adjusted total memory to be positive, got: " + adjustedTotal; + assert free >= 0 : "expected free memory to be positive, got: " + free; + // Extra layer of protection for when assertions are disabled + if (total < 0) { + logger.error("negative total memory [{}] found in memory stats", total); + total = 0; + } + if (adjustedTotal < 0) { + logger.error("negative adjusted total memory [{}] found in memory stats", total); + adjustedTotal = 0; + } + if (free < 0) { + logger.error("negative free memory [{}] found in memory stats", total); + free = 0; + } this.total = total; + this.adjustedTotal = adjustedTotal; this.free = free; } public Mem(StreamInput in) throws IOException { - this.total = in.readLong(); + long total = in.readLong(); assert total >= 0 : "expected total memory to be positive, got: " + total; - this.free = in.readLong(); - assert free >= 0 : "expected free memory to be positive, got: " + total; + // Extra layer of protection for when assertions are disabled + if (total < 0) { + logger.error("negative total memory [{}] deserialized in memory stats", total); + total = 0; + } + this.total = total; + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + long adjustedTotal = in.readLong(); + assert adjustedTotal >= 0 : "expected adjusted total memory to be positive, got: " + adjustedTotal; + if (adjustedTotal < 0) { + logger.error("negative adjusted total memory [{}] deserialized in memory stats", adjustedTotal); + adjustedTotal = 0; + } + this.adjustedTotal = adjustedTotal; + } else { + this.adjustedTotal = total; + } + long free = in.readLong(); + assert free >= 0 : "expected free memory to be positive, got: " + free; + if (free < 0) { + logger.error("negative free memory [{}] deserialized in memory stats", free); + free = 0; + } + this.free = free; } @Override public void writeTo(StreamOutput out) throws IOException { out.writeLong(total); + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + out.writeLong(adjustedTotal); + } out.writeLong(free); } @@ -263,6 +307,10 @@ public ByteSizeValue getTotal() { return new ByteSizeValue(total); } + public ByteSizeValue getAdjustedTotal() { + return new ByteSizeValue(adjustedTotal); + } + public ByteSizeValue getUsed() { if (total == 0) { // The work in https://github.com/elastic/elasticsearch/pull/42725 established that total memory @@ -295,6 +343,7 @@ public short getFreePercent() { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(Fields.MEM); builder.humanReadableField(Fields.TOTAL_IN_BYTES, Fields.TOTAL, getTotal()); + builder.humanReadableField(Fields.ADJUSTED_TOTAL_IN_BYTES, Fields.ADJUSTED_TOTAL, getAdjustedTotal()); builder.humanReadableField(Fields.FREE_IN_BYTES, Fields.FREE, getFree()); builder.humanReadableField(Fields.USED_IN_BYTES, Fields.USED, getUsed()); builder.field(Fields.FREE_PERCENT, getFreePercent()); diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java index 07917ae30cb4f..4272dfa77f2cc 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java @@ -434,7 +434,7 @@ public static NodeStats createNodeStats() { long memTotal = randomNonNegativeLong(); long swapTotal = randomNonNegativeLong(); osStats = new OsStats(System.currentTimeMillis(), new OsStats.Cpu(randomShort(), loadAverages), - new OsStats.Mem(memTotal, randomLongBetween(0, memTotal)), + new OsStats.Mem(memTotal, randomLongBetween(0, memTotal), randomLongBetween(0, memTotal)), new OsStats.Swap(swapTotal, randomLongBetween(0, swapTotal)), new OsStats.Cgroup( randomAlphaOfLength(8), diff --git a/server/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java b/server/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java index 43c94b6154069..23091169f94e1 100644 --- a/server/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java +++ b/server/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java @@ -307,6 +307,19 @@ public void testGetTotalMemFromProcMeminfo() throws Exception { assertThat(probe.getTotalMemFromProcMeminfo(), equalTo(memTotalInKb * 1024L)); } + public void testTotalMemoryOverride() { + assertThat(OsProbe.getTotalMemoryOverride("123456789"), is(123456789L)); + assertThat(OsProbe.getTotalMemoryOverride("123456789123456789"), is(123456789123456789L)); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> OsProbe.getTotalMemoryOverride("-1")); + assertThat(e.getMessage(), is("Negative memory size specified in [es.total_memory_bytes]: [-1]")); + e = expectThrows(IllegalArgumentException.class, () -> OsProbe.getTotalMemoryOverride("abc")); + assertThat(e.getMessage(), is("Invalid value for [es.total_memory_bytes]: [abc]")); + // Although numeric, this value overflows long. This won't be a problem in practice for sensible + // overrides, as it will be a very long time before machines have more than 8 exabytes of RAM. + e = expectThrows(IllegalArgumentException.class, () -> OsProbe.getTotalMemoryOverride("123456789123456789123456789")); + assertThat(e.getMessage(), is("Invalid value for [es.total_memory_bytes]: [123456789123456789123456789]")); + } + public void testGetTotalMemoryOnDebian8() throws Exception { // tests the workaround for JDK bug on debian8: https://github.com/elastic/elasticsearch/issues/67089#issuecomment-756114654 final OsProbe osProbe = new OsProbe(); diff --git a/server/src/test/java/org/elasticsearch/monitor/os/OsStatsTests.java b/server/src/test/java/org/elasticsearch/monitor/os/OsStatsTests.java index e9b0705c69f64..a2ee845d4e69c 100644 --- a/server/src/test/java/org/elasticsearch/monitor/os/OsStatsTests.java +++ b/server/src/test/java/org/elasticsearch/monitor/os/OsStatsTests.java @@ -26,7 +26,7 @@ public void testSerialization() throws IOException { } OsStats.Cpu cpu = new OsStats.Cpu(randomShort(), loadAverages); long memTotal = randomNonNegativeLong(); - OsStats.Mem mem = new OsStats.Mem(memTotal, randomLongBetween(0, memTotal)); + OsStats.Mem mem = new OsStats.Mem(memTotal, randomLongBetween(0, memTotal), randomLongBetween(0, memTotal)); long swapTotal = randomNonNegativeLong(); OsStats.Swap swap = new OsStats.Swap(swapTotal, randomLongBetween(0, swapTotal)); OsStats.Cgroup cgroup = new OsStats.Cgroup( @@ -73,7 +73,7 @@ public void testSerialization() throws IOException { } public void testGetUsedMemoryWithZeroTotal() { - OsStats.Mem mem = new OsStats.Mem(0, randomNonNegativeLong()); + OsStats.Mem mem = new OsStats.Mem(0, 0, randomNonNegativeLong()); assertThat(mem.getUsed().getBytes(), equalTo(0L)); } diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/memory/AutoscalingMemoryInfoServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/memory/AutoscalingMemoryInfoServiceTests.java index ce6ba2557f6f7..fb0a073c02e86 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/memory/AutoscalingMemoryInfoServiceTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/memory/AutoscalingMemoryInfoServiceTests.java @@ -347,7 +347,7 @@ private static NodeStats statsForNode(DiscoveryNode node, long memory) { OsStats osStats = new OsStats( randomNonNegativeLong(), new OsStats.Cpu(randomShort(), null), - new OsStats.Mem(memory, randomLongBetween(0, memory)), + new OsStats.Mem(memory, memory, randomLongBetween(0, memory)), new OsStats.Swap(randomNonNegativeLong(), randomNonNegativeLong()), null ); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 6951ea1f759c5..e93e83133d1c0 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -59,7 +59,6 @@ import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.monitor.jvm.JvmInfo; import org.elasticsearch.monitor.os.OsProbe; -import org.elasticsearch.monitor.os.OsStats; import org.elasticsearch.persistent.PersistentTaskParams; import org.elasticsearch.persistent.PersistentTaskState; import org.elasticsearch.persistent.PersistentTasksExecutor; @@ -415,7 +414,6 @@ import org.elasticsearch.xpack.ml.utils.persistence.ResultsPersisterService; import java.io.IOException; -import java.math.BigInteger; import java.nio.file.Path; import java.time.Clock; import java.util.ArrayList; @@ -673,7 +671,7 @@ public Settings additionalSettings() { addMlNodeAttribute(additionalSettings, maxOpenJobsPerNodeNodeAttrName, String.valueOf(MAX_OPEN_JOBS_PER_NODE.get(settings))); addMlNodeAttribute(additionalSettings, machineMemoryAttrName, - Long.toString(machineMemoryFromStats(OsProbe.getInstance().osStats()))); + Long.toString(OsProbe.getInstance().osStats().getMem().getAdjustedTotal().getBytes())); addMlNodeAttribute(additionalSettings, jvmSizeAttrName, Long.toString(Runtime.getRuntime().maxMemory())); // This is not used in v7 and higher, but users are still prevented from setting it directly to avoid confusion disallowMlNodeAttributes(mlEnabledNodeAttrName); @@ -1283,27 +1281,6 @@ public static boolean allTemplatesInstalled(ClusterState clusterState) { return allPresent; } - /** - * Find the memory size (in bytes) of the machine this node is running on. - * Takes container limits (as used by Docker for example) into account. - */ - static long machineMemoryFromStats(OsStats stats) { - long mem = stats.getMem().getTotal().getBytes(); - OsStats.Cgroup cgroup = stats.getCgroup(); - if (cgroup != null) { - String containerLimitStr = cgroup.getMemoryLimitInBytes(); - if (containerLimitStr != null && containerLimitStr.equals("max") == false) { - BigInteger containerLimit = new BigInteger(containerLimitStr); - if ((containerLimit.compareTo(BigInteger.valueOf(mem)) < 0 && containerLimit.compareTo(BigInteger.ZERO) > 0) - // mem <= 0 means the value couldn't be obtained for some reason - || (mem <= 0 && containerLimit.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) < 0)) { - mem = containerLimit.longValue(); - } - } - } - return mem; - } - @Override public List getNamedXContent() { List namedXContent = new ArrayList<>(); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTests.java index 46a0ff61a21b0..3785d6f89262c 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTests.java @@ -16,14 +16,12 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.license.XPackLicenseState; -import org.elasticsearch.monitor.os.OsStats; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.action.SetUpgradeModeAction; -import java.io.IOException; import java.util.Collections; import java.util.Map; @@ -178,42 +176,6 @@ public void testNoAttributes_givenClash() { "it is reserved for machine learning. If your intention was to customize machine learning, set the [xpack.ml.")); } - public void testMachineMemory_givenStatsFailure() throws IOException { - OsStats stats = mock(OsStats.class); - when(stats.getMem()).thenReturn(new OsStats.Mem(0, 0)); - assertEquals(0L, MachineLearning.machineMemoryFromStats(stats)); - } - - public void testMachineMemory_givenNoCgroup() throws IOException { - OsStats stats = mock(OsStats.class); - when(stats.getMem()).thenReturn(new OsStats.Mem(10_737_418_240L, 5_368_709_120L)); - assertEquals(10_737_418_240L, MachineLearning.machineMemoryFromStats(stats)); - } - - public void testMachineMemory_givenCgroupNullLimit() throws IOException { - OsStats stats = mock(OsStats.class); - when(stats.getMem()).thenReturn(new OsStats.Mem(10_737_418_240L, 5_368_709_120L)); - when(stats.getCgroup()).thenReturn(new OsStats.Cgroup("a", 1, "b", 2, 3, - new OsStats.Cgroup.CpuStat(4, 5, 6), null, null, null)); - assertEquals(10_737_418_240L, MachineLearning.machineMemoryFromStats(stats)); - } - - public void testMachineMemory_givenCgroupNoLimit() throws IOException { - OsStats stats = mock(OsStats.class); - when(stats.getMem()).thenReturn(new OsStats.Mem(10_737_418_240L, 5_368_709_120L)); - when(stats.getCgroup()).thenReturn(new OsStats.Cgroup("a", 1, "b", 2, 3, - new OsStats.Cgroup.CpuStat(4, 5, 6), "c", "18446744073709551615", "4796416")); - assertEquals(10_737_418_240L, MachineLearning.machineMemoryFromStats(stats)); - } - - public void testMachineMemory_givenCgroupLowLimit() throws IOException { - OsStats stats = mock(OsStats.class); - when(stats.getMem()).thenReturn(new OsStats.Mem(10_737_418_240L, 5_368_709_120L)); - when(stats.getCgroup()).thenReturn(new OsStats.Cgroup("a", 1, "b", 2, 3, - new OsStats.Cgroup.CpuStat(4, 5, 6), "c", "7516192768", "4796416")); - assertEquals(7_516_192_768L, MachineLearning.machineMemoryFromStats(stats)); - } - private MachineLearning createMachineLearning(Settings settings) { XPackLicenseState licenseState = mock(XPackLicenseState.class); diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java index 90ab394b0647f..d166d37bbf043 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java @@ -297,7 +297,7 @@ public void testToXContent() throws IOException { final OsStats mockOsStats = mock(OsStats.class); when(mockNodeStats.getOs()).thenReturn(mockOsStats); - when(mockOsStats.getMem()).thenReturn(new OsStats.Mem(100, 79)); + when(mockOsStats.getMem()).thenReturn(new OsStats.Mem(100, 99, 79)); final ProcessStats mockProcessStats = mock(ProcessStats.class); when(mockNodeStats.getProcess()).thenReturn(mockProcessStats); @@ -517,6 +517,7 @@ public void testToXContent() throws IOException { + " ]," + " \"mem\": {" + " \"total_in_bytes\": 100," + + " \"adjusted_total_in_bytes\": 99," + " \"free_in_bytes\": 79," + " \"used_in_bytes\": 21," + " \"free_percent\": 79," diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsMonitoringDocTests.java index 65beae6eb8aad..ad4d99347c069 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsMonitoringDocTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsMonitoringDocTests.java @@ -341,7 +341,7 @@ private static NodeStats mockNodeStats() { final OsStats.Cgroup osCgroup = new OsStats.Cgroup("_cpu_acct_ctrl_group", ++iota, "_cpu_ctrl_group", ++iota, ++iota, osCpuStat, "_memory_ctrl_group", "2000000000", "1000000000"); - final OsStats.Mem osMem = new OsStats.Mem(0, 0); + final OsStats.Mem osMem = new OsStats.Mem(0, 0, 0); final OsStats.Swap osSwap = new OsStats.Swap(0, 0); final OsStats os = new OsStats(no, osCpu, osMem, osSwap, osCgroup);