Add other time accounting in HotThreads (#79392)

Currently, the HotThreads CPU report contains only
the CPU cycles that a thread executes, excluding time
spent in I/O or the JVM runtime (e.g. GC). This PR
introduces the 'other' time accounting to augment the
CPU time for better understanding of user visible time
impact.

server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequest.java

@@ -8,6 +8,7 @@
 
 package org.elasticsearch.action.admin.cluster.node.hotthreads;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.support.nodes.BaseNodesRequest;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -22,6 +23,7 @@ public class NodesHotThreadsRequest extends BaseNodesRequest<NodesHotThreadsRequ
 
     int threads = 3;
     HotThreads.ReportType type = HotThreads.ReportType.CPU;
+    HotThreads.SortOrder sortOrder = HotThreads.SortOrder.TOTAL;
     TimeValue interval = new TimeValue(500, TimeUnit.MILLISECONDS);
     int snapshots = 10;
     boolean ignoreIdleThreads = true;
@@ -34,6 +36,9 @@ public NodesHotThreadsRequest(StreamInput in) throws IOException {
         type = HotThreads.ReportType.of(in.readString());
         interval = in.readTimeValue();
         snapshots = in.readInt();
+        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+            sortOrder = HotThreads.SortOrder.of(in.readString());
+        }
     }
 
     /**
@@ -78,6 +83,15 @@ public HotThreads.ReportType type() {
         return this.type;
     }
 
+    public NodesHotThreadsRequest sortOrder(HotThreads.SortOrder sortOrder) {
+        this.sortOrder = sortOrder;
+        return this;
+    }
+
+    public HotThreads.SortOrder sortOrder() {
+        return this.sortOrder;
+    }
+
     public NodesHotThreadsRequest interval(TimeValue interval) {
         this.interval = interval;
         return this;
@@ -104,5 +118,8 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(type.getTypeValue());
         out.writeTimeValue(interval);
         out.writeInt(snapshots);
+        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+            out.writeString(sortOrder.getOrderValue());
+        }
     }
 }

server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequestBuilder.java

@@ -35,6 +35,11 @@ public NodesHotThreadsRequestBuilder setType(HotThreads.ReportType type) {
         return this;
     }
 
+    public NodesHotThreadsRequestBuilder setSortOrder(HotThreads.SortOrder sortOrder) {
+        request.sortOrder(sortOrder);
+        return this;
+    }
+
     public NodesHotThreadsRequestBuilder setInterval(TimeValue interval) {
         request.interval(interval);
         return this;

server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java

@@ -59,6 +59,7 @@ protected NodeHotThreads nodeOperation(NodeRequest request, Task task) {
         HotThreads hotThreads = new HotThreads()
                 .busiestThreads(request.request.threads)
                 .type(request.request.type)
+                .sortOrder(request.request.sortOrder)
                 .interval(request.request.interval)
                 .threadElementsSnapshotCount(request.request.snapshots)
                 .ignoreIdleThreads(request.request.ignoreIdleThreads);

server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java

@@ -31,6 +31,7 @@
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.jdk.JarHell;
+import org.elasticsearch.monitor.jvm.HotThreads;
 import org.elasticsearch.monitor.jvm.JvmInfo;
 import org.elasticsearch.monitor.os.OsProbe;
 import org.elasticsearch.monitor.process.ProcessProbe;
@@ -156,6 +157,7 @@ static void initializeProbes() {
         ProcessProbe.getInstance();
         OsProbe.getInstance();
         JvmInfo.jvmInfo();
+        HotThreads.initializeRuntimeMonitoring();
     }
 
     private void setup(boolean addShutdownHook, Environment environment) throws BootstrapException {

server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java

@@ -91,9 +91,13 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
         String[] nodesIds = Strings.splitStringByCommaToArray(request.param("nodeId"));
         NodesHotThreadsRequest nodesHotThreadsRequest = new NodesHotThreadsRequest(nodesIds);
         nodesHotThreadsRequest.threads(request.paramAsInt("threads", nodesHotThreadsRequest.threads()));
-        nodesHotThreadsRequest.ignoreIdleThreads(request.paramAsBoolean("ignore_idle_threads", nodesHotThreadsRequest.ignoreIdleThreads()));
+        nodesHotThreadsRequest.ignoreIdleThreads(
+            request.paramAsBoolean("ignore_idle_threads", nodesHotThreadsRequest.ignoreIdleThreads()));
         nodesHotThreadsRequest.type(HotThreads.ReportType.of(request.param("type", nodesHotThreadsRequest.type().getTypeValue())));
-        nodesHotThreadsRequest.interval(TimeValue.parseTimeValue(request.param("interval"), nodesHotThreadsRequest.interval(), "interval"));
+        nodesHotThreadsRequest.sortOrder(
+            HotThreads.SortOrder.of(request.param("sort", nodesHotThreadsRequest.sortOrder().getOrderValue())));
+        nodesHotThreadsRequest.interval(
+            TimeValue.parseTimeValue(request.param("interval"), nodesHotThreadsRequest.interval(), "interval"));
         nodesHotThreadsRequest.snapshots(request.paramAsInt("snapshots", nodesHotThreadsRequest.snapshots()));
         nodesHotThreadsRequest.timeout(request.param("timeout"));
         return channel -> client.admin().cluster().nodesHotThreads(

server/src/main/resources/org/elasticsearch/bootstrap/security.policy

@@ -22,8 +22,6 @@ grant codeBase "${codebase.elasticsearch-secure-sm}" {
 grant codeBase "${codebase.elasticsearch}" {
   // needed for loading plugins which may expect the context class loader to be set
   permission java.lang.RuntimePermission "setContextClassLoader";
-  // needed by HotThreads to enable wait/block time accounting on demand
-  permission java.lang.management.ManagementPermission "control";
 };
 
 //// Very special jar permissions:

server/src/test/java/org/elasticsearch/action/admin/cluster/hotthreads/NodesHotThreadsRequestTests.java

@@ -55,7 +55,7 @@ public void testBWCSerialization() throws IOException {
             out.setVersion(previous);
             request.writeTo(out);
             try (StreamInput in = out.bytes().streamInput()) {
-                in.setVersion(latest);
+                in.setVersion(previous);
                 NodesHotThreadsRequest deserialized = new NodesHotThreadsRequest(in);
                 assertEquals(request.threads(), deserialized.threads());
                 assertEquals(request.ignoreIdleThreads(), deserialized.ignoreIdleThreads());