Record more detailed HTTP stats

docs/changelog/99852.yaml

@@ -0,0 +1,5 @@
+pr: 99852
+summary: Record more detailed HTTP stats
+area: Network
+type: enhancement
+issues: []

qa/smoke-test-multinode/src/yamlRestTest/resources/rest-api-spec/test/smoke_test_multinode/60_http_stats.yml

@@ -0,0 +1,22 @@
+# This test needs multiple nodes, because a single-node cluster does not send any transport actions so these stats are empty
+---
+"http stats":
+  - skip:
+      features: [arbitrary_key]
+
+  - do:
+      search:
+        index: "*"
+        body:
+          query:
+            match_all: {}
+
+  - do:
+      nodes.stats:
+        metric: [ http ]
+        human: true
+
+  - set:
+      nodes._arbitrary_key_: node_id
+
+  - is_true: "nodes.$node_id.http.routes./_cat/nodes"

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -145,6 +145,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion WAIT_FOR_CLUSTER_STATE_IN_RECOVERY_ADDED = def(8_502_00_0);
     public static final TransportVersion RECOVERY_COMMIT_TOO_NEW_EXCEPTION_ADDED = def(8_503_00_0);
     public static final TransportVersion NODE_INFO_COMPONENT_VERSIONS_ADDED = def(8_504_00_0);
+    public static final TransportVersion NODE_STATS_HTTP_ROUTE_STATS_ADDED = def(8_505_00_0);
     /*
      * STOP! READ THIS FIRST! No, really,
      *        ____ _____ ___  ____  _        ____  _____    _    ____    _____ _   _ ___ ____    _____ ___ ____  ____ _____ _

server/src/main/java/org/elasticsearch/common/path/PathTrie.java

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.common.path;
 
+import org.elasticsearch.common.collect.Iterators;
+
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -267,6 +269,15 @@ private void put(Map<String, String> params, TrieNode node, String value) {
             }
         }
 
+        Iterator<T> allNodeValues() {
+            final Iterator<T> childrenIterator = Iterators.flatMap(children.values().iterator(), TrieNode::allNodeValues);
+            if (value == null) {
+                return childrenIterator;
+            } else {
+                return Iterators.concat(Iterators.single(value), childrenIterator);
+            }
+        }
+
         @Override
         public String toString() {
             return key;
@@ -366,4 +377,8 @@ public T next() {
             }
         };
     }
+
+    public Iterator<T> allNodeValues() {
+        return Iterators.concat(Iterators.single(rootValue), root.allNodeValues());
+    }
 }

server/src/main/java/org/elasticsearch/http/AbstractHttpServerTransport.java

@@ -168,7 +168,12 @@ public HttpInfo info() {
 
     @Override
     public HttpStats stats() {
-        return new HttpStats(httpChannels.size(), totalChannelsAccepted.get(), httpClientStatsTracker.getClientStats());
+        return new HttpStats(
+            httpChannels.size(),
+            totalChannelsAccepted.get(),
+            httpClientStatsTracker.getClientStats(),
+            dispatcher.getStats()
+        );
     }
 
     protected void bindServer() {

server/src/main/java/org/elasticsearch/http/HttpRouteStats.java

@@ -0,0 +1,90 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.http;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+
+public record HttpRouteStats(
+    long requestCount,
+    long totalRequestSize,
+    long[] requestSizeHistogram,
+    long responseCount,
+    long totalResponseSize,
+    long[] responseSizeHistogram
+) implements Writeable, ToXContentObject {
+
+    public HttpRouteStats(StreamInput in) throws IOException {
+        this(in.readVLong(), in.readVLong(), in.readVLongArray(), in.readVLong(), in.readVLong(), in.readVLongArray());
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+
+        builder.startObject("requests");
+        builder.field("count", requestCount);
+        builder.humanReadableField("total_size_in_bytes", "total_size", ByteSizeValue.ofBytes(totalRequestSize));
+        histogramToXContent(builder, requestSizeHistogram);
+        builder.endObject();
+
+        builder.startObject("responses");
+        builder.field("count", responseCount);
+        builder.humanReadableField("total_size_in_bytes", "total_size", ByteSizeValue.ofBytes(totalResponseSize));
+        histogramToXContent(builder, responseSizeHistogram);
+        builder.endObject();
+
+        return builder.endObject();
+    }
+
+    static void histogramToXContent(XContentBuilder builder, long[] sizeHistogram) throws IOException {
+        final int[] bucketBounds = HttpRouteStatsTracker.getBucketUpperBounds();
+        assert sizeHistogram.length == bucketBounds.length + 1;
+        builder.startArray("histogram");
+
+        int firstBucket = 0;
+        long remainingCount = 0L;
+        for (int i = 0; i < sizeHistogram.length; i++) {
+            if (remainingCount == 0) {
+                firstBucket = i;
+            }
+            remainingCount += sizeHistogram[i];
+        }
+
+        for (int i = firstBucket; i < sizeHistogram.length && 0 < remainingCount; i++) {
+            builder.startObject();
+            if (i > 0) {
+                builder.humanReadableField("ge_bytes", "ge", ByteSizeValue.ofBytes(bucketBounds[i - 1]));
+            }
+            if (i < bucketBounds.length) {
+                builder.humanReadableField("lt_bytes", "lt", ByteSizeValue.ofBytes(bucketBounds[i]));
+            }
+            builder.field("count", sizeHistogram[i]);
+            builder.endObject();
+            remainingCount -= sizeHistogram[i];
+        }
+        builder.endArray();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVLong(requestCount);
+        out.writeVLong(totalRequestSize);
+        out.writeVLongArray(requestSizeHistogram);
+        out.writeVLong(responseCount);
+        out.writeVLong(totalResponseSize);
+        out.writeVLongArray(responseSizeHistogram);
+    }
+}

server/src/main/java/org/elasticsearch/http/HttpRouteStatsTracker.java

@@ -0,0 +1,89 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.http;
+
+import java.util.concurrent.atomic.AtomicLongArray;
+import java.util.concurrent.atomic.LongAdder;
+
+public class HttpRouteStatsTracker {
+
+    /*
+     * default http.max_content_length is 100 MB so that the last histogram bucket is > 64MB (2^26)
+     */
+
+    public static int[] getBucketUpperBounds() {
+        var bounds = new int[27];
+        for (int i = 0; i < bounds.length; i++) {
+            bounds[i] = 1 << i;
+        }
+        return bounds;
+    }
+
+    private static final int BUCKET_COUNT = getBucketUpperBounds().length + 1;
+
+    private static final long LAST_BUCKET_LOWER_BOUND = getBucketUpperBounds()[BUCKET_COUNT - 2];
+
+    private record StatsTracker(LongAdder count, LongAdder totalSize, AtomicLongArray histogram) {
+        StatsTracker {
+            assert count.longValue() == 0L;
+            assert totalSize.longValue() == 0L;
+            assert histogram.length() == BUCKET_COUNT;
+        }
+
+        StatsTracker() {
+            this(new LongAdder(), new LongAdder(), new AtomicLongArray(BUCKET_COUNT));
+        }
+
+        void addStats(int contentLength) {
+            count().increment();
+            totalSize().add(contentLength);
+            histogram().incrementAndGet(bucket(contentLength));
+        }
+
+        long[] getHistogram() {
+            long[] histogramCopy = new long[BUCKET_COUNT];
+            for (int i = 0; i < BUCKET_COUNT; i++) {
+                histogramCopy[i] = histogram().get(i);
+            }
+            return histogramCopy;
+        }
+    }
+
+    private static int bucket(int contentLength) {
+        if (contentLength <= 0) {
+            return 0;
+        } else if (LAST_BUCKET_LOWER_BOUND <= contentLength) {
+            return BUCKET_COUNT - 1;
+        } else {
+            return Integer.SIZE - Integer.numberOfLeadingZeros(contentLength);
+        }
+    }
+
+    private final StatsTracker requestStats = new StatsTracker();
+    private final StatsTracker responseStats = new StatsTracker();
+
+    public void addRequestStats(int contentLength) {
+        requestStats.addStats(contentLength);
+    }
+
+    public void addResponseStats(int contentLength) {
+        responseStats.addStats(contentLength);
+    }
+
+    public HttpRouteStats getStats() {
+        return new HttpRouteStats(
+            requestStats.count().longValue(),
+            requestStats.totalSize().longValue(),
+            requestStats.getHistogram(),
+            responseStats.count().longValue(),
+            responseStats.totalSize().longValue(),
+            responseStats.getHistogram()
+        );
+    }
+}

server/src/main/java/org/elasticsearch/http/HttpServerTransport.java

@@ -15,6 +15,8 @@
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestRequest;
 
+import java.util.Map;
+
 public interface HttpServerTransport extends LifecycleComponent, ReportingService<HttpInfo> {
 
     String HTTP_PROFILE_NAME = ".http";
@@ -52,5 +54,8 @@ interface Dispatcher {
          */
         void dispatchBadRequest(RestChannel channel, ThreadContext threadContext, Throwable cause);
 
+        default Map<String, HttpRouteStats> getStats() {
+            return Map.of();
+        }
     }
 }

server/src/main/java/org/elasticsearch/http/HttpStats.java

@@ -20,25 +20,39 @@
 import java.io.IOException;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Stream;
 
-public record HttpStats(long serverOpen, long totalOpen, List<ClientStats> clientStats) implements Writeable, ChunkedToXContent {
+import static org.elasticsearch.TransportVersions.NODE_STATS_HTTP_ROUTE_STATS_ADDED;
 
-    public static final HttpStats IDENTITY = new HttpStats(0, 0, List.of());
+public record HttpStats(long serverOpen, long totalOpen, List<ClientStats> clientStats, Map<String, HttpRouteStats> httpRouteStats)
+    implements
+        Writeable,
+        ChunkedToXContent {
+
+    public static final HttpStats IDENTITY = new HttpStats(0, 0, List.of(), Map.of());
 
     public HttpStats(long serverOpen, long totalOpened) {
-        this(serverOpen, totalOpened, List.of());
+        this(serverOpen, totalOpened, List.of(), Map.of());
     }
 
     public HttpStats(StreamInput in) throws IOException {
-        this(in.readVLong(), in.readVLong(), in.readCollectionAsList(ClientStats::new));
+        this(
+            in.readVLong(),
+            in.readVLong(),
+            in.readCollectionAsList(ClientStats::new),
+            in.getTransportVersion().onOrAfter(NODE_STATS_HTTP_ROUTE_STATS_ADDED) ? in.readMap(HttpRouteStats::new) : Map.of()
+        );
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeVLong(serverOpen);
         out.writeVLong(totalOpen);
         out.writeCollection(clientStats);
+        if (out.getTransportVersion().onOrAfter(NODE_STATS_HTTP_ROUTE_STATS_ADDED)) {
+            out.writeMap(httpRouteStats, StreamOutput::writeWriteable);
+        }
     }
 
     public long getServerOpen() {
@@ -57,7 +71,8 @@ public static HttpStats merge(HttpStats first, HttpStats second) {
         return new HttpStats(
             first.serverOpen + second.serverOpen,
             first.totalOpen + second.totalOpen,
-            Stream.concat(first.clientStats.stream(), second.clientStats.stream()).toList()
+            Stream.concat(first.clientStats.stream(), second.clientStats.stream()).toList(),
+            Map.of() // TODO: merge
         );
     }
 
@@ -78,6 +93,7 @@ static final class Fields {
         static final String CLIENT_REQUEST_SIZE_BYTES = "request_size_bytes";
         static final String CLIENT_FORWARDED_FOR = "x_forwarded_for";
         static final String CLIENT_OPAQUE_ID = "x_opaque_id";
+        static final String ROUTES = "routes";
     }
 
     @Override
@@ -90,7 +106,24 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerP
                     .startArray(Fields.CLIENTS)
             ),
             clientStats.iterator(),
-            Iterators.single((builder, params) -> builder.endArray().endObject())
+            Iterators.single((builder, params) -> {
+                builder.endArray();
+                if (httpRouteStats.isEmpty() == false) {
+                    builder.startObject(Fields.ROUTES);
+                }
+                return builder;
+            }),
+            Iterators.map(httpRouteStats.entrySet().iterator(), entry -> (builder, params) -> {
+                builder.field(entry.getKey());
+                entry.getValue().toXContent(builder, params);
+                return builder;
+            }),
+            Iterators.single((builder, params) -> {
+                if (httpRouteStats.isEmpty() == false) {
+                    builder.endObject();
+                }
+                return builder.endObject();
+            })
         );
     }