add response time

qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/DanglingIndicesRestIT.java

@@ -30,7 +30,6 @@
 import static org.elasticsearch.cluster.metadata.IndexGraveyard.SETTING_MAX_TOMBSTONES;
 import static org.elasticsearch.indices.IndicesService.WRITE_DANGLING_INDICES_INFO_SETTING;
 import static org.elasticsearch.rest.RestStatus.ACCEPTED;
-import static org.elasticsearch.rest.RestStatus.OK;
 import static org.elasticsearch.test.XContentTestUtils.createJsonMapView;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
@@ -184,10 +183,6 @@ private List<String> listDanglingIndexIds() throws IOException {
         return danglingIndexIds;
     }
 
-    private void assertOK(Response response) {
-        assertThat(response.getStatusLine().getStatusCode(), equalTo(OK.getStatus()));
-    }
-
     /**
      * Given a node name, finds the corresponding node ID.
      */

qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpSmokeTestCase.java

@@ -7,6 +7,7 @@
  */
 package org.elasticsearch.http;
 
+import org.elasticsearch.client.Response;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
@@ -17,6 +18,9 @@
 import java.util.Collection;
 import java.util.List;
 
+import static org.hamcrest.Matchers.anyOf;
+import static org.hamcrest.Matchers.equalTo;
+
 public abstract class HttpSmokeTestCase extends ESIntegTestCase {
 
     @Override
@@ -42,4 +46,8 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
     protected boolean ignoreExternalCluster() {
         return true;
     }
+
+    public static void assertOK(Response response) {
+        assertThat(response.getStatusLine().getStatusCode(), anyOf(equalTo(200), equalTo(201)));
+    }
 }

qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpStatsIT.java

@@ -0,0 +1,86 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.http;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.XContentTestUtils;
+import org.elasticsearch.xcontent.json.JsonXContent;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.aMapWithSize;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.notNullValue;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 1, numClientNodes = 0)
+public class HttpStatsIT extends HttpSmokeTestCase {
+
+    @SuppressWarnings("unchecked")
+    public void testHttpStats() throws IOException {
+        // basic request
+        final RestClient restClient = getRestClient();
+        assertOK(restClient.performRequest(new Request("GET", "/")));
+        // request with body and URL placeholder
+        final Request searchRequest = new Request("GET", "*/_search");
+        searchRequest.setJsonEntity("""
+            {"query":{"match_all":{}}}""");
+        assertOK(restClient.performRequest(searchRequest));
+        // chunked response
+        assertOK(restClient.performRequest(new Request("GET", "/_cluster/state")));
+        // chunked text response
+        assertOK(restClient.performRequest(new Request("GET", "/_cat/nodes")));
+
+        final Response response = restClient.performRequest(new Request("GET", "/_nodes/stats/http"));
+        assertOK(response);
+
+        final Map<String, Object> responseMap = XContentHelper.convertToMap(
+            JsonXContent.jsonXContent,
+            response.getEntity().getContent(),
+            false
+        );
+        final Map<String, Object> nodesMap = (Map<String, Object>) responseMap.get("nodes");
+
+        assertThat(nodesMap, aMapWithSize(1));
+        final String nodeId = nodesMap.keySet().iterator().next();
+        final XContentTestUtils.JsonMapView nodeView = new XContentTestUtils.JsonMapView((Map<String, Object>) nodesMap.get(nodeId));
+
+        final List<String> routes = List.of("/", "/_cat/nodes", "/{index}/_search", "/_cluster/state");
+
+        for (var route : routes) {
+            assertThat(nodeView.get("http.routes." + route), notNullValue());
+            assertThat(nodeView.get("http.routes." + route + ".requests.count"), equalTo(1));
+            assertThat(nodeView.get("http.routes." + route + ".requests.total_size_in_bytes"), greaterThanOrEqualTo(0));
+            assertThat(nodeView.get("http.routes." + route + ".responses.count"), equalTo(1));
+            assertThat(nodeView.get("http.routes." + route + ".responses.total_size_in_bytes"), greaterThan(1));
+            assertThat(nodeView.get("http.routes." + route + ".requests.size_histogram"), hasSize(1));
+            assertThat(nodeView.get("http.routes." + route + ".requests.size_histogram.0.count"), equalTo(1));
+            assertThat(nodeView.get("http.routes." + route + ".requests.size_histogram.0.lt_bytes"), notNullValue());
+            if (route.equals("/{index}/_search")) {
+                assertThat(nodeView.get("http.routes." + route + ".requests.size_histogram.0.ge_bytes"), notNullValue());
+            }
+            assertThat(nodeView.get("http.routes." + route + ".responses.size_histogram"), hasSize(1));
+            assertThat(nodeView.get("http.routes." + route + ".responses.size_histogram.0.count"), equalTo(1));
+            assertThat(nodeView.get("http.routes." + route + ".responses.size_histogram.0.lt_bytes"), notNullValue());
+            assertThat(nodeView.get("http.routes." + route + ".responses.size_histogram.0.ge_bytes"), notNullValue());
+            assertThat(nodeView.get("http.routes." + route + ".responses.handling_time_histogram"), hasSize(1));
+            assertThat(nodeView.get("http.routes." + route + ".responses.handling_time_histogram.0.count"), equalTo(1));
+            assertThat(nodeView.get("http.routes." + route + ".responses.handling_time_histogram.0.lt_millis"), notNullValue());
+            assertThat(nodeView.get("http.routes." + route + ".responses.handling_time_histogram.0.ge_millis"), notNullValue());
+        }
+    }
+}

server/src/main/java/org/elasticsearch/http/HttpRouteStats.java

@@ -11,6 +11,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.network.HandlingTimeTracker;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -23,11 +24,12 @@ public record HttpRouteStats(
     long[] requestSizeHistogram,
     long responseCount,
     long totalResponseSize,
-    long[] responseSizeHistogram
+    long[] responseSizeHistogram,
+    long[] responseTimeHistogram
 ) implements Writeable, ToXContentObject {
 
     public HttpRouteStats(StreamInput in) throws IOException {
-        this(in.readVLong(), in.readVLong(), in.readVLongArray(), in.readVLong(), in.readVLong(), in.readVLongArray());
+        this(in.readVLong(), in.readVLong(), in.readVLongArray(), in.readVLong(), in.readVLong(), in.readVLongArray(), in.readVLongArray());
     }
 
     @Override
@@ -37,43 +39,50 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.startObject("requests");
         builder.field("count", requestCount);
         builder.humanReadableField("total_size_in_bytes", "total_size", ByteSizeValue.ofBytes(totalRequestSize));
-        histogramToXContent(builder, requestSizeHistogram);
+        histogramToXContent(builder, "size_histogram", "bytes", requestSizeHistogram, HttpRouteStatsTracker.getBucketUpperBounds());
         builder.endObject();
 
         builder.startObject("responses");
         builder.field("count", responseCount);
         builder.humanReadableField("total_size_in_bytes", "total_size", ByteSizeValue.ofBytes(totalResponseSize));
-        histogramToXContent(builder, responseSizeHistogram);
+        histogramToXContent(builder, "size_histogram", "bytes", responseSizeHistogram, HttpRouteStatsTracker.getBucketUpperBounds());
+        histogramToXContent(
+            builder,
+            "handling_time_histogram",
+            "millis",
+            responseTimeHistogram,
+            HandlingTimeTracker.getBucketUpperBounds()
+        );
         builder.endObject();
 
         return builder.endObject();
     }
 
-    static void histogramToXContent(XContentBuilder builder, long[] sizeHistogram) throws IOException {
-        final int[] bucketBounds = HttpRouteStatsTracker.getBucketUpperBounds();
-        assert sizeHistogram.length == bucketBounds.length + 1;
-        builder.startArray("histogram");
+    static void histogramToXContent(XContentBuilder builder, String fieldName, String unitName, long[] histogram, int[] bucketBounds)
+        throws IOException {
+        assert histogram.length == bucketBounds.length + 1;
+        builder.startArray(fieldName);
 
         int firstBucket = 0;
         long remainingCount = 0L;
-        for (int i = 0; i < sizeHistogram.length; i++) {
+        for (int i = 0; i < histogram.length; i++) {
             if (remainingCount == 0) {
                 firstBucket = i;
             }
-            remainingCount += sizeHistogram[i];
+            remainingCount += histogram[i];
         }
 
-        for (int i = firstBucket; i < sizeHistogram.length && 0 < remainingCount; i++) {
+        for (int i = firstBucket; i < histogram.length && 0 < remainingCount; i++) {
             builder.startObject();
             if (i > 0) {
-                builder.humanReadableField("ge_bytes", "ge", ByteSizeValue.ofBytes(bucketBounds[i - 1]));
+                builder.humanReadableField("ge_" + unitName, "ge", ByteSizeValue.ofBytes(bucketBounds[i - 1]));
             }
             if (i < bucketBounds.length) {
-                builder.humanReadableField("lt_bytes", "lt", ByteSizeValue.ofBytes(bucketBounds[i]));
+                builder.humanReadableField("lt_" + unitName, "lt", ByteSizeValue.ofBytes(bucketBounds[i]));
             }
-            builder.field("count", sizeHistogram[i]);
+            builder.field("count", histogram[i]);
             builder.endObject();
-            remainingCount -= sizeHistogram[i];
+            remainingCount -= histogram[i];
         }
         builder.endArray();
     }
@@ -86,5 +95,6 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeVLong(responseCount);
         out.writeVLong(totalResponseSize);
         out.writeVLongArray(responseSizeHistogram);
+        out.writeVLongArray(responseTimeHistogram);
     }
 }

server/src/main/java/org/elasticsearch/http/HttpRouteStatsTracker.java

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.http;
 
+import org.elasticsearch.common.network.HandlingTimeTracker;
+
 import java.util.concurrent.atomic.AtomicLongArray;
 import java.util.concurrent.atomic.LongAdder;
 
@@ -67,6 +69,7 @@ private static int bucket(int contentLength) {
 
     private final StatsTracker requestStats = new StatsTracker();
     private final StatsTracker responseStats = new StatsTracker();
+    private final HandlingTimeTracker responseTimeTracker = new HandlingTimeTracker();
 
     public void addRequestStats(int contentLength) {
         requestStats.addStats(contentLength);
@@ -76,14 +79,19 @@ public void addResponseStats(int contentLength) {
         responseStats.addStats(contentLength);
     }
 
+    public void addResponseTime(long timeMillis) {
+        responseTimeTracker.addHandlingTime(timeMillis);
+    }
+
     public HttpRouteStats getStats() {
         return new HttpRouteStats(
             requestStats.count().longValue(),
             requestStats.totalSize().longValue(),
             requestStats.getHistogram(),
             responseStats.count().longValue(),
             responseStats.totalSize().longValue(),
-            responseStats.getHistogram()
+            responseStats.getHistogram(),
+            responseTimeTracker.getHistogram()
         );
     }
 }

server/src/main/java/org/elasticsearch/rest/ChunkedRestResponseBody.java

@@ -236,6 +236,7 @@ public ReleasableBytesReference encodeChunk(int sizeHint, Recycler<BytesRef> rec
                         () -> Releasables.closeExpectNoException(chunkOutput)
                     );
                     currentOutput = null;
+                    size += result.length();
                     return result;
                 } finally {
                     if (currentOutput != null) {

server/src/main/java/org/elasticsearch/rest/MethodHandlers.java

@@ -88,6 +88,10 @@ public void addResponseStats(int contentLength) {
         statsTracker.addResponseStats(contentLength);
     }
 
+    public void addResponseTime(long timeMillis) {
+        statsTracker.addResponseTime(timeMillis);
+    }
+
     public HttpRouteStats getStats() {
         return statsTracker.getStats();
     }

server/src/main/java/org/elasticsearch/rest/RestController.java

@@ -702,6 +702,7 @@ private static final class ResourceHandlingHttpChannel implements RestChannel {
         private final CircuitBreakerService circuitBreakerService;
         private final int contentLength;
         private final MethodHandlers methodHandlers;
+        private final long startTime;
         private final AtomicBoolean closed = new AtomicBoolean();
 
         ResourceHandlingHttpChannel(
@@ -714,7 +715,7 @@ private static final class ResourceHandlingHttpChannel implements RestChannel {
             this.circuitBreakerService = circuitBreakerService;
             this.contentLength = contentLength;
             this.methodHandlers = methodHandlers;
-            this.methodHandlers.addRequestStats(contentLength);
+            this.startTime = System.currentTimeMillis();
         }
 
         @Override
@@ -773,6 +774,7 @@ public void sendResponse(RestResponse response) {
             boolean success = false;
             try {
                 close();
+                methodHandlers.addRequestStats(contentLength);
                 if (response.isChunked() == false) {
                     methodHandlers.addResponseStats(response.content().length());
                 } else {
@@ -781,6 +783,7 @@ public void sendResponse(RestResponse response) {
                 delegate.sendResponse(response);
                 success = true;
             } finally {
+                methodHandlers.addResponseTime(System.currentTimeMillis() - startTime);
                 if (success == false) {
                     releaseOutputBuffer();
                 }

server/src/test/java/org/elasticsearch/http/HttpStatsTests.java

@@ -42,13 +42,19 @@ public void testToXContent() {
         final var responseSizeHistogram = new long[28];
         responseSizeHistogram[3] = 13;
         responseSizeHistogram[5] = 14;
+
+        final var responseTimeHistogram = new long[18];
+        responseTimeHistogram[4] = 17;
+        responseTimeHistogram[6] = 18;
+
         final HttpRouteStats httpRouteStats = new HttpRouteStats(
             1,
             ByteSizeUnit.MB.toBytes(2),
             requestSizeHistogram,
             3,
             ByteSizeUnit.MB.toBytes(4),
-            responseSizeHistogram
+            responseSizeHistogram,
+            responseTimeHistogram
         );
 
         assertThat(