diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/DanglingIndicesRestIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/DanglingIndicesRestIT.java index 6880c558cc4d6..70df4aaeaf5de 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/DanglingIndicesRestIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/DanglingIndicesRestIT.java @@ -30,7 +30,6 @@ import static org.elasticsearch.cluster.metadata.IndexGraveyard.SETTING_MAX_TOMBSTONES; import static org.elasticsearch.indices.IndicesService.WRITE_DANGLING_INDICES_INFO_SETTING; import static org.elasticsearch.rest.RestStatus.ACCEPTED; -import static org.elasticsearch.rest.RestStatus.OK; import static org.elasticsearch.test.XContentTestUtils.createJsonMapView; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; @@ -184,10 +183,6 @@ private List listDanglingIndexIds() throws IOException { return danglingIndexIds; } - private void assertOK(Response response) { - assertThat(response.getStatusLine().getStatusCode(), equalTo(OK.getStatus())); - } - /** * Given a node name, finds the corresponding node ID. */ diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpSmokeTestCase.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpSmokeTestCase.java index d62a490d092df..804aa5a2bda3a 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpSmokeTestCase.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpSmokeTestCase.java @@ -7,6 +7,7 @@ */ package org.elasticsearch.http; +import org.elasticsearch.client.Response; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.Plugin; @@ -17,6 +18,9 @@ import java.util.Collection; import java.util.List; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.equalTo; + public abstract class HttpSmokeTestCase extends ESIntegTestCase { @Override @@ -42,4 +46,8 @@ protected Collection> nodePlugins() { protected boolean ignoreExternalCluster() { return true; } + + public static void assertOK(Response response) { + assertThat(response.getStatusLine().getStatusCode(), anyOf(equalTo(200), equalTo(201))); + } } diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpStatsIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpStatsIT.java new file mode 100644 index 0000000000000..d082706858258 --- /dev/null +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpStatsIT.java @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.http; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.XContentTestUtils; +import org.elasticsearch.xcontent.json.JsonXContent; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.aMapWithSize; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.notNullValue; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 1, numClientNodes = 0) +public class HttpStatsIT extends HttpSmokeTestCase { + + @SuppressWarnings("unchecked") + public void testHttpStats() throws IOException { + // basic request + final RestClient restClient = getRestClient(); + assertOK(restClient.performRequest(new Request("GET", "/"))); + // request with body and URL placeholder + final Request searchRequest = new Request("GET", "*/_search"); + searchRequest.setJsonEntity(""" + {"query":{"match_all":{}}}"""); + assertOK(restClient.performRequest(searchRequest)); + // chunked response + assertOK(restClient.performRequest(new Request("GET", "/_cluster/state"))); + // chunked text response + assertOK(restClient.performRequest(new Request("GET", "/_cat/nodes"))); + + final Response response = restClient.performRequest(new Request("GET", "/_nodes/stats/http")); + assertOK(response); + + final Map responseMap = XContentHelper.convertToMap( + JsonXContent.jsonXContent, + response.getEntity().getContent(), + false + ); + final Map nodesMap = (Map) responseMap.get("nodes"); + + assertThat(nodesMap, aMapWithSize(1)); + final String nodeId = nodesMap.keySet().iterator().next(); + final XContentTestUtils.JsonMapView nodeView = new XContentTestUtils.JsonMapView((Map) nodesMap.get(nodeId)); + + final List routes = List.of("/", "/_cat/nodes", "/{index}/_search", "/_cluster/state"); + + for (var route : routes) { + assertThat(nodeView.get("http.routes." + route), notNullValue()); + assertThat(nodeView.get("http.routes." + route + ".requests.count"), equalTo(1)); + assertThat(nodeView.get("http.routes." + route + ".requests.total_size_in_bytes"), greaterThanOrEqualTo(0)); + assertThat(nodeView.get("http.routes." + route + ".responses.count"), equalTo(1)); + assertThat(nodeView.get("http.routes." + route + ".responses.total_size_in_bytes"), greaterThan(1)); + assertThat(nodeView.get("http.routes." + route + ".requests.size_histogram"), hasSize(1)); + assertThat(nodeView.get("http.routes." + route + ".requests.size_histogram.0.count"), equalTo(1)); + assertThat(nodeView.get("http.routes." + route + ".requests.size_histogram.0.lt_bytes"), notNullValue()); + if (route.equals("/{index}/_search")) { + assertThat(nodeView.get("http.routes." + route + ".requests.size_histogram.0.ge_bytes"), notNullValue()); + } + assertThat(nodeView.get("http.routes." + route + ".responses.size_histogram"), hasSize(1)); + assertThat(nodeView.get("http.routes." + route + ".responses.size_histogram.0.count"), equalTo(1)); + assertThat(nodeView.get("http.routes." + route + ".responses.size_histogram.0.lt_bytes"), notNullValue()); + assertThat(nodeView.get("http.routes." + route + ".responses.size_histogram.0.ge_bytes"), notNullValue()); + assertThat(nodeView.get("http.routes." + route + ".responses.handling_time_histogram"), hasSize(1)); + assertThat(nodeView.get("http.routes." + route + ".responses.handling_time_histogram.0.count"), equalTo(1)); + assertThat(nodeView.get("http.routes." + route + ".responses.handling_time_histogram.0.lt_millis"), notNullValue()); + assertThat(nodeView.get("http.routes." + route + ".responses.handling_time_histogram.0.ge_millis"), notNullValue()); + } + } +} diff --git a/qa/smoke-test-multinode/src/yamlRestTest/resources/rest-api-spec/test/smoke_test_multinode/60_http_stats.yml b/qa/smoke-test-multinode/src/yamlRestTest/resources/rest-api-spec/test/smoke_test_multinode/60_http_stats.yml deleted file mode 100644 index a72e9f0316b10..0000000000000 --- a/qa/smoke-test-multinode/src/yamlRestTest/resources/rest-api-spec/test/smoke_test_multinode/60_http_stats.yml +++ /dev/null @@ -1,22 +0,0 @@ -# This test needs multiple nodes, because a single-node cluster does not send any transport actions so these stats are empty ---- -"http stats": - - skip: - features: [arbitrary_key] - - - do: - search: - index: "*" - body: - query: - match_all: {} - - - do: - nodes.stats: - metric: [ http ] - human: true - - - set: - nodes._arbitrary_key_: node_id - - - is_true: "nodes.$node_id.http.routes./_cat/nodes" diff --git a/server/src/main/java/org/elasticsearch/http/HttpRouteStats.java b/server/src/main/java/org/elasticsearch/http/HttpRouteStats.java index 60736be15ed3a..45e1964abe3cc 100644 --- a/server/src/main/java/org/elasticsearch/http/HttpRouteStats.java +++ b/server/src/main/java/org/elasticsearch/http/HttpRouteStats.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.network.HandlingTimeTracker; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; @@ -23,11 +24,12 @@ public record HttpRouteStats( long[] requestSizeHistogram, long responseCount, long totalResponseSize, - long[] responseSizeHistogram + long[] responseSizeHistogram, + long[] responseTimeHistogram ) implements Writeable, ToXContentObject { public HttpRouteStats(StreamInput in) throws IOException { - this(in.readVLong(), in.readVLong(), in.readVLongArray(), in.readVLong(), in.readVLong(), in.readVLongArray()); + this(in.readVLong(), in.readVLong(), in.readVLongArray(), in.readVLong(), in.readVLong(), in.readVLongArray(), in.readVLongArray()); } @Override @@ -37,43 +39,50 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject("requests"); builder.field("count", requestCount); builder.humanReadableField("total_size_in_bytes", "total_size", ByteSizeValue.ofBytes(totalRequestSize)); - histogramToXContent(builder, requestSizeHistogram); + histogramToXContent(builder, "size_histogram", "bytes", requestSizeHistogram, HttpRouteStatsTracker.getBucketUpperBounds()); builder.endObject(); builder.startObject("responses"); builder.field("count", responseCount); builder.humanReadableField("total_size_in_bytes", "total_size", ByteSizeValue.ofBytes(totalResponseSize)); - histogramToXContent(builder, responseSizeHistogram); + histogramToXContent(builder, "size_histogram", "bytes", responseSizeHistogram, HttpRouteStatsTracker.getBucketUpperBounds()); + histogramToXContent( + builder, + "handling_time_histogram", + "millis", + responseTimeHistogram, + HandlingTimeTracker.getBucketUpperBounds() + ); builder.endObject(); return builder.endObject(); } - static void histogramToXContent(XContentBuilder builder, long[] sizeHistogram) throws IOException { - final int[] bucketBounds = HttpRouteStatsTracker.getBucketUpperBounds(); - assert sizeHistogram.length == bucketBounds.length + 1; - builder.startArray("histogram"); + static void histogramToXContent(XContentBuilder builder, String fieldName, String unitName, long[] histogram, int[] bucketBounds) + throws IOException { + assert histogram.length == bucketBounds.length + 1; + builder.startArray(fieldName); int firstBucket = 0; long remainingCount = 0L; - for (int i = 0; i < sizeHistogram.length; i++) { + for (int i = 0; i < histogram.length; i++) { if (remainingCount == 0) { firstBucket = i; } - remainingCount += sizeHistogram[i]; + remainingCount += histogram[i]; } - for (int i = firstBucket; i < sizeHistogram.length && 0 < remainingCount; i++) { + for (int i = firstBucket; i < histogram.length && 0 < remainingCount; i++) { builder.startObject(); if (i > 0) { - builder.humanReadableField("ge_bytes", "ge", ByteSizeValue.ofBytes(bucketBounds[i - 1])); + builder.humanReadableField("ge_" + unitName, "ge", ByteSizeValue.ofBytes(bucketBounds[i - 1])); } if (i < bucketBounds.length) { - builder.humanReadableField("lt_bytes", "lt", ByteSizeValue.ofBytes(bucketBounds[i])); + builder.humanReadableField("lt_" + unitName, "lt", ByteSizeValue.ofBytes(bucketBounds[i])); } - builder.field("count", sizeHistogram[i]); + builder.field("count", histogram[i]); builder.endObject(); - remainingCount -= sizeHistogram[i]; + remainingCount -= histogram[i]; } builder.endArray(); } @@ -86,5 +95,6 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVLong(responseCount); out.writeVLong(totalResponseSize); out.writeVLongArray(responseSizeHistogram); + out.writeVLongArray(responseTimeHistogram); } } diff --git a/server/src/main/java/org/elasticsearch/http/HttpRouteStatsTracker.java b/server/src/main/java/org/elasticsearch/http/HttpRouteStatsTracker.java index ceb7ba215ac24..b6cda33bfc1f6 100644 --- a/server/src/main/java/org/elasticsearch/http/HttpRouteStatsTracker.java +++ b/server/src/main/java/org/elasticsearch/http/HttpRouteStatsTracker.java @@ -8,6 +8,8 @@ package org.elasticsearch.http; +import org.elasticsearch.common.network.HandlingTimeTracker; + import java.util.concurrent.atomic.AtomicLongArray; import java.util.concurrent.atomic.LongAdder; @@ -67,6 +69,7 @@ private static int bucket(int contentLength) { private final StatsTracker requestStats = new StatsTracker(); private final StatsTracker responseStats = new StatsTracker(); + private final HandlingTimeTracker responseTimeTracker = new HandlingTimeTracker(); public void addRequestStats(int contentLength) { requestStats.addStats(contentLength); @@ -76,6 +79,10 @@ public void addResponseStats(int contentLength) { responseStats.addStats(contentLength); } + public void addResponseTime(long timeMillis) { + responseTimeTracker.addHandlingTime(timeMillis); + } + public HttpRouteStats getStats() { return new HttpRouteStats( requestStats.count().longValue(), @@ -83,7 +90,8 @@ public HttpRouteStats getStats() { requestStats.getHistogram(), responseStats.count().longValue(), responseStats.totalSize().longValue(), - responseStats.getHistogram() + responseStats.getHistogram(), + responseTimeTracker.getHistogram() ); } } diff --git a/server/src/main/java/org/elasticsearch/rest/ChunkedRestResponseBody.java b/server/src/main/java/org/elasticsearch/rest/ChunkedRestResponseBody.java index 9fa9e5db80562..45b05757c7bcd 100644 --- a/server/src/main/java/org/elasticsearch/rest/ChunkedRestResponseBody.java +++ b/server/src/main/java/org/elasticsearch/rest/ChunkedRestResponseBody.java @@ -236,6 +236,7 @@ public ReleasableBytesReference encodeChunk(int sizeHint, Recycler rec () -> Releasables.closeExpectNoException(chunkOutput) ); currentOutput = null; + size += result.length(); return result; } finally { if (currentOutput != null) { diff --git a/server/src/main/java/org/elasticsearch/rest/MethodHandlers.java b/server/src/main/java/org/elasticsearch/rest/MethodHandlers.java index 5d7b128ee223b..6c62c4cdcd1dc 100644 --- a/server/src/main/java/org/elasticsearch/rest/MethodHandlers.java +++ b/server/src/main/java/org/elasticsearch/rest/MethodHandlers.java @@ -88,6 +88,10 @@ public void addResponseStats(int contentLength) { statsTracker.addResponseStats(contentLength); } + public void addResponseTime(long timeMillis) { + statsTracker.addResponseTime(timeMillis); + } + public HttpRouteStats getStats() { return statsTracker.getStats(); } diff --git a/server/src/main/java/org/elasticsearch/rest/RestController.java b/server/src/main/java/org/elasticsearch/rest/RestController.java index 666b768d13a6b..338c6e346169f 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestController.java +++ b/server/src/main/java/org/elasticsearch/rest/RestController.java @@ -702,6 +702,7 @@ private static final class ResourceHandlingHttpChannel implements RestChannel { private final CircuitBreakerService circuitBreakerService; private final int contentLength; private final MethodHandlers methodHandlers; + private final long startTime; private final AtomicBoolean closed = new AtomicBoolean(); ResourceHandlingHttpChannel( @@ -714,7 +715,7 @@ private static final class ResourceHandlingHttpChannel implements RestChannel { this.circuitBreakerService = circuitBreakerService; this.contentLength = contentLength; this.methodHandlers = methodHandlers; - this.methodHandlers.addRequestStats(contentLength); + this.startTime = System.currentTimeMillis(); } @Override @@ -773,6 +774,7 @@ public void sendResponse(RestResponse response) { boolean success = false; try { close(); + methodHandlers.addRequestStats(contentLength); if (response.isChunked() == false) { methodHandlers.addResponseStats(response.content().length()); } else { @@ -781,6 +783,7 @@ public void sendResponse(RestResponse response) { delegate.sendResponse(response); success = true; } finally { + methodHandlers.addResponseTime(System.currentTimeMillis() - startTime); if (success == false) { releaseOutputBuffer(); } diff --git a/server/src/test/java/org/elasticsearch/http/HttpStatsTests.java b/server/src/test/java/org/elasticsearch/http/HttpStatsTests.java index 8e0704ba3e996..1b49dd084f3f6 100644 --- a/server/src/test/java/org/elasticsearch/http/HttpStatsTests.java +++ b/server/src/test/java/org/elasticsearch/http/HttpStatsTests.java @@ -42,13 +42,19 @@ public void testToXContent() { final var responseSizeHistogram = new long[28]; responseSizeHistogram[3] = 13; responseSizeHistogram[5] = 14; + + final var responseTimeHistogram = new long[18]; + responseTimeHistogram[4] = 17; + responseTimeHistogram[6] = 18; + final HttpRouteStats httpRouteStats = new HttpRouteStats( 1, ByteSizeUnit.MB.toBytes(2), requestSizeHistogram, 3, ByteSizeUnit.MB.toBytes(4), - responseSizeHistogram + responseSizeHistogram, + responseTimeHistogram ); assertThat(