From 2873701711f42b7daaeea9df3cf409ca9c2e784f Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 26 Sep 2023 14:41:36 +1000 Subject: [PATCH] add merge and info stats support --- .../org/elasticsearch/http/HttpStatsIT.java | 90 ++++++++++++------- .../elasticsearch/http/HttpRouteStats.java | 23 +++++ .../org/elasticsearch/http/HttpStats.java | 4 +- 3 files changed, 83 insertions(+), 34 deletions(-) diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpStatsIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpStatsIT.java index d082706858258..5be12958bd17e 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpStatsIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpStatsIT.java @@ -27,25 +27,15 @@ import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.notNullValue; -@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 1, numClientNodes = 0) +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 0, numClientNodes = 0) public class HttpStatsIT extends HttpSmokeTestCase { @SuppressWarnings("unchecked") - public void testHttpStats() throws IOException { - // basic request - final RestClient restClient = getRestClient(); - assertOK(restClient.performRequest(new Request("GET", "/"))); - // request with body and URL placeholder - final Request searchRequest = new Request("GET", "*/_search"); - searchRequest.setJsonEntity(""" - {"query":{"match_all":{}}}"""); - assertOK(restClient.performRequest(searchRequest)); - // chunked response - assertOK(restClient.performRequest(new Request("GET", "/_cluster/state"))); - // chunked text response - assertOK(restClient.performRequest(new Request("GET", "/_cat/nodes"))); + public void testNodeHttpStats() throws IOException { + internalCluster().startNode(); + performHttpRequests(); - final Response response = restClient.performRequest(new Request("GET", "/_nodes/stats/http")); + final Response response = getRestClient().performRequest(new Request("GET", "/_nodes/stats/http")); assertOK(response); final Map responseMap = XContentHelper.convertToMap( @@ -57,30 +47,64 @@ public void testHttpStats() throws IOException { assertThat(nodesMap, aMapWithSize(1)); final String nodeId = nodesMap.keySet().iterator().next(); - final XContentTestUtils.JsonMapView nodeView = new XContentTestUtils.JsonMapView((Map) nodesMap.get(nodeId)); + assertHttpStats(new XContentTestUtils.JsonMapView((Map) nodesMap.get(nodeId))); + } + + @SuppressWarnings("unchecked") + public void testClusterInfoHttpStats() throws IOException { + internalCluster().ensureAtLeastNumDataNodes(3); + performHttpRequests(); + + final Response response = getRestClient().performRequest(new Request("GET", "/_info/http")); + assertOK(response); + + final Map responseMap = XContentHelper.convertToMap( + JsonXContent.jsonXContent, + response.getEntity().getContent(), + false + ); + assertHttpStats(new XContentTestUtils.JsonMapView(responseMap)); + } + + private void performHttpRequests() throws IOException { + // basic request + final RestClient restClient = getRestClient(); + assertOK(restClient.performRequest(new Request("GET", "/"))); + // request with body and URL placeholder + final Request searchRequest = new Request("GET", "*/_search"); + searchRequest.setJsonEntity(""" + {"query":{"match_all":{}}}"""); + assertOK(restClient.performRequest(searchRequest)); + // chunked response + assertOK(restClient.performRequest(new Request("GET", "/_cluster/state"))); + // chunked text response + assertOK(restClient.performRequest(new Request("GET", "/_cat/nodes"))); + } + + private void assertHttpStats(XContentTestUtils.JsonMapView jsonMapView) { final List routes = List.of("/", "/_cat/nodes", "/{index}/_search", "/_cluster/state"); for (var route : routes) { - assertThat(nodeView.get("http.routes." + route), notNullValue()); - assertThat(nodeView.get("http.routes." + route + ".requests.count"), equalTo(1)); - assertThat(nodeView.get("http.routes." + route + ".requests.total_size_in_bytes"), greaterThanOrEqualTo(0)); - assertThat(nodeView.get("http.routes." + route + ".responses.count"), equalTo(1)); - assertThat(nodeView.get("http.routes." + route + ".responses.total_size_in_bytes"), greaterThan(1)); - assertThat(nodeView.get("http.routes." + route + ".requests.size_histogram"), hasSize(1)); - assertThat(nodeView.get("http.routes." + route + ".requests.size_histogram.0.count"), equalTo(1)); - assertThat(nodeView.get("http.routes." + route + ".requests.size_histogram.0.lt_bytes"), notNullValue()); + assertThat(jsonMapView.get("http.routes." + route), notNullValue()); + assertThat(jsonMapView.get("http.routes." + route + ".requests.count"), equalTo(1)); + assertThat(jsonMapView.get("http.routes." + route + ".requests.total_size_in_bytes"), greaterThanOrEqualTo(0)); + assertThat(jsonMapView.get("http.routes." + route + ".responses.count"), equalTo(1)); + assertThat(jsonMapView.get("http.routes." + route + ".responses.total_size_in_bytes"), greaterThan(1)); + assertThat(jsonMapView.get("http.routes." + route + ".requests.size_histogram"), hasSize(1)); + assertThat(jsonMapView.get("http.routes." + route + ".requests.size_histogram.0.count"), equalTo(1)); + assertThat(jsonMapView.get("http.routes." + route + ".requests.size_histogram.0.lt_bytes"), notNullValue()); if (route.equals("/{index}/_search")) { - assertThat(nodeView.get("http.routes." + route + ".requests.size_histogram.0.ge_bytes"), notNullValue()); + assertThat(jsonMapView.get("http.routes." + route + ".requests.size_histogram.0.ge_bytes"), notNullValue()); } - assertThat(nodeView.get("http.routes." + route + ".responses.size_histogram"), hasSize(1)); - assertThat(nodeView.get("http.routes." + route + ".responses.size_histogram.0.count"), equalTo(1)); - assertThat(nodeView.get("http.routes." + route + ".responses.size_histogram.0.lt_bytes"), notNullValue()); - assertThat(nodeView.get("http.routes." + route + ".responses.size_histogram.0.ge_bytes"), notNullValue()); - assertThat(nodeView.get("http.routes." + route + ".responses.handling_time_histogram"), hasSize(1)); - assertThat(nodeView.get("http.routes." + route + ".responses.handling_time_histogram.0.count"), equalTo(1)); - assertThat(nodeView.get("http.routes." + route + ".responses.handling_time_histogram.0.lt_millis"), notNullValue()); - assertThat(nodeView.get("http.routes." + route + ".responses.handling_time_histogram.0.ge_millis"), notNullValue()); + assertThat(jsonMapView.get("http.routes." + route + ".responses.size_histogram"), hasSize(1)); + assertThat(jsonMapView.get("http.routes." + route + ".responses.size_histogram.0.count"), equalTo(1)); + assertThat(jsonMapView.get("http.routes." + route + ".responses.size_histogram.0.lt_bytes"), notNullValue()); + assertThat(jsonMapView.get("http.routes." + route + ".responses.size_histogram.0.ge_bytes"), notNullValue()); + assertThat(jsonMapView.get("http.routes." + route + ".responses.handling_time_histogram"), hasSize(1)); + assertThat(jsonMapView.get("http.routes." + route + ".responses.handling_time_histogram.0.count"), equalTo(1)); + assertThat(jsonMapView.get("http.routes." + route + ".responses.handling_time_histogram.0.lt_millis"), notNullValue()); + assertThat(jsonMapView.get("http.routes." + route + ".responses.handling_time_histogram.0.ge_millis"), notNullValue()); } } } diff --git a/server/src/main/java/org/elasticsearch/http/HttpRouteStats.java b/server/src/main/java/org/elasticsearch/http/HttpRouteStats.java index 45e1964abe3cc..1a69414bc6745 100644 --- a/server/src/main/java/org/elasticsearch/http/HttpRouteStats.java +++ b/server/src/main/java/org/elasticsearch/http/HttpRouteStats.java @@ -17,6 +17,7 @@ import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.stream.IntStream; public record HttpRouteStats( long requestCount, @@ -87,6 +88,28 @@ static void histogramToXContent(XContentBuilder builder, String fieldName, Strin builder.endArray(); } + public static HttpRouteStats merge(HttpRouteStats first, HttpRouteStats second) { + assert first.requestSizeHistogram.length == second.requestSizeHistogram.length + && first.responseSizeHistogram.length == second.responseSizeHistogram.length + && first.responseTimeHistogram.length == second.responseTimeHistogram.length; + + return new HttpRouteStats( + first.requestCount + second.requestCount, + first.totalRequestSize + second.totalRequestSize, + IntStream.range(0, first.requestSizeHistogram.length) + .mapToLong(i -> first.requestSizeHistogram[i] + second.requestSizeHistogram[i]) + .toArray(), + first.responseCount + second.responseCount, + first.totalResponseSize + second.totalResponseSize, + IntStream.range(0, first.responseSizeHistogram.length) + .mapToLong(i -> first.responseSizeHistogram[i] + second.responseSizeHistogram[i]) + .toArray(), + IntStream.range(0, first.responseTimeHistogram.length) + .mapToLong(i -> first.responseTimeHistogram[i] + second.responseTimeHistogram[i]) + .toArray() + ); + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeVLong(requestCount); diff --git a/server/src/main/java/org/elasticsearch/http/HttpStats.java b/server/src/main/java/org/elasticsearch/http/HttpStats.java index ac3e7d64f9a19..45ab382208df4 100644 --- a/server/src/main/java/org/elasticsearch/http/HttpStats.java +++ b/server/src/main/java/org/elasticsearch/http/HttpStats.java @@ -21,6 +21,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import java.util.stream.Stream; import static org.elasticsearch.TransportVersions.NODE_STATS_HTTP_ROUTE_STATS_ADDED; @@ -72,7 +73,8 @@ public static HttpStats merge(HttpStats first, HttpStats second) { first.serverOpen + second.serverOpen, first.totalOpen + second.totalOpen, Stream.concat(first.clientStats.stream(), second.clientStats.stream()).toList(), - Map.of() // TODO: merge + Stream.concat(first.httpRouteStats.entrySet().stream(), second.httpRouteStats.entrySet().stream()) + .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue, HttpRouteStats::merge)) ); }