Record more detailed HTTP stats (#99852)

This PR adds more details HTTP stats breaking down by HTTP routes. Resolves: #95739
elastic · Oct 12, 2023 · 6ed4ad5 · 6ed4ad5
1 parent db6239b
commit 6ed4ad5
Show file tree

Hide file tree

Showing 17 changed files with 912 additions and 22 deletions.
diff --git a/docs/changelog/99852.yaml b/docs/changelog/99852.yaml
@@ -0,0 +1,5 @@
+pr: 99852
+summary: Record more detailed HTTP stats
+area: Network
+type: enhancement
+issues: []
diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/DanglingIndicesRestIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/DanglingIndicesRestIT.java
@@ -30,7 +30,6 @@
 import static org.elasticsearch.cluster.metadata.IndexGraveyard.SETTING_MAX_TOMBSTONES;
 import static org.elasticsearch.indices.IndicesService.WRITE_DANGLING_INDICES_INFO_SETTING;
 import static org.elasticsearch.rest.RestStatus.ACCEPTED;
-import static org.elasticsearch.rest.RestStatus.OK;
 import static org.elasticsearch.test.XContentTestUtils.createJsonMapView;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
@@ -184,10 +183,6 @@ private List<String> listDanglingIndexIds() throws IOException {
         return danglingIndexIds;
     }
 
-    private void assertOK(Response response) {
-        assertThat(response.getStatusLine().getStatusCode(), equalTo(OK.getStatus()));
-    }
-
     /**
      * Given a node name, finds the corresponding node ID.
      */

diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpSmokeTestCase.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpSmokeTestCase.java
@@ -7,6 +7,7 @@
  */
 package org.elasticsearch.http;
 
+import org.elasticsearch.client.Response;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
@@ -17,6 +18,8 @@
 import java.util.Collection;
 import java.util.List;
 
+import static org.hamcrest.Matchers.oneOf;
+
 public abstract class HttpSmokeTestCase extends ESIntegTestCase {
 
     @Override
@@ -42,4 +45,8 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
     protected boolean ignoreExternalCluster() {
         return true;
     }
+
+    public static void assertOK(Response response) {
+        assertThat(response.getStatusLine().getStatusCode(), oneOf(200, 201));
+    }
 }
diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpStatsIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpStatsIT.java
@@ -0,0 +1,116 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.http;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.XContentTestUtils;
+import org.elasticsearch.xcontent.json.JsonXContent;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.aMapWithSize;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 0, numClientNodes = 0)
+public class HttpStatsIT extends HttpSmokeTestCase {
+
+    @SuppressWarnings("unchecked")
+    public void testNodeHttpStats() throws IOException {
+        internalCluster().startNode();
+        performHttpRequests();
+
+        final Response response = getRestClient().performRequest(new Request("GET", "/_nodes/stats/http"));
+        assertOK(response);
+
+        final Map<String, Object> responseMap = XContentHelper.convertToMap(
+            JsonXContent.jsonXContent,
+            response.getEntity().getContent(),
+            false
+        );
+        final Map<String, Object> nodesMap = (Map<String, Object>) responseMap.get("nodes");
+
+        assertThat(nodesMap, aMapWithSize(1));
+        final String nodeId = nodesMap.keySet().iterator().next();
+
+        assertHttpStats(new XContentTestUtils.JsonMapView((Map<String, Object>) nodesMap.get(nodeId)));
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testClusterInfoHttpStats() throws IOException {
+        internalCluster().ensureAtLeastNumDataNodes(3);
+        performHttpRequests();
+
+        final Response response = getRestClient().performRequest(new Request("GET", "/_info/http"));
+        assertOK(response);
+
+        final Map<String, Object> responseMap = XContentHelper.convertToMap(
+            JsonXContent.jsonXContent,
+            response.getEntity().getContent(),
+            false
+        );
+        assertHttpStats(new XContentTestUtils.JsonMapView(responseMap));
+    }
+
+    private void performHttpRequests() throws IOException {
+        // basic request
+        final RestClient restClient = getRestClient();
+        assertOK(restClient.performRequest(new Request("GET", "/")));
+        // request with body and URL placeholder
+        final Request searchRequest = new Request("GET", "*/_search");
+        searchRequest.setJsonEntity("""
+            {"query":{"match_all":{}}}""");
+        assertOK(restClient.performRequest(searchRequest));
+        // chunked response
+        assertOK(restClient.performRequest(new Request("GET", "/_cluster/state")));
+        // chunked text response
+        assertOK(restClient.performRequest(new Request("GET", "/_cat/nodes")));
+    }
+
+    private void assertHttpStats(XContentTestUtils.JsonMapView jsonMapView) {
+        final List<String> routes = List.of("/", "/_cat/nodes", "/{index}/_search", "/_cluster/state");
+
+        for (var route : routes) {
+            assertThat(route, jsonMapView.get("http.routes." + route), notNullValue());
+            assertThat(route, jsonMapView.get("http.routes." + route + ".requests.count"), equalTo(1));
+            assertThat(route, jsonMapView.get("http.routes." + route + ".requests.total_size_in_bytes"), greaterThanOrEqualTo(0));
+            assertThat(route, jsonMapView.get("http.routes." + route + ".responses.count"), equalTo(1));
+            assertThat(route, jsonMapView.get("http.routes." + route + ".responses.total_size_in_bytes"), greaterThan(1));
+            assertThat(route, jsonMapView.get("http.routes." + route + ".requests.size_histogram"), hasSize(1));
+            assertThat(route, jsonMapView.get("http.routes." + route + ".requests.size_histogram.0.count"), equalTo(1));
+            assertThat(route, jsonMapView.get("http.routes." + route + ".requests.size_histogram.0.lt_bytes"), notNullValue());
+            if (route.equals("/{index}/_search")) {
+                assertThat(route, jsonMapView.get("http.routes." + route + ".requests.size_histogram.0.ge_bytes"), notNullValue());
+            }
+            assertThat(route, jsonMapView.get("http.routes." + route + ".responses.size_histogram"), hasSize(1));
+            assertThat(route, jsonMapView.get("http.routes." + route + ".responses.size_histogram.0.count"), equalTo(1));
+            assertThat(route, jsonMapView.get("http.routes." + route + ".responses.size_histogram.0.lt_bytes"), notNullValue());
+            assertThat(route, jsonMapView.get("http.routes." + route + ".responses.size_histogram.0.ge_bytes"), notNullValue());
+            assertThat(route, jsonMapView.get("http.routes." + route + ".responses.handling_time_histogram"), hasSize(1));
+            assertThat(route, jsonMapView.get("http.routes." + route + ".responses.handling_time_histogram.0.count"), equalTo(1));
+            final int ltMillis = jsonMapView.get("http.routes." + route + ".responses.handling_time_histogram.0.lt_millis");
+            assertThat(route, ltMillis, notNullValue());
+            assertThat(
+                route,
+                jsonMapView.get("http.routes." + route + ".responses.handling_time_histogram.0.ge_millis"),
+                ltMillis > 1 ? notNullValue() : nullValue()
+            );
+        }
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java
@@ -138,6 +138,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion PLUGIN_DESCRIPTOR_OPTIONAL_CLASSNAME = def(8_513_00_0);
     public static final TransportVersion UNIVERSAL_PROFILING_LICENSE_ADDED = def(8_514_00_0);
     public static final TransportVersion ELSER_SERVICE_MODEL_VERSION_ADDED = def(8_515_00_0);
+    public static final TransportVersion NODE_STATS_HTTP_ROUTE_STATS_ADDED = def(8_516_00_0);
     /*
      * STOP! READ THIS FIRST! No, really,
      *        ____ _____ ___  ____  _        ____  _____    _    ____    _____ _   _ ___ ____    _____ ___ ____  ____ _____ _

diff --git a/server/src/main/java/org/elasticsearch/common/path/PathTrie.java b/server/src/main/java/org/elasticsearch/common/path/PathTrie.java
@@ -8,6 +8,8 @@
 
 package org.elasticsearch.common.path;
 
+import org.elasticsearch.common.collect.Iterators;
+
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -267,6 +269,15 @@ private void put(Map<String, String> params, TrieNode node, String value) {
             }
         }
 
+        Iterator<T> allNodeValues() {
+            final Iterator<T> childrenIterator = Iterators.flatMap(children.values().iterator(), TrieNode::allNodeValues);
+            if (value == null) {
+                return childrenIterator;
+            } else {
+                return Iterators.concat(Iterators.single(value), childrenIterator);
+            }
+        }
+
         @Override
         public String toString() {
             return key;
@@ -366,4 +377,8 @@ public T next() {
             }
         };
     }
+
+    public Iterator<T> allNodeValues() {
+        return Iterators.concat(Iterators.single(rootValue), root.allNodeValues());
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/http/AbstractHttpServerTransport.java b/server/src/main/java/org/elasticsearch/http/AbstractHttpServerTransport.java
@@ -168,7 +168,12 @@ public HttpInfo info() {
 
     @Override
     public HttpStats stats() {
-        return new HttpStats(httpChannels.size(), totalChannelsAccepted.get(), httpClientStatsTracker.getClientStats());
+        return new HttpStats(
+            httpChannels.size(),
+            totalChannelsAccepted.get(),
+            httpClientStatsTracker.getClientStats(),
+            dispatcher.getStats()
+        );
     }
 
     protected void bindServer() {