[receiver/elasticsearch]: add scraping metrics on cluster level

.chloggen/elastic-cluster-level-metrics.yaml

@@ -0,0 +1,16 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: elasticsearchreceiver
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: add scraping metrics on cluster level
+
+# One or more tracking issues related to the change
+issues: [14635]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: The receiver now emits jvm and cache eviction metrics on cluster level scraped from new endpoint /_cluster/stats.

receiver/elasticsearchreceiver/README.md

@@ -19,7 +19,7 @@ See the [Elasticsearch docs](https://www.elastic.co/guide/en/elasticsearch/refer
 
 The following settings are optional:
 - `metrics` (default: see `DefaultMetricsSettings` [here](./internal/metadata/generated_metrics.go): Allows enabling and disabling specific metrics from being collected in this receiver.
-- `nodes` (default: `["_all"]`): Allows specifying node filters that define which nodes are scraped for node-level metrics. See [the Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/7.9/cluster.html#cluster-nodes) for allowed filters. If this option is left explicitly empty, then no node-level metrics will be scraped.
+- `nodes` (default: `["_all"]`): Allows specifying node filters that define which nodes are scraped for node-level and cluster-level metrics. See [the Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/7.9/cluster.html#cluster-nodes) for allowed filters. If this option is left explicitly empty, then no node-level metrics will be scraped and cluster-level metrics will scrape only metrics related to cluster's health.
 - `skip_cluster_metrics` (default: `false`): If true, cluster-level metrics will not be scraped.
 - `indices` (default: `["_all"]`): Allows specifying index filters that define which indices are scraped for index-level metrics. See [the Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-stats.html#index-stats-api-path-params) for allowed filters. If this option is left explicitly empty, then no index-level metrics will be scraped.
 - `endpoint` (default = `http://localhost:9200`): The base URL of the Elasticsearch API for the cluster to monitor.

receiver/elasticsearchreceiver/client.go

@@ -42,6 +42,7 @@ type elasticsearchClient interface {
 	ClusterHealth(ctx context.Context) (*model.ClusterHealth, error)
 	IndexStats(ctx context.Context, indices []string) (*model.IndexStats, error)
 	ClusterMetadata(ctx context.Context) (*model.ClusterMetadataResponse, error)
+	ClusterStats(ctx context.Context, nodes []string) (*model.ClusterStats, error)
 }
 
 // defaultElasticsearchClient is the main implementation of elasticsearchClient.
@@ -154,6 +155,27 @@ func (c defaultElasticsearchClient) ClusterMetadata(ctx context.Context) (*model
 	return &versionResponse, err
 }
 
+func (c defaultElasticsearchClient) ClusterStats(ctx context.Context, nodes []string) (*model.ClusterStats, error) {
+	var nodesSpec string
+	if len(nodes) > 0 {
+		nodesSpec = strings.Join(nodes, ",")
+	} else {
+		nodesSpec = "_all"
+	}
+
+	clusterStatsPath := fmt.Sprintf("_cluster/stats/%s", nodesSpec)
+
+	body, err := c.doRequest(ctx, clusterStatsPath)
+	if err != nil {
+		return nil, err
+	}
+
+	clusterStats := model.ClusterStats{}
+	err = json.Unmarshal(body, &clusterStats)
+
+	return &clusterStats, err
+}
+
 func (c defaultElasticsearchClient) doRequest(ctx context.Context, path string) ([]byte, error) {
 	endpoint, err := c.endpoint.Parse(path)
 	if err != nil {

receiver/elasticsearchreceiver/client_test.go

@@ -476,6 +476,116 @@ func TestIndexStatsBadAuthentication(t *testing.T) {
 	require.ErrorIs(t, err, errUnauthorized)
 }
 
+func TestClusterStatsNoPassword(t *testing.T) {
+	clusterJSON, err := os.ReadFile("./testdata/sample_payloads/cluster.json")
+	require.NoError(t, err)
+
+	actualClusterStats := model.ClusterStats{}
+	require.NoError(t, json.Unmarshal(clusterJSON, &actualClusterStats))
+
+	elasticsearchMock := mockServer(t, "", "")
+	defer elasticsearchMock.Close()
+
+	client, err := newElasticsearchClient(componenttest.NewNopTelemetrySettings(), Config{
+		HTTPClientSettings: confighttp.HTTPClientSettings{
+			Endpoint: elasticsearchMock.URL,
+		},
+	}, componenttest.NewNopHost())
+	require.NoError(t, err)
+	ctx := context.Background()
+	clusterStats, err := client.ClusterStats(ctx, []string{"_all"})
+	require.NoError(t, err)
+
+	require.Equal(t, &actualClusterStats, clusterStats)
+}
+
+func TestClusterStatsNilNodes(t *testing.T) {
+	clusterJSON, err := os.ReadFile("./testdata/sample_payloads/cluster.json")
+	require.NoError(t, err)
+
+	actualClusterStats := model.ClusterStats{}
+	require.NoError(t, json.Unmarshal(clusterJSON, &actualClusterStats))
+
+	elasticsearchMock := mockServer(t, "", "")
+	defer elasticsearchMock.Close()
+
+	client, err := newElasticsearchClient(componenttest.NewNopTelemetrySettings(), Config{
+		HTTPClientSettings: confighttp.HTTPClientSettings{
+			Endpoint: elasticsearchMock.URL,
+		},
+	}, componenttest.NewNopHost())
+	require.NoError(t, err)
+
+	ctx := context.Background()
+	clusterStats, err := client.ClusterStats(ctx, nil)
+	require.NoError(t, err)
+
+	require.Equal(t, &actualClusterStats, clusterStats)
+}
+
+func TestClusterStatsAuthentication(t *testing.T) {
+	clusterJSON, err := os.ReadFile("./testdata/sample_payloads/cluster.json")
+	require.NoError(t, err)
+
+	actualClusterStats := model.ClusterStats{}
+	require.NoError(t, json.Unmarshal(clusterJSON, &actualClusterStats))
+
+	username := "user"
+	password := "pass"
+
+	elasticsearchMock := mockServer(t, username, password)
+	defer elasticsearchMock.Close()
+
+	client, err := newElasticsearchClient(componenttest.NewNopTelemetrySettings(), Config{
+		HTTPClientSettings: confighttp.HTTPClientSettings{
+			Endpoint: elasticsearchMock.URL,
+		},
+		Username: username,
+		Password: password,
+	}, componenttest.NewNopHost())
+	require.NoError(t, err)
+
+	ctx := context.Background()
+	clusterStats, err := client.ClusterStats(ctx, []string{"_all"})
+	require.NoError(t, err)
+
+	require.Equal(t, &actualClusterStats, clusterStats)
+}
+
+func TestClusterStatsNoAuthentication(t *testing.T) {
+	elasticsearchMock := mockServer(t, "user", "pass")
+	defer elasticsearchMock.Close()
+
+	client, err := newElasticsearchClient(componenttest.NewNopTelemetrySettings(), Config{
+		HTTPClientSettings: confighttp.HTTPClientSettings{
+			Endpoint: elasticsearchMock.URL,
+		},
+	}, componenttest.NewNopHost())
+	require.NoError(t, err)
+
+	ctx := context.Background()
+	_, err = client.ClusterStats(ctx, []string{"_all"})
+	require.ErrorIs(t, err, errUnauthenticated)
+}
+
+func TestClusterStatsBadAuthentication(t *testing.T) {
+	elasticsearchMock := mockServer(t, "user", "pass")
+	defer elasticsearchMock.Close()
+
+	client, err := newElasticsearchClient(componenttest.NewNopTelemetrySettings(), Config{
+		HTTPClientSettings: confighttp.HTTPClientSettings{
+			Endpoint: elasticsearchMock.URL,
+		},
+		Username: "bad_user",
+		Password: "bad_pass",
+	}, componenttest.NewNopHost())
+	require.NoError(t, err)
+
+	ctx := context.Background()
+	_, err = client.ClusterStats(ctx, []string{"_all"})
+	require.ErrorIs(t, err, errUnauthorized)
+}
+
 // mockServer gives a mock elasticsearch server for testing; if username or password is included, they will be required for the client.
 // otherwise, authorization is ignored.
 func mockServer(t *testing.T, username, password string) *httptest.Server {
@@ -487,6 +597,8 @@ func mockServer(t *testing.T, username, password string) *httptest.Server {
 	require.NoError(t, err)
 	metadata, err := os.ReadFile("./testdata/sample_payloads/metadata.json")
 	require.NoError(t, err)
+	cluster, err := os.ReadFile("./testdata/sample_payloads/cluster.json")
+	require.NoError(t, err)
 
 	elasticsearchMock := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
 		if username != "" || password != "" {
@@ -521,6 +633,13 @@ func mockServer(t *testing.T, username, password string) *httptest.Server {
 			return
 		}
 
+		if strings.HasPrefix(req.URL.Path, "/_cluster/stats") {
+			rw.WriteHeader(200)
+			_, err = rw.Write(cluster)
+			require.NoError(t, err)
+			return
+		}
+
 		// metadata check
 		if req.URL.Path == "/" {
 			rw.WriteHeader(200)

receiver/elasticsearchreceiver/config.go

@@ -47,7 +47,7 @@ type Config struct {
 	// See https://www.elastic.co/guide/en/elasticsearch/reference/7.9/cluster.html#cluster-nodes for which selectors may be used here.
 	// If Nodes is empty, no nodes will be scraped.
 	Nodes []string `mapstructure:"nodes"`
-	// SkipClusterMetrics indicates whether cluster level metrics from /_cluster/health should be scraped or not.
+	// SkipClusterMetrics indicates whether cluster level metrics from /_cluster/* endpoints should be scraped or not.
 	SkipClusterMetrics bool `mapstructure:"skip_cluster_metrics"`
 	// Indices defines the indices to scrape.
 	// See https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-stats.html#index-stats-api-path-params

receiver/elasticsearchreceiver/documentation.md

@@ -775,6 +775,20 @@ metrics:
     enabled: true
 ```
 
+### elasticsearch.cluster.indices.cache.evictions
+
+The number of evictions from the cache for indices in cluster.
+
+| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic |
+| ---- | ----------- | ---------- | ----------------------- | --------- |
+| {evictions} | Sum | Int | Cumulative | true |
+
+#### Attributes
+
+| Name | Description | Values |
+| ---- | ----------- | ------ |
+| cache_name | The name of cache. | Str: ``fielddata``, ``query`` |
+
 ### elasticsearch.index.cache.evictions
 
 The number of evictions from the cache for an index.