From e8c655bb44f224e3ee524ca71d84914b71f9f946 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 3 Sep 2020 05:53:09 -0700 Subject: [PATCH] Only request wildcard expansion for hidden indices if supported (#20938) * Refactoring: inverting logic to make room for another case * Expand hidden indices wildcards if monitored ES supports that option * Adding CHANGELOG entry * Fixing formatting * Avoid unnecessary setting * Removing unnecessary suffix existence checks * Fixing feature version * Add test cases to unit test * Updating test --- CHANGELOG.next.asciidoc | 1 + .../module/elasticsearch/elasticsearch.go | 22 +++++++++++------- .../module/elasticsearch/index/index.go | 23 ++++++++++--------- .../module/elasticsearch/index/index_test.go | 22 ++++++++++++++---- 4 files changed, 45 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index e9c35422339..ec86340f3d1 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -338,6 +338,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix resource tags in aws cloudwatch metricset {issue}20326[20326] {pull}20385[20385] - Fix ec2 disk and network metrics to use Sum statistic method. {pull}20680[20680] - Fill cloud.account.name with accountID if account alias doesn't exist. {pull}20736[20736] +- The `elasticsearch/index` metricset only requests wildcard expansion for hidden indices if the monitored Elasticsearch cluster supports it. {pull}20938[20938] *Packetbeat* diff --git a/metricbeat/module/elasticsearch/elasticsearch.go b/metricbeat/module/elasticsearch/elasticsearch.go index 52ee233d88c..a84bf644f3c 100644 --- a/metricbeat/module/elasticsearch/elasticsearch.go +++ b/metricbeat/module/elasticsearch/elasticsearch.go @@ -60,17 +60,23 @@ func NewModule(base mb.BaseModule) (mb.Module, error) { return elastic.NewModule(&base, xpackEnabledMetricSets, logp.NewLogger(ModuleName)) } -// CCRStatsAPIAvailableVersion is the version of Elasticsearch since when the CCR stats API is available. -var CCRStatsAPIAvailableVersion = common.MustNewVersion("6.5.0") +var ( + // CCRStatsAPIAvailableVersion is the version of Elasticsearch since when the CCR stats API is available. + CCRStatsAPIAvailableVersion = common.MustNewVersion("6.5.0") + + // EnrichStatsAPIAvailableVersion is the version of Elasticsearch since when the Enrich stats API is available. + EnrichStatsAPIAvailableVersion = common.MustNewVersion("7.5.0") -// EnrichStatsAPIAvailableVersion is the version of Elasticsearch since when the Enrich stats API is available. -var EnrichStatsAPIAvailableVersion = common.MustNewVersion("7.5.0") + // BulkStatsAvailableVersion is the version since when bulk indexing stats are available + BulkStatsAvailableVersion = common.MustNewVersion("8.0.0") -// BulkStatsAvailableVersion is the version since when bulk indexing stats are available -var BulkStatsAvailableVersion = common.MustNewVersion("8.0.0") + //ExpandWildcardsHiddenAvailableVersion is the version since when the "expand_wildcards" query parameter to + // the Indices Stats API can accept "hidden" as a value. + ExpandWildcardsHiddenAvailableVersion = common.MustNewVersion("7.7.0") -// Global clusterIdCache. Assumption is that the same node id never can belong to a different cluster id. -var clusterIDCache = map[string]string{} + // Global clusterIdCache. Assumption is that the same node id never can belong to a different cluster id. + clusterIDCache = map[string]string{} +) // ModuleName is the name of this module. const ModuleName = "elasticsearch" diff --git a/metricbeat/module/elasticsearch/index/index.go b/metricbeat/module/elasticsearch/index/index.go index 221e78ccfea..69a291aa708 100644 --- a/metricbeat/module/elasticsearch/index/index.go +++ b/metricbeat/module/elasticsearch/index/index.go @@ -38,8 +38,12 @@ func init() { } const ( - statsMetrics = "docs,fielddata,indexing,merge,search,segments,store,refresh,query_cache,request_cache" - statsPath = "/_stats/" + statsMetrics + "?filter_path=indices&expand_wildcards=open,hidden" + statsMetrics = "docs,fielddata,indexing,merge,search,segments,store,refresh,query_cache,request_cache" + expandWildcards = "expand_wildcards=open" + statsPath = "/_stats/" + statsMetrics + "?filter_path=indices&" + expandWildcards + + bulkSuffix = ",bulk" + hiddenSuffix = ",hidden" ) // MetricSet type defines all fields of the MetricSet @@ -114,21 +118,18 @@ func (m *MetricSet) updateServicePath(esVersion common.Version) error { func getServicePath(esVersion common.Version) (string, error) { currPath := statsPath - if esVersion.LessThan(elasticsearch.BulkStatsAvailableVersion) { - // Can't request bulk stats so don't change service URI - return currPath, nil - } - u, err := url.Parse(currPath) if err != nil { return "", err } - if strings.HasSuffix(u.Path, ",bulk") { - // Bulk stats already being requested so don't change service URI - return currPath, nil + if !esVersion.LessThan(elasticsearch.BulkStatsAvailableVersion) { + u.Path += bulkSuffix + } + + if !esVersion.LessThan(elasticsearch.ExpandWildcardsHiddenAvailableVersion) { + u.RawQuery = strings.Replace(u.RawQuery, expandWildcards, expandWildcards+hiddenSuffix, 1) } - u.Path += ",bulk" return u.String(), nil } diff --git a/metricbeat/module/elasticsearch/index/index_test.go b/metricbeat/module/elasticsearch/index/index_test.go index 523a2301e53..fe44dca6ba9 100644 --- a/metricbeat/module/elasticsearch/index/index_test.go +++ b/metricbeat/module/elasticsearch/index/index_test.go @@ -27,18 +27,29 @@ import ( "github.com/stretchr/testify/require" ) -func TestGetServiceURI(t *testing.T) { +func TestGetServiceURIExpectedPath(t *testing.T) { + path770 := strings.Replace(statsPath, expandWildcards, expandWildcards+hiddenSuffix, 1) + path800 := strings.Replace(path770, statsMetrics, statsMetrics+bulkSuffix, 1) + tests := map[string]struct { esVersion *common.Version expectedPath string }{ "bulk_stats_unavailable": { - esVersion: common.MustNewVersion("7.9.0"), + esVersion: common.MustNewVersion("7.6.0"), expectedPath: statsPath, }, "bulk_stats_available": { esVersion: common.MustNewVersion("8.0.0"), - expectedPath: strings.Replace(statsPath, statsMetrics, statsMetrics+",bulk", 1), + expectedPath: path800, + }, + "expand_wildcards_hidden_unavailable": { + esVersion: common.MustNewVersion("7.6.0"), + expectedPath: statsPath, + }, + "expand_wildcards_hidden_available": { + esVersion: common.MustNewVersion("7.7.0"), + expectedPath: path770, }, } @@ -52,6 +63,9 @@ func TestGetServiceURI(t *testing.T) { } func TestGetServiceURIMultipleCalls(t *testing.T) { + path := strings.Replace(statsPath, expandWildcards, expandWildcards+hiddenSuffix, 1) + path = strings.Replace(path, statsMetrics, statsMetrics+bulkSuffix, 1) + err := quick.Check(func(r uint) bool { numCalls := 2 + (r % 10) // between 2 and 11 @@ -64,7 +78,7 @@ func TestGetServiceURIMultipleCalls(t *testing.T) { } } - return err == nil && uri == strings.Replace(statsPath, statsMetrics, statsMetrics+",bulk", 1) + return err == nil && uri == path }, nil) require.NoError(t, err) }