Elasticsearch index summary metricset (elastic#6918)

This provides stats for the summary across all Elasticsearch indicies. Starts are fetched from the _stats endpoint.
leweafan · May 9, 2018 · 68387b0 · 68387b0
1 parent 6c8d305
commit 68387b0
Show file tree

Hide file tree

Showing 21 changed files with 518 additions and 48 deletions.
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -228,6 +228,7 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff]
 - Add dashboards and visualizations for haproxy metrics. {pull}6934[6934]
 - Add message rates to the RabbitMQ queue metricset {issue}6442[6442] {pull}6606[6606]
 - Add exchanges metricset to the RabbitMQ module {issue}6442[6442] {pull}6607[6607]
+- Add Elasticsearch index_summary metricset. {pull}6918[6918]
 
 *Packetbeat*
 

diff --git a/libbeat/tests/compose/compose.go b/libbeat/tests/compose/compose.go
@@ -37,7 +37,7 @@ type serviceInfo struct {
 var oldRegexp = regexp.MustCompile("minute")
 
 // EnsureUp starts all the requested services (must be defined in docker-compose.yml)
-// with a default timeout of 60 seconds
+// with a default timeout of 300 seconds
 func EnsureUp(t *testing.T, services ...string) {
 	EnsureUpWithTimeout(t, 300, services...)
 }

diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -3409,6 +3409,123 @@ format: bytes
 Total number of memory used by the segments in bytes.
 
 
+--
+
+[float]
+== index.summary fields
+
+index
+
+
+
+
+*`elasticsearch.index.summary.primaries.docs.count`*::
++
+--
+type: long
+
+Total number of documents in the index.
+
+
+--
+
+*`elasticsearch.index.summary.primaries.docs.deleted`*::
++
+--
+type: long
+
+Total number of deleted documents in the index.
+
+
+--
+
+*`elasticsearch.index.summary.primaries.store.size.bytes`*::
++
+--
+type: long
+
+format: bytes
+
+Total size of the index in bytes.
+
+
+--
+
+*`elasticsearch.index.summary.primaries.segments.count`*::
++
+--
+type: long
+
+Total number of index segments.
+
+
+--
+
+*`elasticsearch.index.summary.primaries.segments.memory.bytes`*::
++
+--
+type: long
+
+format: bytes
+
+Total number of memory used by the segments in bytes.
+
+
+--
+
+
+*`elasticsearch.index.summary.total.docs.count`*::
++
+--
+type: long
+
+Total number of documents in the index.
+
+
+--
+
+*`elasticsearch.index.summary.total.docs.deleted`*::
++
+--
+type: long
+
+Total number of deleted documents in the index.
+
+
+--
+
+*`elasticsearch.index.summary.total.store.size.bytes`*::
++
+--
+type: long
+
+format: bytes
+
+Total size of the index in bytes.
+
+
+--
+
+*`elasticsearch.index.summary.total.segments.count`*::
++
+--
+type: long
+
+Total number of index segments.
+
+
+--
+
+*`elasticsearch.index.summary.total.segments.memory.bytes`*::
++
+--
+type: long
+
+format: bytes
+
+Total number of memory used by the segments in bytes.
+
+
 --
 
 [float]

diff --git a/metricbeat/docs/modules/elasticsearch.asciidoc b/metricbeat/docs/modules/elasticsearch.asciidoc
@@ -24,6 +24,7 @@ metricbeat.modules:
 - module: elasticsearch
   metricsets:
    #- index
+   #- index_summary
    - node
    - node_stats
   period: 10s
@@ -39,12 +40,16 @@ The following metricsets are available:
 
 * <<metricbeat-metricset-elasticsearch-index,index>>
 
+* <<metricbeat-metricset-elasticsearch-index_summary,index_summary>>
+
 * <<metricbeat-metricset-elasticsearch-node,node>>
 
 * <<metricbeat-metricset-elasticsearch-node_stats,node_stats>>
 
 include::elasticsearch/index.asciidoc[]
 
+include::elasticsearch/index_summary.asciidoc[]
+
 include::elasticsearch/node.asciidoc[]
 
 include::elasticsearch/node_stats.asciidoc[]

diff --git a/metricbeat/docs/modules/elasticsearch/index_summary.asciidoc b/metricbeat/docs/modules/elasticsearch/index_summary.asciidoc
@@ -0,0 +1,23 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[metricbeat-metricset-elasticsearch-index_summary]]
+=== Elasticsearch index_summary metricset
+
+experimental[]
+
+include::../../../module/elasticsearch/index_summary/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-elasticsearch,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/elasticsearch/index_summary/_meta/data.json[]
+----
diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc
@@ -33,7 +33,8 @@ This file is generated! See scripts/docs_collector.py
 |<<metricbeat-module-dropwizard,Dropwizard>>  beta[]   |image:./images/icon-no.png[No prebuilt dashboards]    |  
 .1+| .1+|  |<<metricbeat-metricset-dropwizard-collector,collector>> beta[]  
 |<<metricbeat-module-elasticsearch,Elasticsearch>>  beta[]   |image:./images/icon-no.png[No prebuilt dashboards]    |  
-.3+| .3+|  |<<metricbeat-metricset-elasticsearch-index,index>> experimental[]  
+.4+| .4+|  |<<metricbeat-metricset-elasticsearch-index,index>> experimental[]  
+|<<metricbeat-metricset-elasticsearch-index_summary,index_summary>> experimental[]  
 |<<metricbeat-metricset-elasticsearch-node,node>> beta[]  
 |<<metricbeat-metricset-elasticsearch-node_stats,node_stats>> beta[]  
 |<<metricbeat-module-etcd,Etcd>>  beta[]   |image:./images/icon-no.png[No prebuilt dashboards]    |  

diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go
@@ -37,6 +37,7 @@ import (
 	_ "github.com/elastic/beats/metricbeat/module/dropwizard/collector"
 	_ "github.com/elastic/beats/metricbeat/module/elasticsearch"
 	_ "github.com/elastic/beats/metricbeat/module/elasticsearch/index"
+	_ "github.com/elastic/beats/metricbeat/module/elasticsearch/index_summary"
 	_ "github.com/elastic/beats/metricbeat/module/elasticsearch/node"
 	_ "github.com/elastic/beats/metricbeat/module/elasticsearch/node_stats"
 	_ "github.com/elastic/beats/metricbeat/module/etcd"

diff --git a/metricbeat/mb/testing/data_generator.go b/metricbeat/mb/testing/data_generator.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"path"
 	"testing"
 	"time"
 
@@ -141,17 +142,17 @@ func StandardizeEvent(ms mb.MetricSet, e mb.Event, modifiers ...mb.EventModifier
 // WriteEventToDataJSON writes the given event as "pretty" JSON to
 // a ./_meta/data.json file. If the -data CLI flag is unset or false then the
 // method is a no-op.
-func WriteEventToDataJSON(t testing.TB, fullEvent beat.Event, pathPostfix string) {
+func WriteEventToDataJSON(t testing.TB, fullEvent beat.Event, postfixPath string) {
 	if !*dataFlag {
 		return
 	}
 
-	path, err := os.Getwd()
+	p, err := os.Getwd()
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	path = path + pathPostfix
+	p = path.Join(p, postfixPath, "_meta", "data.json")
 
 	fields := fullEvent.Fields
 	fields["@timestamp"] = fullEvent.Timestamp
@@ -161,7 +162,7 @@ func WriteEventToDataJSON(t testing.TB, fullEvent beat.Event, pathPostfix string
 		t.Fatal(err)
 	}
 
-	if err = ioutil.WriteFile(path+"/_meta/data.json", output, 0644); err != nil {
+	if err = ioutil.WriteFile(p, output, 0644); err != nil {
 		t.Fatal(err)
 	}
 }
diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml
@@ -197,6 +197,7 @@ metricbeat.modules:
 - module: elasticsearch
   metricsets:
    #- index
+   #- index_summary
    - node
    - node_stats
   period: 10s

diff --git a/metricbeat/module/elasticsearch/_meta/config.reference.yml b/metricbeat/module/elasticsearch/_meta/config.reference.yml
@@ -1,6 +1,7 @@
 - module: elasticsearch
   metricsets:
    #- index
+   #- index_summary
    - node
    - node_stats
   period: 10s

diff --git a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go
@@ -14,12 +14,14 @@ import (
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
 
 	_ "github.com/elastic/beats/metricbeat/module/elasticsearch/index"
+	_ "github.com/elastic/beats/metricbeat/module/elasticsearch/index_summary"
 	_ "github.com/elastic/beats/metricbeat/module/elasticsearch/node"
 	_ "github.com/elastic/beats/metricbeat/module/elasticsearch/node_stats"
 )
 
 var metricSets = []string{
 	"index",
+	"index_summary",
 	"node",
 	"node_stats",
 }
@@ -48,7 +50,7 @@ func TestData(t *testing.T) {
 	for _, metricSet := range metricSets {
 		t.Run(metricSet, func(t *testing.T) {
 			f := mbtest.NewReportingMetricSetV2(t, getConfig(metricSet))
-			err := mbtest.WriteEventsReporterV2(f, t, "/"+metricSet)
+			err := mbtest.WriteEventsReporterV2(f, t, metricSet)
 			if err != nil {
 				t.Fatal("write", err)
 			}

diff --git a/metricbeat/module/elasticsearch/index/_meta/data.json b/metricbeat/module/elasticsearch/index/_meta/data.json
@@ -6,25 +6,25 @@
     },
     "elasticsearch": {
         "cluster": {
-            "id": "EFdYsDSoSZmU7fXE2wvJ0g",
-            "name": "docker-cluster"
+            "id": "UziYVLPkTTmCzccc6102Bg",
+            "name": "elasticsearch"
         },
         "index": {
-            "name": ".ml-anomalies-shared",
+            "name": "filebeat-7.0.0-alpha1-2018.05.09",
             "total": {
                 "docs": {
-                    "count": 7,
-                    "deleted": 2
+                    "count": 1,
+                    "deleted": 0
                 },
                 "segments": {
-                    "count": 6,
+                    "count": 1,
                     "memory": {
-                        "bytes": 10886
+                        "bytes": 6983
                     }
                 },
                 "store": {
                     "size": {
-                        "bytes": 44505
+                        "bytes": 19326
                     }
                 }
             }

diff --git a/metricbeat/module/elasticsearch/index_summary/_meta/data.json b/metricbeat/module/elasticsearch/index_summary/_meta/data.json
@@ -0,0 +1,61 @@
+{
+    "@timestamp": "2017-10-12T08:05:34.853Z",
+    "beat": {
+        "hostname": "host.example.com",
+        "name": "host.example.com"
+    },
+    "elasticsearch": {
+        "cluster": {
+            "id": "UziYVLPkTTmCzccc6102Bg",
+            "name": "elasticsearch"
+        },
+        "index": {
+            "summary": {
+                "primaries": {
+                    "docs": {
+                        "count": 231,
+                        "deleted": 56
+                    },
+                    "segments": {
+                        "count": 16,
+                        "memory": {
+                            "bytes": 105245
+                        }
+                    },
+                    "store": {
+                        "size": {
+                            "bytes": 444882
+                        }
+                    }
+                },
+                "total": {
+                    "docs": {
+                        "count": 231,
+                        "deleted": 56
+                    },
+                    "segments": {
+                        "count": 16,
+                        "memory": {
+                            "bytes": 105245
+                        }
+                    },
+                    "store": {
+                        "size": {
+                            "bytes": 444882
+                        }
+                    }
+                }
+            }
+        }
+    },
+    "metricset": {
+        "host": "127.0.0.1:9200",
+        "module": "elasticsearch",
+        "name": "index_summary",
+        "namespace": "elasticsearch.index.summary",
+        "rtt": 115
+    },
+    "service": {
+        "name": "elasticsearch"
+    }
+}
diff --git a/metricbeat/module/elasticsearch/index_summary/_meta/docs.asciidoc b/metricbeat/module/elasticsearch/index_summary/_meta/docs.asciidoc
@@ -0,0 +1 @@
+This is the index summary metricset of the module elasticsearch.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		This is the index summary metricset of the module elasticsearch.