Add Mongo dbstats metricset to Metricbeat (#3228)

* set FailFast = True for direct connections to prevent a nonresponsive node from holding up the reporting * new implementation that establishes direct connetions to mongo hosts * successful reporting using direct node connections * implementing multi-node direct reporting for mongodb.serverStatus
elastic · Jan 11, 2017 · 2ddb800 · 2ddb800
1 parent 2bcf963
commit 2ddb800
Show file tree

Hide file tree

Showing 19 changed files with 640 additions and 21 deletions.
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -61,6 +61,19 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff]
 - Updated to Go 1.7.4. {pull}3277[3277]
 
 *Metricbeat*
+
+- Add experimental filebeat metricset in the beats module. {pull}2297[2297]
+- Add experimental libbeat metricset in the beats module. {pull}2339[2339]
+- Add experimental docker module. Provided by Ingensi and @douaejeouit based on dockbeat.
+- Add username and password config options to the MongoDB module. {pull}2889[2889]
+- Add username and password config options to the PostgreSQL module. {pull}2889[2890]
+- Add system core metricset for Windows. {pull}2883[2883]
+- Add a sample Redis Kibana dashboard. {pull}2916[2916]
+- Add support for MongoDB 3.4 and WiredTiger metrics. {pull}2999[2999]
+- Add experimental kafka module with partition metricset. {pull}2969[2969]
+- Add raw config option for mysql/status metricset. {pull}3001[3001]
+- Add experimental dbstats metricset to MongoDB module. {pull}3228[3228]
+- Use persistent, direct connections to the configured nodes for MongoDB module. {pull}3228[3228]
 - Kafka module broker matching enhancements. {pull}3129[3129]
 - Add a couchbase module with metricsets for node, cluster and bucket. {pull}3081[3081]
 - Export number of cores for cpu module. {pull}3192[3192]

diff --git a/metricbeat/_meta/beat.full.yml b/metricbeat/_meta/beat.full.yml
@@ -147,7 +147,7 @@ metricbeat.modules:
 
 #------------------------------- MongoDB Module ------------------------------
 #- module: mongodb
-  #metricsets: ["status"]
+  #metricsets: ["dbstats", "status"]
   #enabled: true
   #period: 10s
 
@@ -164,7 +164,6 @@ metricbeat.modules:
   # Password to use when connecting to MongoDB. Empty by default.
   #password: pass
 
-
 #-------------------------------- MySQL Module -------------------------------
 #- module: mysql
   #metricsets: ["status"]

diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -2636,6 +2636,102 @@ MongoDB metrics.
 
 
 
+[float]
+== dbstats Fields
+
+dbstats provides an overview of a particular mongo database. This document is most concerned with data volumes of a database.
+
+
+
+[float]
+=== mongodb.dbstats.avg_obj_size.bytes
+
+type: long
+
+format: bytes
+
+[float]
+=== mongodb.dbstats.collections
+
+type: integer
+
+[float]
+=== mongodb.dbstats.data_size.bytes
+
+type: long
+
+format: bytes
+
+[float]
+=== mongodb.dbstats.db
+
+type: keyword
+
+[float]
+=== mongodb.dbstats.file_size.bytes
+
+type: long
+
+format: bytes
+
+[float]
+=== mongodb.dbstats.index_size.bytes
+
+type: long
+
+format: bytes
+
+[float]
+=== mongodb.dbstats.indexes
+
+type: long
+
+[float]
+=== mongodb.dbstats.num_extents
+
+type: long
+
+[float]
+=== mongodb.dbstats.objects
+
+type: long
+
+[float]
+=== mongodb.dbstats.storage_size.bytes
+
+type: long
+
+format: bytes
+
+[float]
+=== mongodb.dbstats.ns_size_mb.mb
+
+type: long
+
+
+[float]
+=== mongodb.dbstats.data_file_version.major
+
+type: long
+
+[float]
+=== mongodb.dbstats.data_file_version.minor
+
+type: long
+
+
+[float]
+=== mongodb.dbstats.extent_free_list.num
+
+type: long
+
+[float]
+=== mongodb.dbstats.extent_free_list.size.bytes
+
+type: long
+
+format: bytes
+
 [float]
 == status Fields
 

diff --git a/metricbeat/docs/modules/mongodb.asciidoc b/metricbeat/docs/modules/mongodb.asciidoc
@@ -64,7 +64,7 @@ in <<configuration-metricbeat>>. Here is an example configuration:
 ----
 metricbeat.modules:
 #- module: mongodb
-  #metricsets: ["status"]
+  #metricsets: ["dbstats", "status"]
   #enabled: true
   #period: 10s
 
@@ -80,15 +80,18 @@ metricbeat.modules:
 
   # Password to use when connecting to MongoDB. Empty by default.
   #password: pass
-
 ----
 
 [float]
 === Metricsets
 
 The following metricsets are available:
 
+* <<metricbeat-metricset-mongodb-dbstats,dbstats>>
+
 * <<metricbeat-metricset-mongodb-status,status>>
 
+include::mongodb/dbstats.asciidoc[]
+
 include::mongodb/status.asciidoc[]
 
diff --git a/metricbeat/docs/modules/mongodb/dbstats.asciidoc b/metricbeat/docs/modules/mongodb/dbstats.asciidoc
@@ -0,0 +1,19 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[metricbeat-metricset-mongodb-dbstats]]
+include::../../../module/mongodb/dbstats/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-mongodb,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/mongodb/dbstats/_meta/data.json[]
+----
diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go
@@ -28,6 +28,7 @@ import (
 	_ "github.com/elastic/beats/metricbeat/module/kafka/consumergroup"
 	_ "github.com/elastic/beats/metricbeat/module/kafka/partition"
 	_ "github.com/elastic/beats/metricbeat/module/mongodb"
+	_ "github.com/elastic/beats/metricbeat/module/mongodb/dbstats"
 	_ "github.com/elastic/beats/metricbeat/module/mongodb/status"
 	_ "github.com/elastic/beats/metricbeat/module/mysql"
 	_ "github.com/elastic/beats/metricbeat/module/mysql/status"

diff --git a/metricbeat/metricbeat.full.yml b/metricbeat/metricbeat.full.yml
@@ -147,7 +147,7 @@ metricbeat.modules:
 
 #------------------------------- MongoDB Module ------------------------------
 #- module: mongodb
-  #metricsets: ["status"]
+  #metricsets: ["dbstats", "status"]
   #enabled: true
   #period: 10s
 
@@ -164,7 +164,6 @@ metricbeat.modules:
   # Password to use when connecting to MongoDB. Empty by default.
   #password: pass
 
-
 #-------------------------------- MySQL Module -------------------------------
 #- module: mysql
   #metricsets: ["status"]

diff --git a/metricbeat/metricbeat.template-es2x.json b/metricbeat/metricbeat.template-es2x.json
@@ -1510,6 +1510,93 @@
         },
         "mongodb": {
           "properties": {
+            "dbstats": {
+              "properties": {
+                "avg_obj_size": {
+                  "properties": {
+                    "bytes": {
+                      "type": "long"
+                    }
+                  }
+                },
+                "collections": {
+                  "type": "long"
+                },
+                "data_file_version": {
+                  "properties": {
+                    "major": {
+                      "type": "long"
+                    },
+                    "minor": {
+                      "type": "long"
+                    }
+                  }
+                },
+                "data_size": {
+                  "properties": {
+                    "bytes": {
+                      "type": "long"
+                    }
+                  }
+                },
+                "db": {
+                  "ignore_above": 1024,
+                  "index": "not_analyzed",
+                  "type": "string"
+                },
+                "extent_free_list": {
+                  "properties": {
+                    "num": {
+                      "type": "long"
+                    },
+                    "size": {
+                      "properties": {
+                        "bytes": {
+                          "type": "long"
+                        }
+                      }
+                    }
+                  }
+                },
+                "file_size": {
+                  "properties": {
+                    "bytes": {
+                      "type": "long"
+                    }
+                  }
+                },
+                "index_size": {
+                  "properties": {
+                    "bytes": {
+                      "type": "long"
+                    }
+                  }
+                },
+                "indexes": {
+                  "type": "long"
+                },
+                "ns_size_mb": {
+                  "properties": {
+                    "mb": {
+                      "type": "long"
+                    }
+                  }
+                },
+                "num_extents": {
+                  "type": "long"
+                },
+                "objects": {
+                  "type": "long"
+                },
+                "storage_size": {
+                  "properties": {
+                    "bytes": {
+                      "type": "long"
+                    }
+                  }
+                }
+              }
+            },
             "status": {
               "properties": {
                 "asserts": {