Add Prometheus module to Metricbeat

This first version implements very basic stats from the prometheus service itself.
elastic · Dec 15, 2016 · d2e3fcd · d2e3fcd
1 parent 241df11
commit d2e3fcd
Show file tree

Hide file tree

Showing 23 changed files with 503 additions and 4 deletions.
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -55,7 +55,7 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff]
 - Kafka module broker matching enhancements. {pull}3129[3129]
 - Add a couchbase module with metricsets for node, cluster and bucker. {pull}3081[3081]
 - Export number of cores for cpu module. {pull}3192[3192]
-
+- Experimental Prometheus module. {pull}3202[3202]
 
 *Packetbeat*
 

diff --git a/metricbeat/_meta/beat.full.yml b/metricbeat/_meta/beat.full.yml
@@ -202,6 +202,13 @@ metricbeat.modules:
   #password: pass
 
 
+#----------------------------- Prometheus Module -----------------------------
+#- module: prometheus
+  #metricsets: ["stats"]
+  #enabled: true
+  #period: 10s
+  #hosts: ["localhost:9090"]
+
 #-------------------------------- Redis Module -------------------------------
 #- module: redis
   #metricsets: ["info", "keyspace"]

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
@@ -6,19 +6,22 @@ services:
       - apache
       - couchbase
       - mongodb
+      - haproxy
       - kafka
       - mysql
       - nginx
       - postgresql
+      - prometheus
       - redis
       - zookeeper
-      - haproxy
     environment:
       - APACHE_HOST=apache
       - APACHE_PORT=80
       - COUCHBASE_HOST=couchbase
       - COUCHBASE_PORT=8091
       - COUCHBASE_DSN=http://Administrator:password@couchbase:8091
+      - HAPROXY_HOST=haproxy
+      - HAPROXY_PORT=14567
       - KAFKA_HOST=kafka
       - KAFKA_PORT=9092
       - NGINX_HOST=nginx
@@ -34,10 +37,10 @@ services:
       - POSTGRESQL_HOST=postgresql
       - POSTGRESQL_PORT=5432
       - POSTGRESQL_USERNAME=postgres
+      - PROMETHEUS_HOST=prometheus
+      - PROMETHEUS_PORT=9090
       - ZOOKEEPER_HOST=zookeeper
       - ZOOKEEPER_PORT=2181
-      - HAPROXY_HOST=haproxy
-      - HAPROXY_PORT=14567
       - TEST_ENVIRONMENT=false
     working_dir: /go/src/github.com/elastic/beats/metricbeat
     volumes:
@@ -82,6 +85,11 @@ services:
   postgresql:
     image: postgres:9.5.3
 
+  prometheus:
+    image: prom/prometheus
+    expose:
+      - 9090
+
   redis:
     image: redis:3.2.4-alpine
 

diff --git a/metricbeat/docker-entrypoint.sh b/metricbeat/docker-entrypoint.sh
@@ -29,6 +29,7 @@ waitFor ${KAFKA_HOST} ${KAFKA_PORT} Kafka
 waitFor ${MYSQL_HOST} ${MYSQL_PORT} MySQL
 waitFor ${NGINX_HOST} ${NGINX_PORT} Nginx
 waitFor ${POSTGRESQL_HOST} ${POSTGRESQL_PORT} Postgresql
+waitFor ${PROMETHEUS_HOST} ${PROMETHEUS_PORT} Prometheus
 waitFor ${REDIS_HOST} ${REDIS_PORT} Redis
 waitFor ${ZOOKEEPER_HOST} ${ZOOKEEPER_PORT} Zookeeper
 exec "$@"
diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -24,6 +24,7 @@ grouped in the following categories:
 * <<exported-fields-mysql>>
 * <<exported-fields-nginx>>
 * <<exported-fields-postgresql>>
+* <<exported-fields-prometheus>>
 * <<exported-fields-redis>>
 * <<exported-fields-system>>
 * <<exported-fields-zookeeper>>
@@ -3956,6 +3957,66 @@ type: date
 Time at which these statistics were last reset.
 
 
+[[exported-fields-prometheus]]
+== Prometheus Fields
+
+prometheus Module
+experimental[]
+
+
+
+[float]
+== prometheus Fields
+
+
+
+
+[float]
+== stats Fields
+
+stats
+
+
+
+[float]
+== notifications Fields
+
+Notification stats
+
+
+
+[float]
+=== prometheus.stats.notifications.queue_length
+
+type: long
+
+Current queue length
+
+
+[float]
+=== prometheus.stats.notifications.dropped
+
+type: long
+
+Dropped queue events
+
+
+[float]
+=== prometheus.stats.processes.open_fds
+
+type: long
+
+Open file descriptors gauge
+
+
+[float]
+=== prometheus.stats.storage.chunks_to_persist
+
+type: long
+
+Gauge on chunks which are not persisted to disk yet
+
+
 [[exported-fields-redis]]
 == Redis Fields
 

diff --git a/metricbeat/docs/modules/prometheus.asciidoc b/metricbeat/docs/modules/prometheus.asciidoc
@@ -0,0 +1,36 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[metricbeat-module-prometheus]]
+== prometheus Module
+
+This is the prometheus Module.
+
+
+
+[float]
+=== Example Configuration
+
+The Prometheus module supports the standard configuration options that are described
+in <<configuration-metricbeat>>. Here is an example configuration:
+
+[source,yaml]
+----
+metricbeat.modules:
+#- module: prometheus
+  #metricsets: ["stats"]
+  #enabled: true
+  #period: 10s
+  #hosts: ["localhost:9090"]
+----
+
+[float]
+=== Metricsets
+
+The following metricsets are available:
+
+* <<metricbeat-metricset-prometheus-stats,stats>>
+
+include::prometheus/stats.asciidoc[]
+
diff --git a/metricbeat/docs/modules/prometheus/stats.asciidoc b/metricbeat/docs/modules/prometheus/stats.asciidoc
@@ -0,0 +1,19 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[metricbeat-metricset-prometheus-stats]]
+include::../../../module/prometheus/stats/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-prometheus,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/prometheus/stats/_meta/data.json[]
+----
diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc
@@ -11,6 +11,7 @@ This file is generated! See scripts/docs_collector.py
   * <<metricbeat-module-mysql,MySQL>>
   * <<metricbeat-module-nginx,Nginx>>
   * <<metricbeat-module-postgresql,PostgreSQL>>
+  * <<metricbeat-module-prometheus,Prometheus>>
   * <<metricbeat-module-redis,Redis>>
   * <<metricbeat-module-system,System>>
   * <<metricbeat-module-zookeeper,ZooKeeper>>
@@ -27,6 +28,7 @@ include::modules/mongodb.asciidoc[]
 include::modules/mysql.asciidoc[]
 include::modules/nginx.asciidoc[]
 include::modules/postgresql.asciidoc[]
+include::modules/prometheus.asciidoc[]
 include::modules/redis.asciidoc[]
 include::modules/system.asciidoc[]
 include::modules/zookeeper.asciidoc[]
diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go
@@ -36,6 +36,8 @@ import (
 	_ "github.com/elastic/beats/metricbeat/module/postgresql/activity"
 	_ "github.com/elastic/beats/metricbeat/module/postgresql/bgwriter"
 	_ "github.com/elastic/beats/metricbeat/module/postgresql/database"
+	_ "github.com/elastic/beats/metricbeat/module/prometheus"
+	_ "github.com/elastic/beats/metricbeat/module/prometheus/stats"
 	_ "github.com/elastic/beats/metricbeat/module/redis"
 	_ "github.com/elastic/beats/metricbeat/module/redis/info"
 	_ "github.com/elastic/beats/metricbeat/module/redis/keyspace"

diff --git a/metricbeat/metricbeat.full.yml b/metricbeat/metricbeat.full.yml
@@ -202,6 +202,13 @@ metricbeat.modules:
   #password: pass
 
 
+#----------------------------- Prometheus Module -----------------------------
+#- module: prometheus
+  #metricsets: ["stats"]
+  #enabled: true
+  #period: 10s
+  #hosts: ["localhost:9090"]
+
 #-------------------------------- Redis Module -------------------------------
 #- module: redis
   #metricsets: ["info", "keyspace"]

diff --git a/metricbeat/metricbeat.template-es2x.json b/metricbeat/metricbeat.template-es2x.json
@@ -2225,6 +2225,38 @@
             }
           }
         },
+        "prometheus": {
+          "properties": {
+            "stats": {
+              "properties": {
+                "notifications": {
+                  "properties": {
+                    "dropped": {
+                      "type": "long"
+                    },
+                    "queue_length": {
+                      "type": "long"
+                    }
+                  }
+                },
+                "processes": {
+                  "properties": {
+                    "open_fds": {
+                      "type": "long"
+                    }
+                  }
+                },
+                "storage": {
+                  "properties": {
+                    "chunks_to_persist": {
+                      "type": "long"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        },
         "redis": {
           "properties": {
             "info": {

diff --git a/metricbeat/metricbeat.template.json b/metricbeat/metricbeat.template.json
@@ -2210,6 +2210,38 @@
             }
           }
         },
+        "prometheus": {
+          "properties": {
+            "stats": {
+              "properties": {
+                "notifications": {
+                  "properties": {
+                    "dropped": {
+                      "type": "long"
+                    },
+                    "queue_length": {
+                      "type": "long"
+                    }
+                  }
+                },
+                "processes": {
+                  "properties": {
+                    "open_fds": {
+                      "type": "long"
+                    }
+                  }
+                },
+                "storage": {
+                  "properties": {
+                    "chunks_to_persist": {
+                      "type": "long"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        },
         "redis": {
           "properties": {
             "info": {

diff --git a/metricbeat/module/prometheus/_meta/config.yml b/metricbeat/module/prometheus/_meta/config.yml
@@ -0,0 +1,5 @@
+#- module: prometheus
+  #metricsets: ["stats"]
+  #enabled: true
+  #period: 10s
+  #hosts: ["localhost:9090"]
diff --git a/metricbeat/module/prometheus/_meta/docs.asciidoc b/metricbeat/module/prometheus/_meta/docs.asciidoc
@@ -0,0 +1,4 @@
+== prometheus Module
+
+This is the prometheus Module.
+
diff --git a/metricbeat/module/prometheus/_meta/fields.yml b/metricbeat/module/prometheus/_meta/fields.yml
@@ -0,0 +1,12 @@
+- key: prometheus
+  title: "Prometheus"
+  description: >
+    prometheus Module
+
+    experimental[]
+  short_config: false
+  fields:
+    - name: prometheus
+      type: group
+      description: >
+      fields:
diff --git a/metricbeat/module/prometheus/doc.go b/metricbeat/module/prometheus/doc.go
@@ -0,0 +1,4 @@
+/*
+Package prometheus is a Metricbeat module that contains MetricSets.
+*/
+package prometheus
diff --git a/metricbeat/module/prometheus/stats/_meta/data.json b/metricbeat/module/prometheus/stats/_meta/data.json
@@ -0,0 +1,28 @@
+{
+    "@timestamp": "2016-05-23T08:05:34.853Z",
+    "beat": {
+        "hostname": "host.example.com",
+        "name": "host.example.com"
+    },
+    "metricset": {
+        "host": "127.0.0.1:9090",
+        "module": "prometheus",
+        "name": "stats",
+        "rtt": 115
+    },
+    "prometheus": {
+        "stats": {
+            "notifications": {
+                "dropped": 0,
+                "queue_length": 0
+            },
+            "processes": {
+                "open_fds": 24
+            },
+            "storage": {
+                "chunks_to_persist": 465
+            }
+        }
+    },
+    "type": "metricsets"
+}
diff --git a/metricbeat/module/prometheus/stats/_meta/docs.asciidoc b/metricbeat/module/prometheus/stats/_meta/docs.asciidoc
@@ -0,0 +1,3 @@
+=== prometheus stats MetricSet
+
+This is the stats metricset of the module prometheus.