Add experimental Kibana module to Metricbeat (#3895)

The Kibana module is marked experimental as it might change or removed at any time.

CHANGELOG.asciidoc

@@ -35,6 +35,7 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff]
 
 *Packetbeat*
 - Remove deprecated geoip. {pull}3766[3766]
+- Add experimental Kibana module. {pull}3895[3895]
 
 *Winlogbeat*
 - Remove metrics endpoint. Replaced by http endpoint in libbeat (see #3717). {pull}3901[3901]

metricbeat/docker-compose.yml

@@ -22,6 +22,7 @@ services:
       - ${PWD}/module/haproxy/_meta/env
       - ${PWD}/module/jolokia/_meta/env
       - ${PWD}/module/kafka/_meta/env
+      - ${PWD}/module/kibana/_meta/env
       - ${PWD}/module/memcached/_meta/env
       - ${PWD}/module/mongodb/_meta/env
       - ${PWD}/module/mysql/_meta/env
@@ -43,6 +44,7 @@ services:
       haproxy:    { condition: service_healthy }
       jolokia:    { condition: service_healthy }
       kafka:      { condition: service_healthy }
+      kibana:      { condition: service_healthy }
       memcached:  { condition: service_healthy }
       mongodb:    { condition: service_healthy }
       mysql:      { condition: service_healthy }
@@ -72,6 +74,11 @@ services:
   kafka:
     build: ${PWD}/module/kafka/_meta
 
+  kibana:
+    extends:
+      file: ${ES_BEATS}/testing/environments/${TESTING_ENVIRONMENT}.yml
+      service: kibana
+
   memcached:
     build: ${PWD}/module/memcached/_meta
 

metricbeat/docs/fields.asciidoc

@@ -23,6 +23,7 @@ grouped in the following categories:
 * <<exported-fields-haproxy>>
 * <<exported-fields-jolokia>>
 * <<exported-fields-kafka>>
+* <<exported-fields-kibana>>
 * <<exported-fields-kubelet>>
 * <<exported-fields-memcached>>
 * <<exported-fields-mongodb>>
@@ -3330,6 +3331,97 @@ type: keyword
 Broker address
 
 
+[[exported-fields-kibana]]
+== kibana Fields
+
+[]experimental
+kibana Module
+
+
+
+[float]
+== kibana Fields
+
+
+
+
+[float]
+== status Fields
+
+Status fields
+
+
+
+[float]
+=== kibana.status.name
+
+type: keyword
+
+Kibana instance name.
+
+
+[float]
+=== kibana.status.uuid
+
+type: keyword
+
+Kibana instance uuid.
+
+
+[float]
+=== kibana.status.version.number
+
+type: keyword
+
+Kibana version number.
+
+
+[float]
+=== kibana.status.status.overall.state
+
+type: keyword
+
+Kibana overall state.
+
+
+[float]
+== metrics Fields
+
+Metrics fields
+
+
+
+[float]
+=== kibana.status.metrics.concurrent_connections
+
+type: long
+
+Current concurrent connections.
+
+
+[float]
+== requests Fields
+
+Request statistics.
+
+
+
+[float]
+=== kibana.status.metrics.requests.disconnects
+
+type: long
+
+Total number of disconnected connections.
+
+
+[float]
+=== kibana.status.metrics.requests.total
+
+type: long
+
+Total number of connections.
+
+
 [[exported-fields-kubelet]]
 == kubelet Fields
 

metricbeat/docs/modules/kibana.asciidoc

@@ -0,0 +1,37 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[metricbeat-module-kibana]]
+== kibana Module
+
+This is the kibana Module.
+
+
+
+[float]
+=== Example Configuration
+
+The kibana module supports the standard configuration options that are described
+in <<configuration-metricbeat>>. Here is an example configuration:
+
+[source,yaml]
+----
+metricbeat.modules:
+- module: kibana
+  metricsets: ["status"]
+  enabled: true
+  period: 10s
+  hosts: ["localhost:5601"]
+
+----
+
+[float]
+=== Metricsets
+
+The following metricsets are available:
+
+* <<metricbeat-metricset-kibana-status,status>>
+
+include::kibana/status.asciidoc[]
+

metricbeat/docs/modules/kibana/status.asciidoc

@@ -0,0 +1,19 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[metricbeat-metricset-kibana-status]]
+include::../../../module/kibana/status/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-kibana,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/kibana/status/_meta/data.json[]
+----

metricbeat/docs/modules_list.asciidoc

@@ -10,6 +10,7 @@ This file is generated! See scripts/docs_collector.py
   * <<metricbeat-module-haproxy,HAProxy>>
   * <<metricbeat-module-jolokia,Jolokia>>
   * <<metricbeat-module-kafka,kafka>>
+  * <<metricbeat-module-kibana,kibana>>
   * <<metricbeat-module-kubelet,kubelet>>
   * <<metricbeat-module-memcached,memcached>>
   * <<metricbeat-module-mongodb,MongoDB>>
@@ -34,6 +35,7 @@ include::modules/golang.asciidoc[]
 include::modules/haproxy.asciidoc[]
 include::modules/jolokia.asciidoc[]
 include::modules/kafka.asciidoc[]
+include::modules/kibana.asciidoc[]
 include::modules/kubelet.asciidoc[]
 include::modules/memcached.asciidoc[]
 include::modules/mongodb.asciidoc[]

metricbeat/include/list.go

@@ -39,6 +39,8 @@ import (
 	_ "github.com/elastic/beats/metricbeat/module/kafka"
 	_ "github.com/elastic/beats/metricbeat/module/kafka/consumergroup"
 	_ "github.com/elastic/beats/metricbeat/module/kafka/partition"
+	_ "github.com/elastic/beats/metricbeat/module/kibana"
+	_ "github.com/elastic/beats/metricbeat/module/kibana/status"
 	_ "github.com/elastic/beats/metricbeat/module/kubelet"
 	_ "github.com/elastic/beats/metricbeat/module/kubelet/container"
 	_ "github.com/elastic/beats/metricbeat/module/kubelet/node"

metricbeat/metricbeat.full.yml

@@ -182,6 +182,14 @@ metricbeat.modules:
   #username: ""
   #password: ""
 
+#------------------------------- kibana Module -------------------------------
+- module: kibana
+  metricsets: ["status"]
+  enabled: true
+  period: 10s
+  hosts: ["localhost:5601"]
+
+
 #------------------------------- kubelet Module ------------------------------
 #- module: kubelet
 #  metricsets: ["node","container","volume","pod","system"]

metricbeat/metricbeat.yml

@@ -46,6 +46,14 @@ metricbeat.modules:
   period: 10s
   processes: ['.*']
 
+#------------------------------- kibana Module -------------------------------
+- module: kibana
+  metricsets: ["status"]
+  enabled: true
+  period: 10s
+  hosts: ["localhost:5601"]
+
+
 
 
 #================================ General =====================================

metricbeat/module/kibana/_meta/config.yml

@@ -0,0 +1,6 @@
+- module: kibana
+  metricsets: ["status"]
+  enabled: true
+  period: 10s
+  hosts: ["localhost:5601"]
+

metricbeat/module/kibana/_meta/docs.asciidoc

@@ -0,0 +1,4 @@
+== kibana Module
+
+This is the kibana Module.
+

metricbeat/module/kibana/_meta/env

@@ -0,0 +1,2 @@
+KIBANA_HOST=kibana
+KIBANA_PORT=5601

metricbeat/module/kibana/_meta/fields.yml

@@ -0,0 +1,11 @@
+- key: kibana
+  title: "kibana"
+  description: >
+    []experimental
+
+    kibana Module
+  fields:
+    - name: kibana
+      type: group
+      description: >
+      fields:

metricbeat/module/kibana/doc.go

@@ -0,0 +1,4 @@
+/*
+Package kibana is a Metricbeat module that contains MetricSets.
+*/
+package kibana

metricbeat/module/kibana/status/_meta/data.json

@@ -0,0 +1,35 @@
+{
+    "@timestamp": "2016-05-23T08:05:34.853Z",
+    "beat": {
+        "hostname": "host.example.com",
+        "name": "host.example.com"
+    },
+    "kibana": {
+        "status": {
+            "metrics": {
+                "concurrent_connections": 0,
+                "requests": {
+                    "disconnects": 0,
+                    "total": 0
+                }
+            },
+            "name": "kibana",
+            "status": {
+                "overall": {
+                    "state": "red"
+                }
+            },
+            "uuid": "c94305f7-ad4e-4e0c-8aba-5c3e869dce59",
+            "version": {
+                "number": "6.0.0-alpha1"
+            }
+        }
+    },
+    "metricset": {
+        "host": "kibana:5601",
+        "module": "kibana",
+        "name": "status",
+        "rtt": 115
+    },
+    "type": "metricsets"
+}

metricbeat/module/kibana/status/_meta/docs.asciidoc

@@ -0,0 +1,7 @@
+=== kibana status MetricSet
+
+[]experimental
+
+This is the status metricset of the module kibana. This status endpoint is available in 6.0 by default and can be enabled in Kibana >= 5.4 with the config option `status.v6ApiFormat: true`.
+
+The indention of this Kibana module is to have a minimal data set that works across Kibana versions.

metricbeat/module/kibana/status/_meta/fields.yml

@@ -0,0 +1,43 @@
+- name: status
+  type: group
+  description: >
+    Status fields
+  fields:
+    - name: name
+      type: keyword
+      description: >
+        Kibana instance name.
+    - name: uuid
+      type: keyword
+      description: >
+        Kibana instance uuid.
+    - name: version.number
+      type: keyword
+      description: >
+        Kibana version number.
+    - name: status.overall.state
+      type: keyword
+      description: >
+        Kibana overall state.
+    - name: metrics
+      type: group
+      description: >
+        Metrics fields
+      fields:
+        - name: concurrent_connections
+          type: long
+          description: >
+            Current concurrent connections.
+        - name: requests
+          type: group
+          description: >
+            Request statistics.
+          fields:
+            - name: disconnects
+              type: long
+              description: >
+                Total number of disconnected connections.
+            - name: total
+              type: long
+              description: >
+                Total number of connections.