Add a generic jmx/jolokia metricset to metricbeat

metricbeat/_meta/beat.full.yml

@@ -121,6 +121,14 @@ metricbeat.modules:
   #period: 10s
   #hosts: ["tcp://127.0.0.1:14567"]
 
+#------------------------------- Jolokia Module ------------------------------
+#- module: jolokia
+  #metricsets: ["dynamic"]
+  #enabled: true
+  #period: 10s
+  #hosts: ["localhost"]
+  #namespace: "jolokia_metrics"
+
 #-------------------------------- kafka Module -------------------------------
 #- module: kafka
   #metricsets: ["partition"]

metricbeat/docs/fields.asciidoc

@@ -19,6 +19,7 @@ grouped in the following categories:
 * <<exported-fields-couchbase>>
 * <<exported-fields-docker>>
 * <<exported-fields-haproxy>>
+* <<exported-fields-jolokia>>
 * <<exported-fields-kafka>>
 * <<exported-fields-mongodb>>
 * <<exported-fields-mysql>>
@@ -2289,7 +2290,7 @@ Number of data transfers aborted by the server. This value is included in haprox
 
 type: integer
 
-Number of backend servers that are active, meaning that they are healthy and can receive requests from the load balancer. 
+Number of backend servers that are active, meaning that they are healthy and can receive requests from the load balancer.
 
 
 [float]
@@ -2390,6 +2391,47 @@ type: integer
 The average queue time in ms over the last 1024 requests.
 
 
+[[exported-fields-jolokia]]
+== Jolokia Fields
+
+Jolokia Module
+
+
+
+[float]
+== jolokia Fields
+
+
+
+
+[float]
+== dynamic Fields
+
+dynamic
+
+
+[float]
+=== jolokia.dynamic._namespace
+
+type: keyword
+
+Namespace
+
+[float]
+=== jolokia.dynamic.application
+
+type: keyword
+
+Monitored Application
+
+[float]
+=== jolokia.dynamic.instance
+
+type: keyword
+
+Monitored Application Instance
+
+
 [[exported-fields-kafka]]
 == kafka Fields
 

metricbeat/docs/modules/jolokia.asciidoc

@@ -0,0 +1,38 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[metricbeat-module-jolokia]]
+== jolokia Module
+
+This is the jolokia Module.
+
+
+
+[float]
+=== Example Configuration
+
+The Jolokia module supports the standard configuration options that are described
+in <<configuration-metricbeat>>. Here is an example configuration:
+
+[source,yaml]
+----
+metricbeat.modules:
+- module: jolokia
+  metricsets: ["dynamic"]
+  enabled: true
+  period: 10s
+  hosts: ["localhost"]
+  namespace: "jolokia_metrics"
+
+----
+
+[float]
+=== Metricsets
+
+The following metricsets are available:
+
+* <<metricbeat-metricset-jolokia-dynamic,dynamic>>
+
+include::jolokia/dynamic.asciidoc[]
+

metricbeat/docs/modules/jolokia/dynamic.asciidoc

@@ -0,0 +1,19 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[metricbeat-metricset-jolokia-dynamic]]
+include::../../../module/jolokia/dynamic/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-jolokia,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/jolokia/dynamic/_meta/data.json[]
+----

metricbeat/docs/modules_list.asciidoc

@@ -6,6 +6,7 @@ This file is generated! See scripts/docs_collector.py
   * <<metricbeat-module-couchbase,couchbase>>
   * <<metricbeat-module-docker,Docker>>
   * <<metricbeat-module-haproxy,HAProxy>>
+  * <<metricbeat-module-jolokia,Jolokia>>
   * <<metricbeat-module-kafka,kafka>>
   * <<metricbeat-module-mongodb,MongoDB>>
   * <<metricbeat-module-mysql,MySQL>>
@@ -23,6 +24,7 @@ include::modules/apache.asciidoc[]
 include::modules/couchbase.asciidoc[]
 include::modules/docker.asciidoc[]
 include::modules/haproxy.asciidoc[]
+include::modules/jolokia.asciidoc[]
 include::modules/kafka.asciidoc[]
 include::modules/mongodb.asciidoc[]
 include::modules/mysql.asciidoc[]

metricbeat/include/list.go

@@ -24,6 +24,8 @@ import (
 	_ "github.com/elastic/beats/metricbeat/module/haproxy"
 	_ "github.com/elastic/beats/metricbeat/module/haproxy/info"
 	_ "github.com/elastic/beats/metricbeat/module/haproxy/stat"
+	_ "github.com/elastic/beats/metricbeat/module/jolokia"
+	_ "github.com/elastic/beats/metricbeat/module/jolokia/dynamic"
 	_ "github.com/elastic/beats/metricbeat/module/kafka"
 	_ "github.com/elastic/beats/metricbeat/module/kafka/consumergroup"
 	_ "github.com/elastic/beats/metricbeat/module/kafka/partition"

metricbeat/metricbeat.full.yml

@@ -121,6 +121,14 @@ metricbeat.modules:
   #period: 10s
   #hosts: ["tcp://127.0.0.1:14567"]
 
+#------------------------------- Jolokia Module ------------------------------
+#- module: jolokia
+  #metricsets: ["dynamic"]
+  #enabled: true
+  #period: 10s
+  #hosts: ["localhost"]
+  #namespace: "jolokia_metrics"
+
 #-------------------------------- kafka Module -------------------------------
 #- module: kafka
   #metricsets: ["partition"]

metricbeat/metricbeat.template-es2x.json

@@ -1309,6 +1309,26 @@
             }
           }
         },
+        "jolokia": {
+          "properties": {
+            "dynamic": {
+              "properties": {
+                "_namespace": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "application": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "instance": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                }
+              }
+            }
+          }
+        },
         "kafka": {
           "properties": {
             "consumergroup": {
@@ -3887,4 +3907,4 @@
     "index.refresh_interval": "5s"
   },
   "template": "metricbeat-*"
-}
+}

metricbeat/metricbeat.template.json

@@ -1313,6 +1313,26 @@
             }
           }
         },
+        "jolokia": {
+          "properties": {
+            "dynamic": {
+              "properties": {
+                "_namespace": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "application": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "instance": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                }
+              }
+            }
+          }
+        },
         "kafka": {
           "properties": {
             "consumergroup": {
@@ -3839,4 +3859,4 @@
     "index.refresh_interval": "5s"
   },
   "template": "metricbeat-*"
-}
+}

metricbeat/module/jolokia/_meta/config.yml

@@ -0,0 +1,7 @@
+- module: jolokia
+  metricsets: ["dynamic"]
+  enabled: true
+  period: 10s
+  hosts: ["localhost"]
+  namespace: "jolokia_metrics"
+

metricbeat/module/jolokia/_meta/docs.asciidoc

@@ -0,0 +1,4 @@
+== jolokia Module
+
+This is the jolokia Module.
+

metricbeat/module/jolokia/_meta/fields.yml

@@ -0,0 +1,10 @@
+- key: jolokia
+  title: "Jolokia"
+  description: >
+    Jolokia Module
+  fields:
+    - name: jolokia
+      type: group
+      description: >
+          jolokia contains metrics exposed via jolokia agent
+      fields:

metricbeat/module/jolokia/doc.go

@@ -0,0 +1,4 @@
+/*
+Package jolokia is a Metricbeat module that contains MetricSets.
+*/
+package jolokia

metricbeat/module/jolokia/dynamic/_meta/data.json

@@ -0,0 +1,36 @@
+{
+    "@timestamp": "2016-08-27T20:26:21.837Z",
+    "beat": {
+        "hostname": "seu",
+        "name": "seu"
+    },
+    "jolokia": {
+        "dynamic": {
+            "gc": {
+                "cms_collection_count": 3,
+                "cms_collection_time": 207
+            },
+            "memory": {
+                "heap_usage": {
+                    "committed": 1.052770304e+09,
+                    "init": 1.073741824e+09,
+                    "max": 1.052770304e+09,
+                    "used": 1.73407544e+08
+                },
+                "non_heap_usage": {
+                    "committed": 7.0098944e+07,
+                    "init": 2.555904e+06,
+                    "max": -1,
+                    "used": 6.8390328e+07
+                }
+            },
+            "uptime": 6.964576e+06
+        }
+    },
+    "metricset": {
+        "module": "jolokia",
+        "name": "dynamic",
+        "rtt": 53279
+    },
+    "type": "metricsets"
+}

metricbeat/module/jolokia/dynamic/_meta/docs.asciidoc

@@ -0,0 +1,48 @@
+=== jolokia dynamic MetricSet
+
+This is the dynamic metricset of the module jolokia.
+
+[float]
+=== Features and configuration
+Tested with Jolokia 1.3.4.
+
+Metrics to be collected from each Jolokia instance are defined in the mapping section with an MBean ObjectName and
+an array of Attributes to be requested with Elastic field names under which the return values should be saved.
+
+For example: to get the "Uptime" attribute from the "java.lang:type=Runtime" MBean and map it to something like
+"uptime" (actually "jolokia.dynamic.uptime", the prexif is added by beats framework) you have to configure following
+mapping:
+....
+ - mbean: 'java.lang:type=Runtime'
+   attributes:
+      - attr: Uptime
+        field: uptime
+....
+
+In case the underlying attribute is an object (e.g. see HeapMemoryUsage attribute in java.lang:type=Memory) it`s
+structure will be published to Elastic "as is".
+
+It is possible to configure nested metric aliases by using dots in the mapping name, although
+nesting depth is limited to 1 at the moment (e.g. gc.cms_collection_time). For examples please refer to the
+/jolokia/dynamic/test/config.yml.
+
+All metrics from a single mapping will be POSTed to the defined host/port and sent to Elastic as a single event.
+To make it possible to differentiate between metrics from multiple similar applications running on the same host, please
+ use fields "application" and "instance" in the config file. These two will be added into the event under jolokia
+ .dynamic.application and jolokia.dynamic.instance if set.
+
+It is required to set a namespace in the general module config section.
+
+[float]
+=== Limitations
+No authentication against Jolokia is supported yet.
+No wildcards in Jolokia requests supported yet.
+You can get max 30 attributes from the same MBean (see Add function in config.go).
+All Jolokia requests have canonicalNaming set to false (details see here: https://jolokia.org/reference/html/protocol
+.html).
+
+
+[float]
+=== Exposed fields, Dashboards, Indexes, etc.
+Since this is a very general module that can be tailored for any application that exposes it`s metrics over Jolokia, it
+comes with no exposed fields description, dashboards or index patterns.

metricbeat/module/jolokia/dynamic/_meta/fields.yml

@@ -0,0 +1,18 @@
+- name: dynamic
+  type: group
+  description: >
+    Dynamically configured metrics
+  fields:
+    - name: _namespace
+      type: keyword
+      description: >
+        Namespace
+  fields:
+    - name: application
+      type: keyword
+      description: >
+        Monitored Application
+    - name: instance
+      type: keyword
+      description: >
+        Monitored Application Instance