From 9d5b11b249cb8fd3d772e97ff7399297795bf5e6 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Tue, 14 Feb 2017 09:19:45 +0100 Subject: [PATCH] Jolokia Module with dynamic JMX Metricset (#3570) This is the implementation of a module for Jolokia which contains a dynamic jmx metricset. An example configuration looks as following: ``` - module: jolokia metricsets: ["jmx"] enabled: true period: 1s hosts: ["localhost:8778"] namespace: "metrics" jmx.mappings: - mbean: 'java.lang:type=Runtime' attributes: - attr: Uptime field: uptime - mbean: 'java.lang:type=GarbageCollector,name=ConcurrentMarkSweep' attributes: - attr: CollectionTime field: gc.cms_collection_time - attr: CollectionCount field: gc.cms_collection_count - mbean: 'java.lang:type=Memory' attributes: - attr: HeapMemoryUsage field: memory.heap_usage - attr: NonHeapMemoryUsage field: memory.non_heap_usage ``` For each mbeat the attributes which should be fetched can be defined. The field defines under which field name the event will be put. The namespace defines the metricset namespace. This PR replaces https://github.com/elastic/beats/pull/3051 Further changes: * Added support for method and body to http helper * Handle empty fields in generators. This happens for a module which only contains dynamic metricsets which is currently the case for jolokia. TODO: * [x] Add system tests * [x] Check documentation * [x] Add integration test * [ ] Open issue for metricset which contains basic memory info --- CHANGELOG.asciidoc | 1 + libbeat/scripts/generate_index_pattern.py | 4 + libbeat/scripts/generate_template.py | 4 + libbeat/tests/system/beat/beat.py | 4 + metricbeat/docker-compose.yml | 5 + metricbeat/docs/fields.asciidoc | 15 +++ metricbeat/docs/modules/jolokia.asciidoc | 43 +++++++++ metricbeat/docs/modules/jolokia/jmx.asciidoc | 19 ++++ metricbeat/docs/modules_list.asciidoc | 2 + metricbeat/helper/http.go | 22 ++++- metricbeat/include/list.go | 2 + metricbeat/metricbeat.full.yml | 12 +++ metricbeat/module/jolokia/_meta/Dockerfile | 10 ++ metricbeat/module/jolokia/_meta/config.yml | 10 ++ metricbeat/module/jolokia/_meta/docs.asciidoc | 6 ++ metricbeat/module/jolokia/_meta/env | 2 + metricbeat/module/jolokia/_meta/fields.yml | 13 +++ metricbeat/module/jolokia/doc.go | 4 + metricbeat/module/jolokia/jmx/_meta/data.json | 34 +++++++ .../module/jolokia/jmx/_meta/docs.asciidoc | 49 ++++++++++ .../module/jolokia/jmx/_meta/fields.yml | 0 .../module/jolokia/jmx/_meta/test/config.yml | 68 +++++++++++++ .../jmx/_meta/test/jolokia_response.json | 56 +++++++++++ metricbeat/module/jolokia/jmx/config.go | 61 ++++++++++++ metricbeat/module/jolokia/jmx/data.go | 87 +++++++++++++++++ metricbeat/module/jolokia/jmx/data_test.go | 44 +++++++++ metricbeat/module/jolokia/jmx/jmx.go | 96 +++++++++++++++++++ .../jolokia/jmx/jmx_integration_test.go | 93 ++++++++++++++++++ .../tests/system/config/metricbeat.yml.j2 | 7 ++ metricbeat/tests/system/test_jolokia.py | 44 +++++++++ 30 files changed, 816 insertions(+), 1 deletion(-) create mode 100644 metricbeat/docs/modules/jolokia.asciidoc create mode 100644 metricbeat/docs/modules/jolokia/jmx.asciidoc create mode 100644 metricbeat/module/jolokia/_meta/Dockerfile create mode 100644 metricbeat/module/jolokia/_meta/config.yml create mode 100644 metricbeat/module/jolokia/_meta/docs.asciidoc create mode 100644 metricbeat/module/jolokia/_meta/env create mode 100644 metricbeat/module/jolokia/_meta/fields.yml create mode 100644 metricbeat/module/jolokia/doc.go create mode 100644 metricbeat/module/jolokia/jmx/_meta/data.json create mode 100644 metricbeat/module/jolokia/jmx/_meta/docs.asciidoc create mode 100644 metricbeat/module/jolokia/jmx/_meta/fields.yml create mode 100644 metricbeat/module/jolokia/jmx/_meta/test/config.yml create mode 100644 metricbeat/module/jolokia/jmx/_meta/test/jolokia_response.json create mode 100644 metricbeat/module/jolokia/jmx/config.go create mode 100644 metricbeat/module/jolokia/jmx/data.go create mode 100644 metricbeat/module/jolokia/jmx/data_test.go create mode 100644 metricbeat/module/jolokia/jmx/jmx.go create mode 100644 metricbeat/module/jolokia/jmx/jmx_integration_test.go create mode 100644 metricbeat/tests/system/test_jolokia.py diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 9fda09da6578..a50c3b2a341b 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -170,6 +170,7 @@ https://github.com/elastic/beats/compare/v5.1.2...v5.2.0[View commits] - Experimental Prometheus module. {pull}3202[3202] - Add system socket module that reports all TCP sockets. {pull}3246[3246] - Kafka consumer groups metricset. {pull}3240[3240] +- Add jolokia module with dynamic jmx metricset. {pull}3570[3570] *Winlogbeat* diff --git a/libbeat/scripts/generate_index_pattern.py b/libbeat/scripts/generate_index_pattern.py index 94c40a673ed7..2e748f35cb1e 100644 --- a/libbeat/scripts/generate_index_pattern.py +++ b/libbeat/scripts/generate_index_pattern.py @@ -19,6 +19,10 @@ def fields_to_json(section, path, output): + # Need in case there are no fields + if section["fields"] is None: + section["fields"] = {} + for field in section["fields"]: if path == "": newpath = field["name"] diff --git a/libbeat/scripts/generate_template.py b/libbeat/scripts/generate_template.py index 8abf5d5190c4..a27543a327a4 100644 --- a/libbeat/scripts/generate_template.py +++ b/libbeat/scripts/generate_template.py @@ -135,6 +135,10 @@ def dedot(group): fields = [] dedotted = {} + # Need in case there are no fields + if group["fields"] is None: + group["fields"] = {} + for field in group["fields"]: if "." in field["name"]: # dedot diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index e1eb9f24ab64..d3aba834c749 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -390,6 +390,10 @@ def load_fields(self, fields_doc="../../_meta/fields.generated.yml"): def extract_fields(doc_list, name): fields = [] dictfields = [] + + if doc_list is None: + return fields, dictfields + for field in doc_list: # Chain together names diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml index ff41f65678e3..626682417f54 100644 --- a/metricbeat/docker-compose.yml +++ b/metricbeat/docker-compose.yml @@ -18,6 +18,7 @@ services: - ceph - couchbase - haproxy + - jolokia - kafka - mongodb - mysql @@ -33,6 +34,7 @@ services: - ${PWD}/module/ceph/_meta/env - ${PWD}/module/couchbase/_meta/env - ${PWD}/module/haproxy/_meta/env + - ${PWD}/module/jolokia/_meta/env - ${PWD}/module/kafka/_meta/env - ${PWD}/module/mongodb/_meta/env - ${PWD}/module/mysql/_meta/env @@ -56,6 +58,9 @@ services: haproxy: build: ${PWD}/module/haproxy/_meta + jolokia: + build: ${PWD}/module/jolokia/_meta + kafka: build: ${PWD}/module/kafka/_meta diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index b228d9cb821f..f74dc2588511 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -20,6 +20,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -2804,6 +2805,20 @@ type: integer The average queue time in ms over the last 1024 requests. +[[exported-fields-jolokia]] +== Jolokia Fields + +[]beta +Jolokia Module + + + +[float] +== jolokia Fields + +jolokia contains metrics exposed via jolokia agent + + [[exported-fields-kafka]] == kafka Fields diff --git a/metricbeat/docs/modules/jolokia.asciidoc b/metricbeat/docs/modules/jolokia.asciidoc new file mode 100644 index 000000000000..ea006397f8cd --- /dev/null +++ b/metricbeat/docs/modules/jolokia.asciidoc @@ -0,0 +1,43 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-module-jolokia]] +== Jolokia Module + +beta[] + +This is the Jolokia Module. + + + +[float] +=== Example Configuration + +The Jolokia module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +#- module: jolokia +# metricsets: ["jmx"] +# enabled: true +# period: 10s +# hosts: ["localhost"] +# namespace: "metrics" +# path: "/jolokia/?ignoreErrors=true&canonicalNaming=false" +# jmx.mapping: +# jmx.application: +# jmx.instance: +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +include::jolokia/jmx.asciidoc[] + diff --git a/metricbeat/docs/modules/jolokia/jmx.asciidoc b/metricbeat/docs/modules/jolokia/jmx.asciidoc new file mode 100644 index 000000000000..99290cdf069c --- /dev/null +++ b/metricbeat/docs/modules/jolokia/jmx.asciidoc @@ -0,0 +1,19 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-jolokia-jmx]] +include::../../../module/jolokia/jmx/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/jolokia/jmx/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index ceecfd51fc55..f56c3d6c1ee4 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -7,6 +7,7 @@ This file is generated! See scripts/docs_collector.py * <> * <> * <> + * <> * <> * <> * <> @@ -26,6 +27,7 @@ include::modules/ceph.asciidoc[] include::modules/couchbase.asciidoc[] include::modules/docker.asciidoc[] include::modules/haproxy.asciidoc[] +include::modules/jolokia.asciidoc[] include::modules/kafka.asciidoc[] include::modules/mongodb.asciidoc[] include::modules/mysql.asciidoc[] diff --git a/metricbeat/helper/http.go b/metricbeat/helper/http.go index f3eb49d5d372..d103cad5d798 100644 --- a/metricbeat/helper/http.go +++ b/metricbeat/helper/http.go @@ -5,6 +5,7 @@ import ( "bytes" "encoding/json" "fmt" + "io" "io/ioutil" "net/http" @@ -15,6 +16,8 @@ type HTTP struct { base mb.BaseMetricSet client *http.Client // HTTP client that is reused across requests. headers map[string]string + method string + body []byte } // NewHTTP creates new http helper @@ -23,6 +26,8 @@ func NewHTTP(base mb.BaseMetricSet) *HTTP { base: base, client: &http.Client{Timeout: base.Module().Config().Timeout}, headers: map[string]string{}, + method: "GET", + body: nil, } } @@ -30,7 +35,14 @@ func NewHTTP(base mb.BaseMetricSet) *HTTP { // It's important that resp.Body has to be closed if this method is used. Before using this method // check if one of the other Fetch* methods could be used as they ensure that the Body is properly closed. func (h *HTTP) FetchResponse() (*http.Response, error) { - req, err := http.NewRequest("GET", h.base.HostData().SanitizedURI, nil) + + // Create a fresh reader every time + var reader io.Reader + if h.body != nil { + reader = bytes.NewReader(h.body) + } + + req, err := http.NewRequest(h.method, h.base.HostData().SanitizedURI, reader) if h.base.HostData().User != "" || h.base.HostData().Password != "" { req.SetBasicAuth(h.base.HostData().User, h.base.HostData().Password) } @@ -51,6 +63,14 @@ func (h *HTTP) SetHeader(key, value string) { h.headers[key] = value } +func (h *HTTP) SetMethod(method string) { + h.method = method +} + +func (h *HTTP) SetBody(body []byte) { + h.body = body +} + // FetchContent makes an HTTP request to the configured url and returns the body content. func (h *HTTP) FetchContent() ([]byte, error) { resp, err := h.FetchResponse() diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go index 68f6d060d4b1..b1deebf725b4 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list.go @@ -31,6 +31,8 @@ import ( _ "github.com/elastic/beats/metricbeat/module/haproxy" _ "github.com/elastic/beats/metricbeat/module/haproxy/info" _ "github.com/elastic/beats/metricbeat/module/haproxy/stat" + _ "github.com/elastic/beats/metricbeat/module/jolokia" + _ "github.com/elastic/beats/metricbeat/module/jolokia/jmx" _ "github.com/elastic/beats/metricbeat/module/kafka" _ "github.com/elastic/beats/metricbeat/module/kafka/consumergroup" _ "github.com/elastic/beats/metricbeat/module/kafka/partition" diff --git a/metricbeat/metricbeat.full.yml b/metricbeat/metricbeat.full.yml index 04649c091d69..7f539dbd9c50 100644 --- a/metricbeat/metricbeat.full.yml +++ b/metricbeat/metricbeat.full.yml @@ -128,6 +128,18 @@ metricbeat.modules: #period: 10s #hosts: ["tcp://127.0.0.1:14567"] +#------------------------------- Jolokia Module ------------------------------ +#- module: jolokia +# metricsets: ["jmx"] +# enabled: true +# period: 10s +# hosts: ["localhost"] +# namespace: "metrics" +# path: "/jolokia/?ignoreErrors=true&canonicalNaming=false" +# jmx.mapping: +# jmx.application: +# jmx.instance: + #-------------------------------- kafka Module ------------------------------- #- module: kafka #metricsets: ["partition"] diff --git a/metricbeat/module/jolokia/_meta/Dockerfile b/metricbeat/module/jolokia/_meta/Dockerfile new file mode 100644 index 000000000000..fdd0e30b8063 --- /dev/null +++ b/metricbeat/module/jolokia/_meta/Dockerfile @@ -0,0 +1,10 @@ +# Tomcat is started to fetch Jolokia metrics from it +FROM jolokia/java-jolokia:7 +ENV TOMCAT_VERSION 7.0.55 +ENV TC apache-tomcat-${TOMCAT_VERSION} + +EXPOSE 8778 +RUN wget http://archive.apache.org/dist/tomcat/tomcat-7/v${TOMCAT_VERSION}/bin/${TC}.tar.gz +RUN tar xzf ${TC}.tar.gz -C /opt + +CMD env CATALINA_OPTS=$(jolokia_opts) /opt/${TC}/bin/catalina.sh run diff --git a/metricbeat/module/jolokia/_meta/config.yml b/metricbeat/module/jolokia/_meta/config.yml new file mode 100644 index 000000000000..8658d1f282b7 --- /dev/null +++ b/metricbeat/module/jolokia/_meta/config.yml @@ -0,0 +1,10 @@ +#- module: jolokia +# metricsets: ["jmx"] +# enabled: true +# period: 10s +# hosts: ["localhost"] +# namespace: "metrics" +# path: "/jolokia/?ignoreErrors=true&canonicalNaming=false" +# jmx.mapping: +# jmx.application: +# jmx.instance: diff --git a/metricbeat/module/jolokia/_meta/docs.asciidoc b/metricbeat/module/jolokia/_meta/docs.asciidoc new file mode 100644 index 000000000000..af59ed28f1cf --- /dev/null +++ b/metricbeat/module/jolokia/_meta/docs.asciidoc @@ -0,0 +1,6 @@ +== Jolokia Module + +beta[] + +This is the Jolokia Module. + diff --git a/metricbeat/module/jolokia/_meta/env b/metricbeat/module/jolokia/_meta/env new file mode 100644 index 000000000000..9c0340b6f3c5 --- /dev/null +++ b/metricbeat/module/jolokia/_meta/env @@ -0,0 +1,2 @@ +JOLOKIA_HOST=jolokia +JOLOKIA_PORT=8778 diff --git a/metricbeat/module/jolokia/_meta/fields.yml b/metricbeat/module/jolokia/_meta/fields.yml new file mode 100644 index 000000000000..dde458c49807 --- /dev/null +++ b/metricbeat/module/jolokia/_meta/fields.yml @@ -0,0 +1,13 @@ +- key: jolokia + title: "Jolokia" + description: > + []beta + + Jolokia Module + short_config: false + fields: + - name: jolokia + type: group + description: > + jolokia contains metrics exposed via jolokia agent + fields: diff --git a/metricbeat/module/jolokia/doc.go b/metricbeat/module/jolokia/doc.go new file mode 100644 index 000000000000..149effe70b0d --- /dev/null +++ b/metricbeat/module/jolokia/doc.go @@ -0,0 +1,4 @@ +/* +Package jolokia is a Metricbeat module that contains MetricSets. +*/ +package jolokia diff --git a/metricbeat/module/jolokia/jmx/_meta/data.json b/metricbeat/module/jolokia/jmx/_meta/data.json new file mode 100644 index 000000000000..2405a1c4da87 --- /dev/null +++ b/metricbeat/module/jolokia/jmx/_meta/data.json @@ -0,0 +1,34 @@ +{ + "@timestamp": "2016-05-23T08:05:34.853Z", + "beat": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "jolokia": { + "testnamespace": { + "memory": { + "heap_usage": { + "committed": 1.09051904e+08, + "init": 3.2753408e+07, + "max": 6.20756992e+08, + "used": 5.8796168e+07 + }, + "non_heap_usage": { + "committed": 3.244032e+07, + "init": 2.4576e+07, + "max": 2.24395264e+08, + "used": 1.7975176e+07 + } + }, + "uptime": 6.1802139e+07 + } + }, + "metricset": { + "host": "127.0.0.1:8778", + "module": "jolokia", + "name": "jmx", + "namespace": "testnamespace", + "rtt": 115 + }, + "type": "metricsets" +} \ No newline at end of file diff --git a/metricbeat/module/jolokia/jmx/_meta/docs.asciidoc b/metricbeat/module/jolokia/jmx/_meta/docs.asciidoc new file mode 100644 index 000000000000..901bb62a5c08 --- /dev/null +++ b/metricbeat/module/jolokia/jmx/_meta/docs.asciidoc @@ -0,0 +1,49 @@ +=== jolokia jmx MetricSet + +This is the jmx metricset of the module jolokia. + +[float] +=== Features and configuration +Tested with Jolokia 1.3.4. + +Metrics to be collected from each Jolokia instance are defined in the mapping section with an MBean ObjectName and +an array of Attributes to be requested with Elastic field names under which the return values should be saved. + +For example: to get the "Uptime" attribute from the "java.lang:type=Runtime" MBean and map it to something like +"uptime" (actually "jolokia.jmx.uptime", the prexif is added by beats framework) you have to configure following +mapping: + +``` +- module: jolokia + metricsets: ["jmx"] + hosts: ["localhost:8778"] + namespace: "metrics" + jmx.mappings: + - mbean: 'java.lang:type=Runtime' + attributes: + - attr: Uptime + field: uptime +``` + +In case the underlying attribute is an object (e.g. see HeapMemoryUsage attribute in java.lang:type=Memory) it`s +structure will be published to Elastic "as is". + +It is possible to configure nested metric aliases by using dots in the mapping name (e.g. gc.cms_collection_time). For examples please refer to the +/jolokia/jmx/test/config.yml. + +All metrics from a single mapping will be POSTed to the defined host/port and sent to Elastic as a single event. +To make it possible to differentiate between metrics from multiple similar applications running on the same host, +please configure multiple modules. + +It is required to set a namespace in the general module config section. + +[float] +=== Limitations +No authentication against Jolokia is supported yet. No wildcards in Jolokia requests supported yet. +All Jolokia requests have canonicalNaming set to false (details see here: https://jolokia.org/reference/html/protocol.html). + + +[float] +=== Exposed fields, Dashboards, Indexes, etc. +Since this is a very general module that can be tailored for any application that exposes it's metrics over Jolokia, it +comes with no exposed fields description, dashboards or index patterns. diff --git a/metricbeat/module/jolokia/jmx/_meta/fields.yml b/metricbeat/module/jolokia/jmx/_meta/fields.yml new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/metricbeat/module/jolokia/jmx/_meta/test/config.yml b/metricbeat/module/jolokia/jmx/_meta/test/config.yml new file mode 100644 index 000000000000..97ccd8f683ec --- /dev/null +++ b/metricbeat/module/jolokia/jmx/_meta/test/config.yml @@ -0,0 +1,68 @@ +###################### Metricbeat Configuration Example ####################### + +#========================== Modules configuration ============================ +metricbeat.modules: + +#------------------------------ jolokia Module ----------------------------- +- module: jolokia + metricsets: ["jmx"] + enabled: true + period: 10s + namespace: "jolokia_metrics" + hosts: ["localhost:4008"] + jmx.mappings: + - mbean: 'java.lang:type=Runtime' + attributes: + - attr: Uptime + field: uptime + - mbean: 'java.lang:type=GarbageCollector,name=ConcurrentMarkSweep' + attributes: + - attr: CollectionTime + field: gc.cms_collection_time + - attr: CollectionCount + field: gc.cms_collection_count + - mbean: 'java.lang:type=Memory' + attributes: + - attr: HeapMemoryUsage + field: memory.heap_usage + - attr: NonHeapMemoryUsage + field: memory.non_heap_usage + +- module: jolokia + metricsets: ["jmx"] + enabled: true + period: 10s + namespace: "jolokia_metrics" + hosts: ["localhost:4002"] + jmx.mappings: + - mbean: 'org.apache.cassandra.metrics:type=ClientRequest,scope=Read,name=Latency' + attributes: + - attr: OneMinuteRate + field: client_request.read_latency_one_min_rate + - attr: Count + field: client_request.read_latency + - mbean: 'org.apache.cassandra.metrics:type=ClientRequest,scope=Write,name=Latency' + attributes: + - attr: OneMinuteRate + field: client_request.write_latency_one_min_rate + - attr: Count + field: client_request.write_latency + - mbean: 'org.apache.cassandra.metrics:type=Compaction,name=CompletedTasks' + attributes: + - attr: Value + field: compaction.completed_tasks + - mbean: 'org.apache.cassandra.metrics:type=Compaction,name=PendingTasks' + attributes: + - attr: Value + field: compaction.pending_tasks +#================================ Outputs ===================================== + +#-------------------------- Elasticsearch output ------------------------------ +output.elasticsearch: + # Array of hosts to connect to. + hosts: ["localhost:9200"] + + # Optional protocol and basic auth credentials. + #protocol: "https" + #username: "elastic" + #password: "changeme" diff --git a/metricbeat/module/jolokia/jmx/_meta/test/jolokia_response.json b/metricbeat/module/jolokia/jmx/_meta/test/jolokia_response.json new file mode 100644 index 000000000000..effa8f9fc510 --- /dev/null +++ b/metricbeat/module/jolokia/jmx/_meta/test/jolokia_response.json @@ -0,0 +1,56 @@ +[ + { + "request": { + "mbean": "java.lang:type=Runtime", + "attribute": "Uptime", + "type": "read" + }, + "value": { + "Uptime": 47283 + }, + "timestamp": 1472298687, + "status": 200 + }, + { + "request": { + "mbean": "java.lang:type=GarbageCollector,name=ConcurrentMarkSweep", + "attribute": [ + "CollectionTime", + "CollectionCount" + ], + "type": "read" + }, + "value": { + "CollectionTime": 53, + "CollectionCount": 1 + }, + "timestamp": 1472298687, + "status": 200 + }, + { + "request": { + "mbean": "java.lang:type=Memory", + "attribute": [ + "HeapMemoryUsage", + "NonHeapMemoryUsage" + ], + "type": "read" + }, + "value": { + "HeapMemoryUsage": { + "init": 1073741824, + "committed": 1037959168, + "max": 1037959168, + "used": 227420472 + }, + "NonHeapMemoryUsage": { + "init": 2555904, + "committed": 53477376, + "max": -1, + "used": 50519768 + } + }, + "timestamp": 1472298687, + "status": 200 + } +] diff --git a/metricbeat/module/jolokia/jmx/config.go b/metricbeat/module/jolokia/jmx/config.go new file mode 100644 index 000000000000..26a263e6e97f --- /dev/null +++ b/metricbeat/module/jolokia/jmx/config.go @@ -0,0 +1,61 @@ +package jmx + +import "encoding/json" + +type JMXMapping struct { + MBean string + Attributes []Attribute +} + +type Attribute struct { + Attr string + Field string +} + +// RequestBlock is used to build the request blocks of the following format: +// +// [ +// { +// "type":"read", +// "mbean":"java.lang:type=Runtime", +// "attribute":[ +// "Uptime" +// ] +// }, +// { +// "type":"read", +// "mbean":"java.lang:type=GarbageCollector,name=ConcurrentMarkSweep", +// "attribute":[ +// "CollectionTime", +// "CollectionCount" +// ] +// } +// ] +type RequestBlock struct { + Type string `json:"type"` + MBean string `json:"mbean"` + Attribute []string `json:"attribute"` +} + +func buildRequestBodyAndMapping(mappings []JMXMapping) ([]byte, map[string]string, error) { + + responseMapping := map[string]string{} + blocks := []RequestBlock{} + + for _, mapping := range mappings { + + rb := RequestBlock{ + Type: "read", + MBean: mapping.MBean, + } + + for _, attribute := range mapping.Attributes { + rb.Attribute = append(rb.Attribute, attribute.Attr) + responseMapping[mapping.MBean+"_"+attribute.Attr] = attribute.Field + } + blocks = append(blocks, rb) + } + + content, err := json.Marshal(blocks) + return content, responseMapping, err +} diff --git a/metricbeat/module/jolokia/jmx/data.go b/metricbeat/module/jolokia/jmx/data.go new file mode 100644 index 000000000000..37ca5b5d8797 --- /dev/null +++ b/metricbeat/module/jolokia/jmx/data.go @@ -0,0 +1,87 @@ +package jmx + +import ( + "encoding/json" + "fmt" + + "github.com/elastic/beats/libbeat/common" + "github.com/joeshaw/multierror" +) + +type Entry struct { + Request struct { + Mbean string `json:"mbean"` + } + Value map[string]interface{} +} + +// Map responseBody to common.MapStr +// +// A response has the following structure +// [ +// { +// "request": { +// "mbean": "java.lang:type=Memory", +// "attribute": [ +// "HeapMemoryUsage", +// "NonHeapMemoryUsage" +// ], +// "type": "read" +// }, +// "value": { +// "HeapMemoryUsage": { +// "init": 1073741824, +// "committed": 1037959168, +// "max": 1037959168, +// "used": 227420472 +// }, +// "NonHeapMemoryUsage": { +// "init": 2555904, +// "committed": 53477376, +// "max": -1, +// "used": 50519768 +// } +// }, +// "timestamp": 1472298687, +// "status": 200 +// } +// ] +func eventMapping(content []byte, mapping map[string]string) (common.MapStr, error) { + + var entries []Entry + err := json.Unmarshal(content, &entries) + if err != nil { + return nil, fmt.Errorf("Cannot unmarshal json response: %s", err) + } + + event := common.MapStr{} + var errs multierror.Errors + + for _, v := range entries { + for attribute, value := range v.Value { + // Extend existing event + err := parseResponseEntry(v.Request.Mbean, attribute, value, event, mapping) + if err != nil { + errs = append(errs, err) + } + } + } + + return event, errs.Err() + +} + +func parseResponseEntry(mbeanName string, attributeName string, attibuteValue interface{}, + event common.MapStr, mapping map[string]string) error { + + //create metric name by merging mbean and attribute fields + var metricName = mbeanName + "_" + attributeName + + key, exists := mapping[metricName] + if !exists { + return fmt.Errorf("No key found for metric: '%s', skipping...", metricName) + } + + _, err := event.Put(key, attibuteValue) + return err +} diff --git a/metricbeat/module/jolokia/jmx/data_test.go b/metricbeat/module/jolokia/jmx/data_test.go new file mode 100644 index 000000000000..73f00660b195 --- /dev/null +++ b/metricbeat/module/jolokia/jmx/data_test.go @@ -0,0 +1,44 @@ +package jmx + +import ( + "io/ioutil" + "path/filepath" + "testing" + + "github.com/elastic/beats/libbeat/common" + "github.com/stretchr/testify/assert" +) + +func TestEventMapper(t *testing.T) { + absPath, err := filepath.Abs("./_meta/test") + + assert.NotNil(t, absPath) + assert.Nil(t, err) + + jolokiaResponse, err := ioutil.ReadFile(absPath + "/jolokia_response.json") + + assert.Nil(t, err) + + var mapping = map[string]string{ + "java.lang:type=Runtime_Uptime": "uptime", + "java.lang:type=GarbageCollector,name=ConcurrentMarkSweep_CollectionTime": "gc.cms_collection_time", + "java.lang:type=GarbageCollector,name=ConcurrentMarkSweep_CollectionCount": "gc.cms_collection_count", + "java.lang:type=Memory_HeapMemoryUsage": "memory.heap_usage", + "java.lang:type=Memory_NonHeapMemoryUsage": "memory.non_heap_usage", + } + + event, err := eventMapping(jolokiaResponse, mapping) + + assert.Nil(t, err) + assert.EqualValues(t, 47283, event["uptime"]) + assert.EqualValues(t, 53, event["gc"].(common.MapStr)["cms_collection_time"]) + assert.EqualValues(t, 1, event["gc"].(common.MapStr)["cms_collection_count"]) + assert.EqualValues(t, 1073741824, event["memory"].(common.MapStr)["heap_usage"].(map[string]interface{})["init"]) + assert.EqualValues(t, 1037959168, event["memory"].(common.MapStr)["heap_usage"].(map[string]interface{})["committed"]) + assert.EqualValues(t, 1037959168, event["memory"].(common.MapStr)["heap_usage"].(map[string]interface{})["max"]) + assert.EqualValues(t, 227420472, event["memory"].(common.MapStr)["heap_usage"].(map[string]interface{})["used"]) + assert.EqualValues(t, 2555904, event["memory"].(common.MapStr)["non_heap_usage"].(map[string]interface{})["init"]) + assert.EqualValues(t, 53477376, event["memory"].(common.MapStr)["non_heap_usage"].(map[string]interface{})["committed"]) + assert.EqualValues(t, -1, event["memory"].(common.MapStr)["non_heap_usage"].(map[string]interface{})["max"]) + assert.EqualValues(t, 50519768, event["memory"].(common.MapStr)["non_heap_usage"].(map[string]interface{})["used"]) +} diff --git a/metricbeat/module/jolokia/jmx/jmx.go b/metricbeat/module/jolokia/jmx/jmx.go new file mode 100644 index 000000000000..b2b1de38bad7 --- /dev/null +++ b/metricbeat/module/jolokia/jmx/jmx.go @@ -0,0 +1,96 @@ +package jmx + +import ( + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/metricbeat/helper" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/mb/parse" +) + +var ( + debugf = logp.MakeDebug("jolokia-jmx") +) + +// init registers the MetricSet with the central registry. +func init() { + if err := mb.Registry.AddMetricSet("jolokia", "jmx", New, hostParser); err != nil { + panic(err) + } +} + +const ( + // defaultScheme is the default scheme to use when it is not specified in + // the host config. + defaultScheme = "http" + + // defaultPath is the default path to the ngx_http_stub_status_module endpoint on Nginx. + defaultPath = "/jolokia/?ignoreErrors=true&canonicalNaming=false" +) + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + PathConfigKey: "path", + DefaultPath: defaultPath, + }.Build() +) + +// MetricSet type defines all fields of the MetricSet +type MetricSet struct { + mb.BaseMetricSet + mapping map[string]string + namespace string + http *helper.HTTP +} + +// New create a new instance of the MetricSet +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + logp.Warn("BETA: The jolokia jmx metricset is beta") + + // Additional configuration options + config := struct { + Namespace string `config:"namespace" validate:"required"` + Mappings []JMXMapping `config:"jmx.mappings" validate:"required"` + }{} + + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + + body, mapping, err := buildRequestBodyAndMapping(config.Mappings) + if err != nil { + return nil, err + } + + http := helper.NewHTTP(base) + http.SetMethod("POST") + http.SetBody(body) + + return &MetricSet{ + BaseMetricSet: base, + mapping: mapping, + namespace: config.Namespace, + http: http, + }, nil + +} + +// Fetch methods implements the data gathering and data conversion to the right format +func (m *MetricSet) Fetch() (common.MapStr, error) { + + body, err := m.http.FetchContent() + if err != nil { + return nil, err + } + + event, err := eventMapping(body, m.mapping) + if err != nil { + return nil, err + } + + // Set dynamic namespace + event["_namespace"] = m.namespace + + return event, nil +} diff --git a/metricbeat/module/jolokia/jmx/jmx_integration_test.go b/metricbeat/module/jolokia/jmx/jmx_integration_test.go new file mode 100644 index 000000000000..f3df73272737 --- /dev/null +++ b/metricbeat/module/jolokia/jmx/jmx_integration_test.go @@ -0,0 +1,93 @@ +// +build integration + +package jmx + +import ( + "os" + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/stretchr/testify/assert" +) + +func TestFetch(t *testing.T) { + f := mbtest.NewEventFetcher(t, getConfig()) + event, err := f.Fetch() + if !assert.NoError(t, err) { + t.FailNow() + } + + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event) +} + +func TestData(t *testing.T) { + f := mbtest.NewEventFetcher(t, getConfig()) + err := mbtest.WriteEvent(f, t) + if err != nil { + t.Fatal("write", err) + } +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "jolokia", + "metricsets": []string{"jmx"}, + "hosts": []string{getEnvHost() + ":" + getEnvPort()}, + "namespace": "testnamespace", + "jmx.mappings": []map[string]interface{}{ + { + "mbean": "java.lang:type=Runtime", + "attributes": []map[string]string{ + { + "attr": "Uptime", + "field": "uptime", + }, + }, + }, + { + "mbean": "java.lang:type=GarbageCollector,name=ConcurrentMarkSweep", + "attributes": []map[string]string{ + { + "attr": "CollectionTime", + "field": "gc.cms_collection_time", + }, + { + "attr": "CollectionCount", + "field": "gc.cms_collection_count", + }, + }, + }, + { + "mbean": "java.lang:type=Memory", + "attributes": []map[string]string{ + { + "attr": "HeapMemoryUsage", + "field": "memory.heap_usage", + }, + { + "attr": "NonHeapMemoryUsage", + "field": "memory.non_heap_usage", + }, + }, + }, + }, + } +} + +func getEnvHost() string { + host := os.Getenv("JOLOKIA_HOST") + + if len(host) == 0 { + host = "127.0.0.1" + } + return host +} + +func getEnvPort() string { + port := os.Getenv("JOLOKIA_PORT") + + if len(port) == 0 { + port = "8778" + } + return port +} diff --git a/metricbeat/tests/system/config/metricbeat.yml.j2 b/metricbeat/tests/system/config/metricbeat.yml.j2 index da5a6b156098..4a0fab297e26 100644 --- a/metricbeat/tests/system/config/metricbeat.yml.j2 +++ b/metricbeat/tests/system/config/metricbeat.yml.j2 @@ -41,6 +41,10 @@ metricbeat.modules: timeout: {{ m.timeout }} {% endif -%} + {% if m.namespace -%} + namespace: {{ m.namespace }} + {% endif -%} + {% if m.processes -%} processes: {{ m.processes }} {% endif -%} @@ -78,6 +82,9 @@ metricbeat.modules: {{ k }}: {{ v }} {% endfor %} {% endif -%} + {% if m.additional_content -%} + {{ m.additional_content }} + {% endif -%} {%- endfor %} {% if reload -%} diff --git a/metricbeat/tests/system/test_jolokia.py b/metricbeat/tests/system/test_jolokia.py new file mode 100644 index 000000000000..6c37b46f25ab --- /dev/null +++ b/metricbeat/tests/system/test_jolokia.py @@ -0,0 +1,44 @@ +import os +import metricbeat +import unittest +from nose.plugins.attrib import attr + + +class Test(metricbeat.BaseTest): + + @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") + def test_jmx(self): + """ + jolokia jmx metricset test + """ + + additional_content = """ + jmx.mappings: + - mbean: 'java.lang:type=Runtime' + attributes: + - attr: Uptime + field: uptime +""" + + self.render_config_template(modules=[{ + "name": "jolokia", + "metricsets": ["jmx"], + "hosts": self.get_hosts(), + "period": "1s", + "namespace": "test", + "additional_content": additional_content, + }]) + proc = self.start_beat() + self.wait_until(lambda: self.output_lines() > 0, max_timeout=20) + proc.check_kill_and_wait() + + output = self.read_output_json() + self.assertTrue(len(output) >= 1) + evt = output[0] + print(evt) + + assert evt["jolokia"]["test"]["uptime"] > 0 + + def get_hosts(self): + return [os.getenv('JOLOKIA_HOST', 'localhost') + ':' + + os.getenv('JOLOKIA_PORT', '8778')]