diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 4f755970603..c1d3ad2d3aa 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -143,6 +143,7 @@ https://github.com/elastic/beats/compare/v5.1.2...v5.2.0[View commits] - Experimental Prometheus module. {pull}3202[3202] - Add system socket module that reports all TCP sockets. {pull}3246[3246] - Kafka consumer groups metricset. {pull}3240[3240] +- Add jolokia module with dynamic jmx metricset. {pull}3570[3570] *Winlogbeat* diff --git a/libbeat/scripts/generate_index_pattern.py b/libbeat/scripts/generate_index_pattern.py index 6efa6d78ab5..74d96ce2ba5 100644 --- a/libbeat/scripts/generate_index_pattern.py +++ b/libbeat/scripts/generate_index_pattern.py @@ -18,6 +18,10 @@ def fields_to_json(section, path, output): + # Need in case there are no fields + if section["fields"] is None: + section["fields"] = {} + for field in section["fields"]: if path == "": newpath = field["name"] diff --git a/libbeat/scripts/generate_template.py b/libbeat/scripts/generate_template.py index 2b6fb53eb5f..565669676b2 100644 --- a/libbeat/scripts/generate_template.py +++ b/libbeat/scripts/generate_template.py @@ -128,6 +128,10 @@ def dedot(group): fields = [] dedotted = {} + # Need in case there are no fields + if group["fields"] is None: + group["fields"] = {} + for field in group["fields"]: if "." in field["name"]: # dedot diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index 6e3379a6e8f..f77fce5c39c 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -387,6 +387,10 @@ def load_fields(self, fields_doc="../../_meta/fields.generated.yml"): def extract_fields(doc_list, name): fields = [] dictfields = [] + + if doc_list is None: + return fields, dictfields + for field in doc_list: # Chain together names diff --git a/metricbeat/_meta/beat.full.yml b/metricbeat/_meta/beat.full.yml index 294e79b7751..b9a2f285eca 100644 --- a/metricbeat/_meta/beat.full.yml +++ b/metricbeat/_meta/beat.full.yml @@ -102,6 +102,18 @@ metricbeat.modules: #period: 10s #hosts: ["tcp://127.0.0.1:14567"] +#------------------------------- Jolokia Module ------------------------------ +#- module: jolokia +# metricsets: ["jmx"] +# enabled: true +# period: 10s +# hosts: ["localhost"] +# namespace: "metrics" +# path: "/jolokia/?ignoreErrors=true&canonicalNaming=false" +# jmx.mapping: +# jmx.application: +# jmx.instance: + #-------------------------------- kafka Module ------------------------------- #- module: kafka #metricsets: ["partition"] diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml index 3034072513c..9615e42d908 100644 --- a/metricbeat/docker-compose.yml +++ b/metricbeat/docker-compose.yml @@ -7,6 +7,7 @@ services: - couchbase - mongodb - haproxy + - jolokia - kafka - mysql - nginx @@ -52,6 +53,19 @@ services: # Overloading kibana with a simple image as it is not needed here kibana: image: alpine:latest + env_file: + - ${PWD}/module/apache/_meta/env + - ${PWD}/module/couchbase/_meta/env + - ${PWD}/module/haproxy/_meta/env + - ${PWD}/module/jolokia/_meta/env + - ${PWD}/module/kafka/_meta/env + - ${PWD}/module/mongodb/_meta/env + - ${PWD}/module/mysql/_meta/env + - ${PWD}/module/nginx/_meta/env + - ${PWD}/module/postgresql/_meta/env + - ${PWD}/module/prometheus/_meta/env + - ${PWD}/module/redis/_meta/env + - ${PWD}/module/zookeeper/_meta/env # Modules apache: @@ -60,6 +74,12 @@ services: couchbase: build: ${PWD}/module/couchbase/_meta + haproxy: + build: ${PWD}/module/haproxy/_meta + + jolokia: + build: ${PWD}/module/jolokia/_meta + kafka: image: spotify/kafka expose: diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 0466dfedeb9..43b87cd85ba 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -19,6 +19,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -2387,6 +2388,20 @@ type: integer The average queue time in ms over the last 1024 requests. +[[exported-fields-jolokia]] +== Jolokia Fields + +[]beta +Jolokia Module + + + +[float] +== jolokia Fields + +jolokia contains metrics exposed via jolokia agent + + [[exported-fields-kafka]] == kafka Fields diff --git a/metricbeat/docs/modules/jolokia.asciidoc b/metricbeat/docs/modules/jolokia.asciidoc new file mode 100644 index 00000000000..ea006397f8c --- /dev/null +++ b/metricbeat/docs/modules/jolokia.asciidoc @@ -0,0 +1,43 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-module-jolokia]] +== Jolokia Module + +beta[] + +This is the Jolokia Module. + + + +[float] +=== Example Configuration + +The Jolokia module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +#- module: jolokia +# metricsets: ["jmx"] +# enabled: true +# period: 10s +# hosts: ["localhost"] +# namespace: "metrics" +# path: "/jolokia/?ignoreErrors=true&canonicalNaming=false" +# jmx.mapping: +# jmx.application: +# jmx.instance: +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +include::jolokia/jmx.asciidoc[] + diff --git a/metricbeat/docs/modules/jolokia/jmx.asciidoc b/metricbeat/docs/modules/jolokia/jmx.asciidoc new file mode 100644 index 00000000000..99290cdf069 --- /dev/null +++ b/metricbeat/docs/modules/jolokia/jmx.asciidoc @@ -0,0 +1,19 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-jolokia-jmx]] +include::../../../module/jolokia/jmx/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/jolokia/jmx/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 05e489a1ebe..3ca776f4108 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -6,6 +6,7 @@ This file is generated! See scripts/docs_collector.py * <> * <> * <> + * <> * <> * <> * <> @@ -23,6 +24,7 @@ include::modules/apache.asciidoc[] include::modules/couchbase.asciidoc[] include::modules/docker.asciidoc[] include::modules/haproxy.asciidoc[] +include::modules/jolokia.asciidoc[] include::modules/kafka.asciidoc[] include::modules/mongodb.asciidoc[] include::modules/mysql.asciidoc[] diff --git a/metricbeat/helper/http.go b/metricbeat/helper/http.go new file mode 100644 index 00000000000..d103cad5d79 --- /dev/null +++ b/metricbeat/helper/http.go @@ -0,0 +1,116 @@ +package helper + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + + "github.com/elastic/beats/metricbeat/mb" +) + +type HTTP struct { + base mb.BaseMetricSet + client *http.Client // HTTP client that is reused across requests. + headers map[string]string + method string + body []byte +} + +// NewHTTP creates new http helper +func NewHTTP(base mb.BaseMetricSet) *HTTP { + return &HTTP{ + base: base, + client: &http.Client{Timeout: base.Module().Config().Timeout}, + headers: map[string]string{}, + method: "GET", + body: nil, + } +} + +// FetchResponse fetches a response for the http metricset. +// It's important that resp.Body has to be closed if this method is used. Before using this method +// check if one of the other Fetch* methods could be used as they ensure that the Body is properly closed. +func (h *HTTP) FetchResponse() (*http.Response, error) { + + // Create a fresh reader every time + var reader io.Reader + if h.body != nil { + reader = bytes.NewReader(h.body) + } + + req, err := http.NewRequest(h.method, h.base.HostData().SanitizedURI, reader) + if h.base.HostData().User != "" || h.base.HostData().Password != "" { + req.SetBasicAuth(h.base.HostData().User, h.base.HostData().Password) + } + + for k, v := range h.headers { + req.Header.Set(k, v) + } + + resp, err := h.client.Do(req) + if err != nil { + return nil, fmt.Errorf("error making http request: %v", err) + } + + return resp, nil +} + +func (h *HTTP) SetHeader(key, value string) { + h.headers[key] = value +} + +func (h *HTTP) SetMethod(method string) { + h.method = method +} + +func (h *HTTP) SetBody(body []byte) { + h.body = body +} + +// FetchContent makes an HTTP request to the configured url and returns the body content. +func (h *HTTP) FetchContent() ([]byte, error) { + resp, err := h.FetchResponse() + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return nil, fmt.Errorf("HTTP error %d in %s: %s", resp.StatusCode, h.base.Name(), resp.Status) + } + + return ioutil.ReadAll(resp.Body) +} + +// FetchScanner returns a Scanner for the content. +func (h *HTTP) FetchScanner() (*bufio.Scanner, error) { + content, err := h.FetchContent() + if err != nil { + return nil, err + } + + return bufio.NewScanner(bytes.NewReader(content)), nil +} + +// FetchJSON makes an HTTP request to the configured url and returns the JSON content. +// This only works if the JSON output needed is in map[string]interface format. +func (h *HTTP) FetchJSON() (map[string]interface{}, error) { + + body, err := h.FetchContent() + if err != nil { + return nil, err + } + + var data map[string]interface{} + + err = json.Unmarshal(body, &data) + if err != nil { + return nil, err + } + + return data, nil +} diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go index 8d0782555c1..68b1db4bac8 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list.go @@ -24,6 +24,8 @@ import ( _ "github.com/elastic/beats/metricbeat/module/haproxy" _ "github.com/elastic/beats/metricbeat/module/haproxy/info" _ "github.com/elastic/beats/metricbeat/module/haproxy/stat" + _ "github.com/elastic/beats/metricbeat/module/jolokia" + _ "github.com/elastic/beats/metricbeat/module/jolokia/jmx" _ "github.com/elastic/beats/metricbeat/module/kafka" _ "github.com/elastic/beats/metricbeat/module/kafka/consumergroup" _ "github.com/elastic/beats/metricbeat/module/kafka/partition" diff --git a/metricbeat/metricbeat.full.yml b/metricbeat/metricbeat.full.yml index ef13df49bc9..63cebdf851a 100644 --- a/metricbeat/metricbeat.full.yml +++ b/metricbeat/metricbeat.full.yml @@ -102,6 +102,18 @@ metricbeat.modules: #period: 10s #hosts: ["tcp://127.0.0.1:14567"] +#------------------------------- Jolokia Module ------------------------------ +#- module: jolokia +# metricsets: ["jmx"] +# enabled: true +# period: 10s +# hosts: ["localhost"] +# namespace: "metrics" +# path: "/jolokia/?ignoreErrors=true&canonicalNaming=false" +# jmx.mapping: +# jmx.application: +# jmx.instance: + #-------------------------------- kafka Module ------------------------------- #- module: kafka #metricsets: ["partition"] diff --git a/metricbeat/module/jolokia/_meta/Dockerfile b/metricbeat/module/jolokia/_meta/Dockerfile new file mode 100644 index 00000000000..fdd0e30b806 --- /dev/null +++ b/metricbeat/module/jolokia/_meta/Dockerfile @@ -0,0 +1,10 @@ +# Tomcat is started to fetch Jolokia metrics from it +FROM jolokia/java-jolokia:7 +ENV TOMCAT_VERSION 7.0.55 +ENV TC apache-tomcat-${TOMCAT_VERSION} + +EXPOSE 8778 +RUN wget http://archive.apache.org/dist/tomcat/tomcat-7/v${TOMCAT_VERSION}/bin/${TC}.tar.gz +RUN tar xzf ${TC}.tar.gz -C /opt + +CMD env CATALINA_OPTS=$(jolokia_opts) /opt/${TC}/bin/catalina.sh run diff --git a/metricbeat/module/jolokia/_meta/config.yml b/metricbeat/module/jolokia/_meta/config.yml new file mode 100644 index 00000000000..8658d1f282b --- /dev/null +++ b/metricbeat/module/jolokia/_meta/config.yml @@ -0,0 +1,10 @@ +#- module: jolokia +# metricsets: ["jmx"] +# enabled: true +# period: 10s +# hosts: ["localhost"] +# namespace: "metrics" +# path: "/jolokia/?ignoreErrors=true&canonicalNaming=false" +# jmx.mapping: +# jmx.application: +# jmx.instance: diff --git a/metricbeat/module/jolokia/_meta/docs.asciidoc b/metricbeat/module/jolokia/_meta/docs.asciidoc new file mode 100644 index 00000000000..af59ed28f1c --- /dev/null +++ b/metricbeat/module/jolokia/_meta/docs.asciidoc @@ -0,0 +1,6 @@ +== Jolokia Module + +beta[] + +This is the Jolokia Module. + diff --git a/metricbeat/module/jolokia/_meta/env b/metricbeat/module/jolokia/_meta/env new file mode 100644 index 00000000000..9c0340b6f3c --- /dev/null +++ b/metricbeat/module/jolokia/_meta/env @@ -0,0 +1,2 @@ +JOLOKIA_HOST=jolokia +JOLOKIA_PORT=8778 diff --git a/metricbeat/module/jolokia/_meta/fields.yml b/metricbeat/module/jolokia/_meta/fields.yml new file mode 100644 index 00000000000..dde458c4980 --- /dev/null +++ b/metricbeat/module/jolokia/_meta/fields.yml @@ -0,0 +1,13 @@ +- key: jolokia + title: "Jolokia" + description: > + []beta + + Jolokia Module + short_config: false + fields: + - name: jolokia + type: group + description: > + jolokia contains metrics exposed via jolokia agent + fields: diff --git a/metricbeat/module/jolokia/doc.go b/metricbeat/module/jolokia/doc.go new file mode 100644 index 00000000000..149effe70b0 --- /dev/null +++ b/metricbeat/module/jolokia/doc.go @@ -0,0 +1,4 @@ +/* +Package jolokia is a Metricbeat module that contains MetricSets. +*/ +package jolokia diff --git a/metricbeat/module/jolokia/jmx/_meta/data.json b/metricbeat/module/jolokia/jmx/_meta/data.json new file mode 100644 index 00000000000..2405a1c4da8 --- /dev/null +++ b/metricbeat/module/jolokia/jmx/_meta/data.json @@ -0,0 +1,34 @@ +{ + "@timestamp": "2016-05-23T08:05:34.853Z", + "beat": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "jolokia": { + "testnamespace": { + "memory": { + "heap_usage": { + "committed": 1.09051904e+08, + "init": 3.2753408e+07, + "max": 6.20756992e+08, + "used": 5.8796168e+07 + }, + "non_heap_usage": { + "committed": 3.244032e+07, + "init": 2.4576e+07, + "max": 2.24395264e+08, + "used": 1.7975176e+07 + } + }, + "uptime": 6.1802139e+07 + } + }, + "metricset": { + "host": "127.0.0.1:8778", + "module": "jolokia", + "name": "jmx", + "namespace": "testnamespace", + "rtt": 115 + }, + "type": "metricsets" +} \ No newline at end of file diff --git a/metricbeat/module/jolokia/jmx/_meta/docs.asciidoc b/metricbeat/module/jolokia/jmx/_meta/docs.asciidoc new file mode 100644 index 00000000000..901bb62a5c0 --- /dev/null +++ b/metricbeat/module/jolokia/jmx/_meta/docs.asciidoc @@ -0,0 +1,49 @@ +=== jolokia jmx MetricSet + +This is the jmx metricset of the module jolokia. + +[float] +=== Features and configuration +Tested with Jolokia 1.3.4. + +Metrics to be collected from each Jolokia instance are defined in the mapping section with an MBean ObjectName and +an array of Attributes to be requested with Elastic field names under which the return values should be saved. + +For example: to get the "Uptime" attribute from the "java.lang:type=Runtime" MBean and map it to something like +"uptime" (actually "jolokia.jmx.uptime", the prexif is added by beats framework) you have to configure following +mapping: + +``` +- module: jolokia + metricsets: ["jmx"] + hosts: ["localhost:8778"] + namespace: "metrics" + jmx.mappings: + - mbean: 'java.lang:type=Runtime' + attributes: + - attr: Uptime + field: uptime +``` + +In case the underlying attribute is an object (e.g. see HeapMemoryUsage attribute in java.lang:type=Memory) it`s +structure will be published to Elastic "as is". + +It is possible to configure nested metric aliases by using dots in the mapping name (e.g. gc.cms_collection_time). For examples please refer to the +/jolokia/jmx/test/config.yml. + +All metrics from a single mapping will be POSTed to the defined host/port and sent to Elastic as a single event. +To make it possible to differentiate between metrics from multiple similar applications running on the same host, +please configure multiple modules. + +It is required to set a namespace in the general module config section. + +[float] +=== Limitations +No authentication against Jolokia is supported yet. No wildcards in Jolokia requests supported yet. +All Jolokia requests have canonicalNaming set to false (details see here: https://jolokia.org/reference/html/protocol.html). + + +[float] +=== Exposed fields, Dashboards, Indexes, etc. +Since this is a very general module that can be tailored for any application that exposes it's metrics over Jolokia, it +comes with no exposed fields description, dashboards or index patterns. diff --git a/metricbeat/module/jolokia/jmx/_meta/fields.yml b/metricbeat/module/jolokia/jmx/_meta/fields.yml new file mode 100644 index 00000000000..e69de29bb2d diff --git a/metricbeat/module/jolokia/jmx/_meta/test/config.yml b/metricbeat/module/jolokia/jmx/_meta/test/config.yml new file mode 100644 index 00000000000..97ccd8f683e --- /dev/null +++ b/metricbeat/module/jolokia/jmx/_meta/test/config.yml @@ -0,0 +1,68 @@ +###################### Metricbeat Configuration Example ####################### + +#========================== Modules configuration ============================ +metricbeat.modules: + +#------------------------------ jolokia Module ----------------------------- +- module: jolokia + metricsets: ["jmx"] + enabled: true + period: 10s + namespace: "jolokia_metrics" + hosts: ["localhost:4008"] + jmx.mappings: + - mbean: 'java.lang:type=Runtime' + attributes: + - attr: Uptime + field: uptime + - mbean: 'java.lang:type=GarbageCollector,name=ConcurrentMarkSweep' + attributes: + - attr: CollectionTime + field: gc.cms_collection_time + - attr: CollectionCount + field: gc.cms_collection_count + - mbean: 'java.lang:type=Memory' + attributes: + - attr: HeapMemoryUsage + field: memory.heap_usage + - attr: NonHeapMemoryUsage + field: memory.non_heap_usage + +- module: jolokia + metricsets: ["jmx"] + enabled: true + period: 10s + namespace: "jolokia_metrics" + hosts: ["localhost:4002"] + jmx.mappings: + - mbean: 'org.apache.cassandra.metrics:type=ClientRequest,scope=Read,name=Latency' + attributes: + - attr: OneMinuteRate + field: client_request.read_latency_one_min_rate + - attr: Count + field: client_request.read_latency + - mbean: 'org.apache.cassandra.metrics:type=ClientRequest,scope=Write,name=Latency' + attributes: + - attr: OneMinuteRate + field: client_request.write_latency_one_min_rate + - attr: Count + field: client_request.write_latency + - mbean: 'org.apache.cassandra.metrics:type=Compaction,name=CompletedTasks' + attributes: + - attr: Value + field: compaction.completed_tasks + - mbean: 'org.apache.cassandra.metrics:type=Compaction,name=PendingTasks' + attributes: + - attr: Value + field: compaction.pending_tasks +#================================ Outputs ===================================== + +#-------------------------- Elasticsearch output ------------------------------ +output.elasticsearch: + # Array of hosts to connect to. + hosts: ["localhost:9200"] + + # Optional protocol and basic auth credentials. + #protocol: "https" + #username: "elastic" + #password: "changeme" diff --git a/metricbeat/module/jolokia/jmx/_meta/test/jolokia_response.json b/metricbeat/module/jolokia/jmx/_meta/test/jolokia_response.json new file mode 100644 index 00000000000..effa8f9fc51 --- /dev/null +++ b/metricbeat/module/jolokia/jmx/_meta/test/jolokia_response.json @@ -0,0 +1,56 @@ +[ + { + "request": { + "mbean": "java.lang:type=Runtime", + "attribute": "Uptime", + "type": "read" + }, + "value": { + "Uptime": 47283 + }, + "timestamp": 1472298687, + "status": 200 + }, + { + "request": { + "mbean": "java.lang:type=GarbageCollector,name=ConcurrentMarkSweep", + "attribute": [ + "CollectionTime", + "CollectionCount" + ], + "type": "read" + }, + "value": { + "CollectionTime": 53, + "CollectionCount": 1 + }, + "timestamp": 1472298687, + "status": 200 + }, + { + "request": { + "mbean": "java.lang:type=Memory", + "attribute": [ + "HeapMemoryUsage", + "NonHeapMemoryUsage" + ], + "type": "read" + }, + "value": { + "HeapMemoryUsage": { + "init": 1073741824, + "committed": 1037959168, + "max": 1037959168, + "used": 227420472 + }, + "NonHeapMemoryUsage": { + "init": 2555904, + "committed": 53477376, + "max": -1, + "used": 50519768 + } + }, + "timestamp": 1472298687, + "status": 200 + } +] diff --git a/metricbeat/module/jolokia/jmx/config.go b/metricbeat/module/jolokia/jmx/config.go new file mode 100644 index 00000000000..26a263e6e97 --- /dev/null +++ b/metricbeat/module/jolokia/jmx/config.go @@ -0,0 +1,61 @@ +package jmx + +import "encoding/json" + +type JMXMapping struct { + MBean string + Attributes []Attribute +} + +type Attribute struct { + Attr string + Field string +} + +// RequestBlock is used to build the request blocks of the following format: +// +// [ +// { +// "type":"read", +// "mbean":"java.lang:type=Runtime", +// "attribute":[ +// "Uptime" +// ] +// }, +// { +// "type":"read", +// "mbean":"java.lang:type=GarbageCollector,name=ConcurrentMarkSweep", +// "attribute":[ +// "CollectionTime", +// "CollectionCount" +// ] +// } +// ] +type RequestBlock struct { + Type string `json:"type"` + MBean string `json:"mbean"` + Attribute []string `json:"attribute"` +} + +func buildRequestBodyAndMapping(mappings []JMXMapping) ([]byte, map[string]string, error) { + + responseMapping := map[string]string{} + blocks := []RequestBlock{} + + for _, mapping := range mappings { + + rb := RequestBlock{ + Type: "read", + MBean: mapping.MBean, + } + + for _, attribute := range mapping.Attributes { + rb.Attribute = append(rb.Attribute, attribute.Attr) + responseMapping[mapping.MBean+"_"+attribute.Attr] = attribute.Field + } + blocks = append(blocks, rb) + } + + content, err := json.Marshal(blocks) + return content, responseMapping, err +} diff --git a/metricbeat/module/jolokia/jmx/data.go b/metricbeat/module/jolokia/jmx/data.go new file mode 100644 index 00000000000..37ca5b5d879 --- /dev/null +++ b/metricbeat/module/jolokia/jmx/data.go @@ -0,0 +1,87 @@ +package jmx + +import ( + "encoding/json" + "fmt" + + "github.com/elastic/beats/libbeat/common" + "github.com/joeshaw/multierror" +) + +type Entry struct { + Request struct { + Mbean string `json:"mbean"` + } + Value map[string]interface{} +} + +// Map responseBody to common.MapStr +// +// A response has the following structure +// [ +// { +// "request": { +// "mbean": "java.lang:type=Memory", +// "attribute": [ +// "HeapMemoryUsage", +// "NonHeapMemoryUsage" +// ], +// "type": "read" +// }, +// "value": { +// "HeapMemoryUsage": { +// "init": 1073741824, +// "committed": 1037959168, +// "max": 1037959168, +// "used": 227420472 +// }, +// "NonHeapMemoryUsage": { +// "init": 2555904, +// "committed": 53477376, +// "max": -1, +// "used": 50519768 +// } +// }, +// "timestamp": 1472298687, +// "status": 200 +// } +// ] +func eventMapping(content []byte, mapping map[string]string) (common.MapStr, error) { + + var entries []Entry + err := json.Unmarshal(content, &entries) + if err != nil { + return nil, fmt.Errorf("Cannot unmarshal json response: %s", err) + } + + event := common.MapStr{} + var errs multierror.Errors + + for _, v := range entries { + for attribute, value := range v.Value { + // Extend existing event + err := parseResponseEntry(v.Request.Mbean, attribute, value, event, mapping) + if err != nil { + errs = append(errs, err) + } + } + } + + return event, errs.Err() + +} + +func parseResponseEntry(mbeanName string, attributeName string, attibuteValue interface{}, + event common.MapStr, mapping map[string]string) error { + + //create metric name by merging mbean and attribute fields + var metricName = mbeanName + "_" + attributeName + + key, exists := mapping[metricName] + if !exists { + return fmt.Errorf("No key found for metric: '%s', skipping...", metricName) + } + + _, err := event.Put(key, attibuteValue) + return err +} diff --git a/metricbeat/module/jolokia/jmx/data_test.go b/metricbeat/module/jolokia/jmx/data_test.go new file mode 100644 index 00000000000..73f00660b19 --- /dev/null +++ b/metricbeat/module/jolokia/jmx/data_test.go @@ -0,0 +1,44 @@ +package jmx + +import ( + "io/ioutil" + "path/filepath" + "testing" + + "github.com/elastic/beats/libbeat/common" + "github.com/stretchr/testify/assert" +) + +func TestEventMapper(t *testing.T) { + absPath, err := filepath.Abs("./_meta/test") + + assert.NotNil(t, absPath) + assert.Nil(t, err) + + jolokiaResponse, err := ioutil.ReadFile(absPath + "/jolokia_response.json") + + assert.Nil(t, err) + + var mapping = map[string]string{ + "java.lang:type=Runtime_Uptime": "uptime", + "java.lang:type=GarbageCollector,name=ConcurrentMarkSweep_CollectionTime": "gc.cms_collection_time", + "java.lang:type=GarbageCollector,name=ConcurrentMarkSweep_CollectionCount": "gc.cms_collection_count", + "java.lang:type=Memory_HeapMemoryUsage": "memory.heap_usage", + "java.lang:type=Memory_NonHeapMemoryUsage": "memory.non_heap_usage", + } + + event, err := eventMapping(jolokiaResponse, mapping) + + assert.Nil(t, err) + assert.EqualValues(t, 47283, event["uptime"]) + assert.EqualValues(t, 53, event["gc"].(common.MapStr)["cms_collection_time"]) + assert.EqualValues(t, 1, event["gc"].(common.MapStr)["cms_collection_count"]) + assert.EqualValues(t, 1073741824, event["memory"].(common.MapStr)["heap_usage"].(map[string]interface{})["init"]) + assert.EqualValues(t, 1037959168, event["memory"].(common.MapStr)["heap_usage"].(map[string]interface{})["committed"]) + assert.EqualValues(t, 1037959168, event["memory"].(common.MapStr)["heap_usage"].(map[string]interface{})["max"]) + assert.EqualValues(t, 227420472, event["memory"].(common.MapStr)["heap_usage"].(map[string]interface{})["used"]) + assert.EqualValues(t, 2555904, event["memory"].(common.MapStr)["non_heap_usage"].(map[string]interface{})["init"]) + assert.EqualValues(t, 53477376, event["memory"].(common.MapStr)["non_heap_usage"].(map[string]interface{})["committed"]) + assert.EqualValues(t, -1, event["memory"].(common.MapStr)["non_heap_usage"].(map[string]interface{})["max"]) + assert.EqualValues(t, 50519768, event["memory"].(common.MapStr)["non_heap_usage"].(map[string]interface{})["used"]) +} diff --git a/metricbeat/module/jolokia/jmx/jmx.go b/metricbeat/module/jolokia/jmx/jmx.go new file mode 100644 index 00000000000..b2b1de38bad --- /dev/null +++ b/metricbeat/module/jolokia/jmx/jmx.go @@ -0,0 +1,96 @@ +package jmx + +import ( + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/metricbeat/helper" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/mb/parse" +) + +var ( + debugf = logp.MakeDebug("jolokia-jmx") +) + +// init registers the MetricSet with the central registry. +func init() { + if err := mb.Registry.AddMetricSet("jolokia", "jmx", New, hostParser); err != nil { + panic(err) + } +} + +const ( + // defaultScheme is the default scheme to use when it is not specified in + // the host config. + defaultScheme = "http" + + // defaultPath is the default path to the ngx_http_stub_status_module endpoint on Nginx. + defaultPath = "/jolokia/?ignoreErrors=true&canonicalNaming=false" +) + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + PathConfigKey: "path", + DefaultPath: defaultPath, + }.Build() +) + +// MetricSet type defines all fields of the MetricSet +type MetricSet struct { + mb.BaseMetricSet + mapping map[string]string + namespace string + http *helper.HTTP +} + +// New create a new instance of the MetricSet +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + logp.Warn("BETA: The jolokia jmx metricset is beta") + + // Additional configuration options + config := struct { + Namespace string `config:"namespace" validate:"required"` + Mappings []JMXMapping `config:"jmx.mappings" validate:"required"` + }{} + + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + + body, mapping, err := buildRequestBodyAndMapping(config.Mappings) + if err != nil { + return nil, err + } + + http := helper.NewHTTP(base) + http.SetMethod("POST") + http.SetBody(body) + + return &MetricSet{ + BaseMetricSet: base, + mapping: mapping, + namespace: config.Namespace, + http: http, + }, nil + +} + +// Fetch methods implements the data gathering and data conversion to the right format +func (m *MetricSet) Fetch() (common.MapStr, error) { + + body, err := m.http.FetchContent() + if err != nil { + return nil, err + } + + event, err := eventMapping(body, m.mapping) + if err != nil { + return nil, err + } + + // Set dynamic namespace + event["_namespace"] = m.namespace + + return event, nil +} diff --git a/metricbeat/module/jolokia/jmx/jmx_integration_test.go b/metricbeat/module/jolokia/jmx/jmx_integration_test.go new file mode 100644 index 00000000000..f3df7327273 --- /dev/null +++ b/metricbeat/module/jolokia/jmx/jmx_integration_test.go @@ -0,0 +1,93 @@ +// +build integration + +package jmx + +import ( + "os" + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/stretchr/testify/assert" +) + +func TestFetch(t *testing.T) { + f := mbtest.NewEventFetcher(t, getConfig()) + event, err := f.Fetch() + if !assert.NoError(t, err) { + t.FailNow() + } + + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event) +} + +func TestData(t *testing.T) { + f := mbtest.NewEventFetcher(t, getConfig()) + err := mbtest.WriteEvent(f, t) + if err != nil { + t.Fatal("write", err) + } +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "jolokia", + "metricsets": []string{"jmx"}, + "hosts": []string{getEnvHost() + ":" + getEnvPort()}, + "namespace": "testnamespace", + "jmx.mappings": []map[string]interface{}{ + { + "mbean": "java.lang:type=Runtime", + "attributes": []map[string]string{ + { + "attr": "Uptime", + "field": "uptime", + }, + }, + }, + { + "mbean": "java.lang:type=GarbageCollector,name=ConcurrentMarkSweep", + "attributes": []map[string]string{ + { + "attr": "CollectionTime", + "field": "gc.cms_collection_time", + }, + { + "attr": "CollectionCount", + "field": "gc.cms_collection_count", + }, + }, + }, + { + "mbean": "java.lang:type=Memory", + "attributes": []map[string]string{ + { + "attr": "HeapMemoryUsage", + "field": "memory.heap_usage", + }, + { + "attr": "NonHeapMemoryUsage", + "field": "memory.non_heap_usage", + }, + }, + }, + }, + } +} + +func getEnvHost() string { + host := os.Getenv("JOLOKIA_HOST") + + if len(host) == 0 { + host = "127.0.0.1" + } + return host +} + +func getEnvPort() string { + port := os.Getenv("JOLOKIA_PORT") + + if len(port) == 0 { + port = "8778" + } + return port +} diff --git a/metricbeat/tests/system/config/metricbeat.yml.j2 b/metricbeat/tests/system/config/metricbeat.yml.j2 index 8f49d711aa7..26c50b4e500 100644 --- a/metricbeat/tests/system/config/metricbeat.yml.j2 +++ b/metricbeat/tests/system/config/metricbeat.yml.j2 @@ -41,6 +41,10 @@ metricbeat.modules: timeout: {{ m.timeout }} {% endif -%} + {% if m.namespace -%} + namespace: {{ m.namespace }} + {% endif -%} + {% if m.processes -%} processes: {{ m.processes }} {% endif -%} @@ -78,6 +82,9 @@ metricbeat.modules: {{ k }}: {{ v }} {% endfor %} {% endif -%} + {% if m.additional_content -%} + {{ m.additional_content }} + {% endif -%} {%- endfor %} #================================ General ===================================== diff --git a/metricbeat/tests/system/test_jolokia.py b/metricbeat/tests/system/test_jolokia.py new file mode 100644 index 00000000000..6c37b46f25a --- /dev/null +++ b/metricbeat/tests/system/test_jolokia.py @@ -0,0 +1,44 @@ +import os +import metricbeat +import unittest +from nose.plugins.attrib import attr + + +class Test(metricbeat.BaseTest): + + @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") + def test_jmx(self): + """ + jolokia jmx metricset test + """ + + additional_content = """ + jmx.mappings: + - mbean: 'java.lang:type=Runtime' + attributes: + - attr: Uptime + field: uptime +""" + + self.render_config_template(modules=[{ + "name": "jolokia", + "metricsets": ["jmx"], + "hosts": self.get_hosts(), + "period": "1s", + "namespace": "test", + "additional_content": additional_content, + }]) + proc = self.start_beat() + self.wait_until(lambda: self.output_lines() > 0, max_timeout=20) + proc.check_kill_and_wait() + + output = self.read_output_json() + self.assertTrue(len(output) >= 1) + evt = output[0] + print(evt) + + assert evt["jolokia"]["test"]["uptime"] > 0 + + def get_hosts(self): + return [os.getenv('JOLOKIA_HOST', 'localhost') + ':' + + os.getenv('JOLOKIA_PORT', '8778')]