From 5bb43b4e7055d56b5684740fa4d014e0956a561a Mon Sep 17 00:00:00 2001 From: ruflin Date: Mon, 3 Apr 2017 16:04:48 +0200 Subject: [PATCH] Add experimental Kibana module to Metricbeat The Kibana module is marked experimental as it might change or removed at any time. --- CHANGELOG.asciidoc | 1 + metricbeat/docker-compose.yml | 7 ++ metricbeat/docs/fields.asciidoc | 92 ++++++++++++++++++ metricbeat/docs/modules/kibana.asciidoc | 37 ++++++++ .../docs/modules/kibana/status.asciidoc | 19 ++++ metricbeat/docs/modules_list.asciidoc | 2 + metricbeat/include/list.go | 2 + metricbeat/metricbeat.full.yml | 8 ++ metricbeat/metricbeat.yml | 8 ++ metricbeat/module/kibana/_meta/config.yml | 6 ++ metricbeat/module/kibana/_meta/docs.asciidoc | 4 + metricbeat/module/kibana/_meta/env | 2 + metricbeat/module/kibana/_meta/fields.yml | 11 +++ metricbeat/module/kibana/doc.go | 4 + .../module/kibana/status/_meta/data.json | 35 +++++++ .../module/kibana/status/_meta/docs.asciidoc | 7 ++ .../module/kibana/status/_meta/fields.yml | 43 +++++++++ .../kibana/status/_meta/test/input.json | 95 +++++++++++++++++++ metricbeat/module/kibana/status/data.go | 43 +++++++++ metricbeat/module/kibana/status/data_test.go | 22 +++++ metricbeat/module/kibana/status/status.go | 51 ++++++++++ .../kibana/status/status_integration_test.go | 55 +++++++++++ metricbeat/tests/system/test_kibana.py | 32 +++++++ 23 files changed, 586 insertions(+) create mode 100644 metricbeat/docs/modules/kibana.asciidoc create mode 100644 metricbeat/docs/modules/kibana/status.asciidoc create mode 100644 metricbeat/module/kibana/_meta/config.yml create mode 100644 metricbeat/module/kibana/_meta/docs.asciidoc create mode 100644 metricbeat/module/kibana/_meta/env create mode 100644 metricbeat/module/kibana/_meta/fields.yml create mode 100644 metricbeat/module/kibana/doc.go create mode 100644 metricbeat/module/kibana/status/_meta/data.json create mode 100644 metricbeat/module/kibana/status/_meta/docs.asciidoc create mode 100644 metricbeat/module/kibana/status/_meta/fields.yml create mode 100644 metricbeat/module/kibana/status/_meta/test/input.json create mode 100644 metricbeat/module/kibana/status/data.go create mode 100644 metricbeat/module/kibana/status/data_test.go create mode 100644 metricbeat/module/kibana/status/status.go create mode 100644 metricbeat/module/kibana/status/status_integration_test.go create mode 100644 metricbeat/tests/system/test_kibana.py diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 1d11eea8440..81df42a25b3 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -35,6 +35,7 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff] *Packetbeat* - Remove deprecated geoip. {pull}3766[3766] +- Add experimental Kibana module. {pull}3895[3895] *Winlogbeat* - Remove metrics endpoint. Replaced by http endpoint in libbeat (see #3717). {pull}3901[3901] diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml index 6f02b070f60..ad1ecec93b4 100644 --- a/metricbeat/docker-compose.yml +++ b/metricbeat/docker-compose.yml @@ -22,6 +22,7 @@ services: - ${PWD}/module/haproxy/_meta/env - ${PWD}/module/jolokia/_meta/env - ${PWD}/module/kafka/_meta/env + - ${PWD}/module/kibana/_meta/env - ${PWD}/module/memcached/_meta/env - ${PWD}/module/mongodb/_meta/env - ${PWD}/module/mysql/_meta/env @@ -43,6 +44,7 @@ services: haproxy: { condition: service_healthy } jolokia: { condition: service_healthy } kafka: { condition: service_healthy } + kibana: { condition: service_healthy } memcached: { condition: service_healthy } mongodb: { condition: service_healthy } mysql: { condition: service_healthy } @@ -72,6 +74,11 @@ services: kafka: build: ${PWD}/module/kafka/_meta + kibana: + extends: + file: ${ES_BEATS}/testing/environments/${TESTING_ENVIRONMENT}.yml + service: kibana + memcached: build: ${PWD}/module/memcached/_meta diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 2f8e117112f..a228a9d9098 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -23,6 +23,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -3330,6 +3331,97 @@ type: keyword Broker address +[[exported-fields-kibana]] +== kibana Fields + +[]experimental +kibana Module + + + +[float] +== kibana Fields + + + + +[float] +== status Fields + +Status fields + + + +[float] +=== kibana.status.name + +type: keyword + +Kibana instance name. + + +[float] +=== kibana.status.uuid + +type: keyword + +Kibana instance uuid. + + +[float] +=== kibana.status.version.number + +type: keyword + +Kibana version number. + + +[float] +=== kibana.status.status.overall.state + +type: keyword + +Kibana overall state. + + +[float] +== metrics Fields + +Metrics fields + + + +[float] +=== kibana.status.metrics.concurrent_connections + +type: long + +Current concurrent connections. + + +[float] +== requests Fields + +Request statistics. + + + +[float] +=== kibana.status.metrics.requests.disconnects + +type: long + +Total number of disconnected connections. + + +[float] +=== kibana.status.metrics.requests.total + +type: long + +Total number of connections. + + [[exported-fields-kubelet]] == kubelet Fields diff --git a/metricbeat/docs/modules/kibana.asciidoc b/metricbeat/docs/modules/kibana.asciidoc new file mode 100644 index 00000000000..a7b49692507 --- /dev/null +++ b/metricbeat/docs/modules/kibana.asciidoc @@ -0,0 +1,37 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-module-kibana]] +== kibana Module + +This is the kibana Module. + + + +[float] +=== Example Configuration + +The kibana module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: kibana + metricsets: ["status"] + enabled: true + period: 10s + hosts: ["localhost:5601"] + +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +include::kibana/status.asciidoc[] + diff --git a/metricbeat/docs/modules/kibana/status.asciidoc b/metricbeat/docs/modules/kibana/status.asciidoc new file mode 100644 index 00000000000..8a71633e986 --- /dev/null +++ b/metricbeat/docs/modules/kibana/status.asciidoc @@ -0,0 +1,19 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-kibana-status]] +include::../../../module/kibana/status/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/kibana/status/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 505dd3e6341..43adfa393c0 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -10,6 +10,7 @@ This file is generated! See scripts/docs_collector.py * <> * <> * <> + * <> * <> * <> * <> @@ -34,6 +35,7 @@ include::modules/golang.asciidoc[] include::modules/haproxy.asciidoc[] include::modules/jolokia.asciidoc[] include::modules/kafka.asciidoc[] +include::modules/kibana.asciidoc[] include::modules/kubelet.asciidoc[] include::modules/memcached.asciidoc[] include::modules/mongodb.asciidoc[] diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go index 93dd66bf6e7..28581540a11 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list.go @@ -39,6 +39,8 @@ import ( _ "github.com/elastic/beats/metricbeat/module/kafka" _ "github.com/elastic/beats/metricbeat/module/kafka/consumergroup" _ "github.com/elastic/beats/metricbeat/module/kafka/partition" + _ "github.com/elastic/beats/metricbeat/module/kibana" + _ "github.com/elastic/beats/metricbeat/module/kibana/status" _ "github.com/elastic/beats/metricbeat/module/kubelet" _ "github.com/elastic/beats/metricbeat/module/kubelet/container" _ "github.com/elastic/beats/metricbeat/module/kubelet/node" diff --git a/metricbeat/metricbeat.full.yml b/metricbeat/metricbeat.full.yml index 8dac5646c54..b809e7de2d4 100644 --- a/metricbeat/metricbeat.full.yml +++ b/metricbeat/metricbeat.full.yml @@ -182,6 +182,14 @@ metricbeat.modules: #username: "" #password: "" +#------------------------------- kibana Module ------------------------------- +- module: kibana + metricsets: ["status"] + enabled: true + period: 10s + hosts: ["localhost:5601"] + + #------------------------------- kubelet Module ------------------------------ #- module: kubelet # metricsets: ["node","container","volume","pod","system"] diff --git a/metricbeat/metricbeat.yml b/metricbeat/metricbeat.yml index 657ea61828d..9153956e01d 100644 --- a/metricbeat/metricbeat.yml +++ b/metricbeat/metricbeat.yml @@ -46,6 +46,14 @@ metricbeat.modules: period: 10s processes: ['.*'] +#------------------------------- kibana Module ------------------------------- +- module: kibana + metricsets: ["status"] + enabled: true + period: 10s + hosts: ["localhost:5601"] + + #================================ General ===================================== diff --git a/metricbeat/module/kibana/_meta/config.yml b/metricbeat/module/kibana/_meta/config.yml new file mode 100644 index 00000000000..464773ae911 --- /dev/null +++ b/metricbeat/module/kibana/_meta/config.yml @@ -0,0 +1,6 @@ +- module: kibana + metricsets: ["status"] + enabled: true + period: 10s + hosts: ["localhost:5601"] + diff --git a/metricbeat/module/kibana/_meta/docs.asciidoc b/metricbeat/module/kibana/_meta/docs.asciidoc new file mode 100644 index 00000000000..b88abe0ee65 --- /dev/null +++ b/metricbeat/module/kibana/_meta/docs.asciidoc @@ -0,0 +1,4 @@ +== kibana Module + +This is the kibana Module. + diff --git a/metricbeat/module/kibana/_meta/env b/metricbeat/module/kibana/_meta/env new file mode 100644 index 00000000000..a22fc93ec70 --- /dev/null +++ b/metricbeat/module/kibana/_meta/env @@ -0,0 +1,2 @@ +KIBANA_HOST=kibana +KIBANA_PORT=5601 \ No newline at end of file diff --git a/metricbeat/module/kibana/_meta/fields.yml b/metricbeat/module/kibana/_meta/fields.yml new file mode 100644 index 00000000000..499a9f03431 --- /dev/null +++ b/metricbeat/module/kibana/_meta/fields.yml @@ -0,0 +1,11 @@ +- key: kibana + title: "kibana" + description: > + []experimental + + kibana Module + fields: + - name: kibana + type: group + description: > + fields: diff --git a/metricbeat/module/kibana/doc.go b/metricbeat/module/kibana/doc.go new file mode 100644 index 00000000000..2ddab58b555 --- /dev/null +++ b/metricbeat/module/kibana/doc.go @@ -0,0 +1,4 @@ +/* +Package kibana is a Metricbeat module that contains MetricSets. +*/ +package kibana diff --git a/metricbeat/module/kibana/status/_meta/data.json b/metricbeat/module/kibana/status/_meta/data.json new file mode 100644 index 00000000000..6e0d8089239 --- /dev/null +++ b/metricbeat/module/kibana/status/_meta/data.json @@ -0,0 +1,35 @@ +{ + "@timestamp": "2016-05-23T08:05:34.853Z", + "beat": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "kibana": { + "status": { + "metrics": { + "concurrent_connections": 0, + "requests": { + "disconnects": 0, + "total": 0 + } + }, + "name": "kibana", + "status": { + "overall": { + "state": "red" + } + }, + "uuid": "c94305f7-ad4e-4e0c-8aba-5c3e869dce59", + "version": { + "number": "6.0.0-alpha1" + } + } + }, + "metricset": { + "host": "kibana:5601", + "module": "kibana", + "name": "status", + "rtt": 115 + }, + "type": "metricsets" +} \ No newline at end of file diff --git a/metricbeat/module/kibana/status/_meta/docs.asciidoc b/metricbeat/module/kibana/status/_meta/docs.asciidoc new file mode 100644 index 00000000000..a38c46b6af7 --- /dev/null +++ b/metricbeat/module/kibana/status/_meta/docs.asciidoc @@ -0,0 +1,7 @@ +=== kibana status MetricSet + +[]experimental + +This is the status metricset of the module kibana. This status endpoint is available in 6.0 by default and can be enabled in Kibana >= 5.4 with the config option `status.v6ApiFormat: true`. + +The indention of this Kibana module is to have a minimal data set that works across Kibana versions. diff --git a/metricbeat/module/kibana/status/_meta/fields.yml b/metricbeat/module/kibana/status/_meta/fields.yml new file mode 100644 index 00000000000..3a109d349c9 --- /dev/null +++ b/metricbeat/module/kibana/status/_meta/fields.yml @@ -0,0 +1,43 @@ +- name: status + type: group + description: > + Status fields + fields: + - name: name + type: keyword + description: > + Kibana instance name. + - name: uuid + type: keyword + description: > + Kibana instance uuid. + - name: version.number + type: keyword + description: > + Kibana version number. + - name: status.overall.state + type: keyword + description: > + Kibana overall state. + - name: metrics + type: group + description: > + Metrics fields + fields: + - name: concurrent_connections + type: long + description: > + Current concurrent connections. + - name: requests + type: group + description: > + Request statistics. + fields: + - name: disconnects + type: long + description: > + Total number of disconnected connections. + - name: total + type: long + description: > + Total number of connections. diff --git a/metricbeat/module/kibana/status/_meta/test/input.json b/metricbeat/module/kibana/status/_meta/test/input.json new file mode 100644 index 00000000000..9d7f53ba7a2 --- /dev/null +++ b/metricbeat/module/kibana/status/_meta/test/input.json @@ -0,0 +1,95 @@ +{ + "name": "ruflin", + "uuid": "5b2de169-2785-441b-ae8c-186a1936b17d", + "version": { + "number": "6.0.0-alpha1", + "build_hash": "6cb7fec4e154faa0a4a3fee4b33dfef91b9870d9", + "build_number": 8467, + "build_snapshot": false + }, + "status": { + "overall": { + "state": "green", + "title": "Green", + "nickname": "Looking good", + "icon": "success", + "since": "2017-02-17T13:03:36.960Z" + }, + "statuses": [ + { + "id": "ui settings", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2017-02-17T13:03:40.088Z" + }, + { + "id": "plugin:kibana@6.0.0-alpha1", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2017-02-17T13:03:36.960Z" + }, + { + "id": "plugin:elasticsearch@6.0.0-alpha1", + "state": "green", + "icon": "success", + "message": "Kibana index ready", + "since": "2017-02-17T13:03:37.265Z" + }, + { + "id": "plugin:thor@6.0.0-alpha1", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2017-02-17T13:03:37.162Z" + }, + { + "id": "plugin:console@6.0.0-alpha1", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2017-02-17T13:03:37.210Z" + }, + { + "id": "plugin:timelion@6.0.0-alpha1", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2017-02-17T13:03:40.047Z" + } + ] + }, + "metrics": { + "last_updated": "2017-02-17T13:05:00.083Z", + "collection_interval_in_millis": 5000, + "uptime_in_millis": 90637, + "process": { + "mem": { + "heap_max_in_bytes": 150245376, + "heap_used_in_bytes": 114918592 + } + }, + "os": { + "cpu": { + "load_average": { + "1m": 7.412109375, + "5m": 5.3896484375, + "15m": 5.3896484375 + } + } + }, + "response_times": { + "avg_in_millis": null, + "max_in_millis": 0 + }, + "requests": { + "total": 241, + "disconnects": 3, + "status_codes": { + + } + }, + "concurrent_connections": 12 + } +} \ No newline at end of file diff --git a/metricbeat/module/kibana/status/data.go b/metricbeat/module/kibana/status/data.go new file mode 100644 index 00000000000..ce72d24dfe5 --- /dev/null +++ b/metricbeat/module/kibana/status/data.go @@ -0,0 +1,43 @@ +package status + +import ( + "encoding/json" + + "github.com/elastic/beats/libbeat/common" + s "github.com/elastic/beats/metricbeat/schema" + c "github.com/elastic/beats/metricbeat/schema/mapstriface" +) + +var ( + schema = s.Schema{ + "uuid": c.Str("uuid"), + "name": c.Str("name"), + "version": c.Dict("version", s.Schema{ + "number": c.Str("number"), + }), + "status": c.Dict("status", s.Schema{ + "overall": c.Dict("overall", s.Schema{ + "state": c.Str("state"), + }), + }), + "metrics": c.Dict("metrics", s.Schema{ + "requests": c.Dict("requests", s.Schema{ + "total": c.Int("total"), + "disconnects": c.Int("disconnects"), + }), + "concurrent_connections": c.Int("concurrent_connections"), + }), + } +) + +type OverallMetrics struct { + Metrics map[string][][]uint64 +} + +func eventMapping(content []byte) common.MapStr { + + var data map[string]interface{} + json.Unmarshal(content, &data) + event, _ := schema.Apply(data) + return event +} diff --git a/metricbeat/module/kibana/status/data_test.go b/metricbeat/module/kibana/status/data_test.go new file mode 100644 index 00000000000..d95083a5c7a --- /dev/null +++ b/metricbeat/module/kibana/status/data_test.go @@ -0,0 +1,22 @@ +// +build !integration + +package status + +import ( + "io/ioutil" + "testing" + + "github.com/elastic/beats/libbeat/common" + + "github.com/stretchr/testify/assert" +) + +func TestEventMapping(t *testing.T) { + + content, err := ioutil.ReadFile("./_meta/test/input.json") + assert.NoError(t, err) + + event := eventMapping(content) + + assert.Equal(t, event["metrics"].(common.MapStr)["concurrent_connections"], int64(12)) +} diff --git a/metricbeat/module/kibana/status/status.go b/metricbeat/module/kibana/status/status.go new file mode 100644 index 00000000000..06abb578c39 --- /dev/null +++ b/metricbeat/module/kibana/status/status.go @@ -0,0 +1,51 @@ +package status + +import ( + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/helper" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/mb/parse" +) + +// init registers the MetricSet with the central registry. +// The New method will be called after the setup of the module and before starting to fetch data +func init() { + if err := mb.Registry.AddMetricSet("kibana", "status", New, hostParser); err != nil { + panic(err) + } +} + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: "http", + PathConfigKey: "path", + DefaultPath: "api/status", + }.Build() +) + +// MetricSet type defines all fields of the MetricSet +type MetricSet struct { + mb.BaseMetricSet + http *helper.HTTP +} + +// New create a new instance of the MetricSet +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + return &MetricSet{ + base, + helper.NewHTTP(base), + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right format +// It returns the event which is then forward to the output. In case of an error, a +// descriptive error must be returned. +func (m *MetricSet) Fetch() (common.MapStr, error) { + + content, err := m.http.FetchContent() + if err != nil { + return nil, err + } + + return eventMapping(content), nil +} diff --git a/metricbeat/module/kibana/status/status_integration_test.go b/metricbeat/module/kibana/status/status_integration_test.go new file mode 100644 index 00000000000..4103b33c862 --- /dev/null +++ b/metricbeat/module/kibana/status/status_integration_test.go @@ -0,0 +1,55 @@ +// +build integration + +package status + +import ( + "os" + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/stretchr/testify/assert" +) + +func TestFetch(t *testing.T) { + f := mbtest.NewEventFetcher(t, getConfig()) + event, err := f.Fetch() + if !assert.NoError(t, err) { + t.FailNow() + } + + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event) +} + +func TestData(t *testing.T) { + f := mbtest.NewEventFetcher(t, getConfig()) + err := mbtest.WriteEvent(f, t) + if err != nil { + t.Fatal("write", err) + } +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "kibana", + "metricsets": []string{"status"}, + "hosts": []string{GetEnvHost() + ":" + GetEnvPort()}, + } +} + +func GetEnvHost() string { + host := os.Getenv("KIBANA_HOST") + + if len(host) == 0 { + host = "127.0.0.1" + } + return host +} + +func GetEnvPort() string { + port := os.Getenv("KIBANA_PORT") + + if len(port) == 0 { + port = "5601" + } + return port +} diff --git a/metricbeat/tests/system/test_kibana.py b/metricbeat/tests/system/test_kibana.py new file mode 100644 index 00000000000..e112680f7ac --- /dev/null +++ b/metricbeat/tests/system/test_kibana.py @@ -0,0 +1,32 @@ +import os +import metricbeat +import unittest + + +class Test(metricbeat.BaseTest): + + @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") + def test_status(self): + """ + kibana status metricset test + """ + self.render_config_template(modules=[{ + "name": "kibana", + "metricsets": ["status"], + "hosts": self.get_hosts(), + "period": "1s" + }]) + proc = self.start_beat() + self.wait_until(lambda: self.output_lines() > 0, max_timeout=20) + proc.check_kill_and_wait() + + output = self.read_output_json() + self.assertTrue(len(output) >= 1) + evt = output[0] + print evt + + self.assert_fields_are_documented(evt) + + def get_hosts(self): + return [os.getenv('KIBANA_HOST', 'localhost') + ':' + + os.getenv('KIBANA_PORT', '5601')]