Foundation for Logstash Metricbeat module

CHANGELOG.asciidoc

@@ -120,6 +120,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 - Add etcd module. {issue}4970[4970]
 - Add ip address of docker containers to event. {pull}5379[5379]
 - Add ceph osd tree infomation to metricbeat {pull}5498[5498]
+- Add basic Logstash module. {pull}5540[5540]
 
 *Packetbeat*
 

metricbeat/docker-compose.yml

@@ -24,6 +24,7 @@ services:
       - ./module/kafka/_meta/env
       - ./module/kibana/_meta/env
       - ./module/kubernetes/_meta/env
+      - ./module/logstash/_meta/env
       - ./module/memcached/_meta/env
       - ./module/mongodb/_meta/env
       - ./module/mysql/_meta/env
@@ -96,6 +97,9 @@ services:
     depends_on:
       - kubernetes
 
+  logstash:
+    build: ./module/logstash/_meta
+
   memcached:
     build: ./module/memcached/_meta
 

metricbeat/docs/fields.asciidoc

@@ -33,6 +33,7 @@ grouped in the following categories:
 * <<exported-fields-kibana>>
 * <<exported-fields-kubernetes-processor>>
 * <<exported-fields-kubernetes>>
+* <<exported-fields-logstash>>
 * <<exported-fields-memcached>>
 * <<exported-fields-mongodb>>
 * <<exported-fields-mysql>>
@@ -6147,6 +6148,66 @@ type: long
 Total inodes
 
 
+[[exported-fields-logstash]]
+== Logstash fields
+
+experimental[]
+Logstash module
+
+
+
+[float]
+== logstash fields
+
+
+
+
+[float]
+== node fields
+
+node
+
+
+
+[float]
+=== `logstash.node.host`
+
+type: keyword
+
+Host name
+
+
+[float]
+=== `logstash.node.version`
+
+type: keyword
+
+Logstash Version
+
+
+[float]
+== jvm fields
+
+JVM Info
+
+
+
+[float]
+=== `logstash.node.jvm.version`
+
+type: keyword
+
+Version
+
+
+[float]
+=== `logstash.node.jvm.pid`
+
+type: long
+
+Pid
+
+
 [[exported-fields-memcached]]
 == Memcached fields
 

metricbeat/docs/modules/logstash.asciidoc

@@ -0,0 +1,37 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[metricbeat-module-logstash]]
+== logstash module
+
+This is the logstash module.
+
+
+
+[float]
+=== Example configuration
+
+The Logstash module supports the standard configuration options that are described
+in <<configuration-metricbeat>>. Here is an example configuration:
+
+[source,yaml]
+----
+metricbeat.modules:
+- module: logstash
+  metricsets: ["node"]
+  enabled: false
+  period: 10s
+  hosts: ["localhost"]
+
+----
+
+[float]
+=== Metricsets
+
+The following metricsets are available:
+
+* <<metricbeat-metricset-logstash-node,node>>
+
+include::logstash/node.asciidoc[]
+

metricbeat/docs/modules/logstash/node.asciidoc

@@ -0,0 +1,19 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[metricbeat-metricset-logstash-node]]
+include::../../../module/logstash/node/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-logstash,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/logstash/node/_meta/data.json[]
+----

metricbeat/docs/modules_list.asciidoc

@@ -18,6 +18,7 @@ This file is generated! See scripts/docs_collector.py
   * <<metricbeat-module-kafka,Kafka>>
   * <<metricbeat-module-kibana,Kibana>>
   * <<metricbeat-module-kubernetes,Kubernetes>>
+  * <<metricbeat-module-logstash,Logstash>>
   * <<metricbeat-module-memcached,Memcached>>
   * <<metricbeat-module-mongodb,MongoDB>>
   * <<metricbeat-module-mysql,MySQL>>
@@ -51,6 +52,7 @@ include::modules/jolokia.asciidoc[]
 include::modules/kafka.asciidoc[]
 include::modules/kibana.asciidoc[]
 include::modules/kubernetes.asciidoc[]
+include::modules/logstash.asciidoc[]
 include::modules/memcached.asciidoc[]
 include::modules/mongodb.asciidoc[]
 include::modules/mysql.asciidoc[]

metricbeat/include/list.go

@@ -72,6 +72,8 @@ import (
 	_ "github.com/elastic/beats/metricbeat/module/kubernetes/system"
 	_ "github.com/elastic/beats/metricbeat/module/kubernetes/util"
 	_ "github.com/elastic/beats/metricbeat/module/kubernetes/volume"
+	_ "github.com/elastic/beats/metricbeat/module/logstash"
+	_ "github.com/elastic/beats/metricbeat/module/logstash/node"
 	_ "github.com/elastic/beats/metricbeat/module/memcached"
 	_ "github.com/elastic/beats/metricbeat/module/memcached/stats"
 	_ "github.com/elastic/beats/metricbeat/module/mongodb"

metricbeat/metricbeat.reference.yml

@@ -287,6 +287,14 @@ metricbeat.modules:
   metricsets:
     - event
 
+#------------------------------ Logstash Module ------------------------------
+- module: logstash
+  metricsets: ["node"]
+  enabled: false
+  period: 10s
+  hosts: ["localhost"]
+
+
 #------------------------------ Memcached Module -----------------------------
 - module: memcached
   metricsets: ["stats"]

metricbeat/module/logstash/_meta/Dockerfile

@@ -0,0 +1,4 @@
+FROM docker.elastic.co/logstash/logstash:6.0.0-rc2
+
+ENV XPACK_MONITORING_ENABLED=FALSE
+HEALTHCHECK --interval=1s --retries=90 CMD curl -f http://localhost:9600

metricbeat/module/logstash/_meta/config.yml

@@ -0,0 +1,6 @@
+- module: logstash
+  metricsets: ["node"]
+  enabled: false
+  period: 10s
+  hosts: ["localhost"]
+

metricbeat/module/logstash/_meta/docs.asciidoc

@@ -0,0 +1,4 @@
+== logstash module
+
+This is the logstash module.
+

metricbeat/module/logstash/_meta/env

@@ -0,0 +1,2 @@
+LOGSTASH_HOST=logstash
+LOGSTASH_PORT=9600

metricbeat/module/logstash/_meta/fields.yml

@@ -0,0 +1,13 @@
+- key: logstash
+  title: "Logstash"
+  description: >
+    experimental[]
+
+    Logstash module
+  fields:
+    - name: logstash
+      type: group
+      description: >
+      fields:
+
+

metricbeat/module/logstash/doc.go

@@ -0,0 +1,4 @@
+/*
+Package logstash is a Metricbeat module that contains MetricSets.
+*/
+package logstash

metricbeat/module/logstash/node/_meta/data.json

@@ -0,0 +1,23 @@
+{
+    "@timestamp": "2017-10-12T08:05:34.853Z",
+    "beat": {
+        "hostname": "host.example.com",
+        "name": "host.example.com"
+    },
+    "logstash": {
+        "node": {
+            "host": "9b61b9bdb06a",
+            "jvm": {
+                "pid": 1,
+                "version": "1.8.0_151"
+            },
+            "version": "6.0.0-rc2"
+        }
+    },
+    "metricset": {
+        "host": "127.0.0.1:9600",
+        "module": "logstash",
+        "name": "node",
+        "rtt": 115
+    }
+}

metricbeat/module/logstash/node/_meta/docs.asciidoc

@@ -0,0 +1,3 @@
+=== logstash node MetricSet
+
+This is the node metricset of the module logstash.

metricbeat/module/logstash/node/_meta/fields.yml

@@ -0,0 +1,26 @@
+- name: node
+  type: group
+  description: >
+    node
+  fields:
+    - name: host
+      type: keyword
+      description: >
+        Host name
+    - name: version
+      type: keyword
+      description: >
+        Logstash Version
+    - name: jvm
+      type: group
+      description: >
+        JVM Info
+      fields:
+        - name: version
+          type: keyword
+          description: >
+            Version
+        - name: pid
+          type: long
+          description: >
+            Pid

metricbeat/module/logstash/node/data.go

@@ -0,0 +1,22 @@
+package node
+
+import (
+	"github.com/elastic/beats/libbeat/common"
+	s "github.com/elastic/beats/libbeat/common/schema"
+	c "github.com/elastic/beats/libbeat/common/schema/mapstriface"
+)
+
+var (
+	schema = s.Schema{
+		"host":    c.Str("host"),
+		"version": c.Str("version"),
+		"jvm": c.Dict("jvm", s.Schema{
+			"version": c.Str("version"),
+			"pid":     c.Int("pid"),
+		}),
+	}
+)
+
+func eventMapping(node map[string]interface{}) (common.MapStr, *s.Errors) {
+	return schema.Apply(node)
+}

metricbeat/module/logstash/node/node.go

@@ -0,0 +1,51 @@
+package node
+
+import (
+	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/metricbeat/helper"
+	"github.com/elastic/beats/metricbeat/mb"
+	"github.com/elastic/beats/metricbeat/mb/parse"
+)
+
+// init registers the MetricSet with the central registry.
+// The New method will be called after the setup of the module and before starting to fetch data
+func init() {
+	if err := mb.Registry.AddMetricSet("logstash", "node", New, hostParser); err != nil {
+		panic(err)
+	}
+}
+
+var (
+	hostParser = parse.URLHostParserBuilder{
+		DefaultScheme: "http",
+		PathConfigKey: "path",
+		DefaultPath:   "_node",
+	}.Build()
+)
+
+// MetricSet type defines all fields of the MetricSet
+type MetricSet struct {
+	mb.BaseMetricSet
+	http *helper.HTTP
+}
+
+// New create a new instance of the MetricSet
+func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
+	return &MetricSet{
+		base,
+		helper.NewHTTP(base),
+	}, nil
+}
+
+// Fetch methods implements the data gathering and data conversion to the right format
+// It returns the event which is then forward to the output. In case of an error, a
+// descriptive error must be returned.
+func (m *MetricSet) Fetch() (common.MapStr, error) {
+	data, err := m.http.FetchJSON()
+	if err != nil {
+		return nil, err
+	}
+
+	event, _ := eventMapping(data)
+	return event, nil
+}