filebeat: add Kafka log module

This module is based on
https://www.elastic.co/blog/monitoring-kafka-with-elastic-stack-1-filebeat

CHANGELOG.asciidoc

@@ -74,6 +74,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta1...master[Check the HEAD di
 *Filebeat*
 
 - Add PostgreSQL module with slowlog support. {pull}4763[4763]
+- Add Kafka log module. {pull}4885[4885]
 
 *Heartbeat*
 

filebeat/docs/fields.asciidoc

@@ -18,6 +18,7 @@ grouped in the following categories:
 * <<exported-fields-cloud>>
 * <<exported-fields-docker-processor>>
 * <<exported-fields-icinga>>
+* <<exported-fields-kafka>>
 * <<exported-fields-kubernetes-processor>>
 * <<exported-fields-log>>
 * <<exported-fields-mysql>>
@@ -749,6 +750,95 @@ type: text
 The logged message.
 
 
+[[exported-fields-kafka]]
+== Kafka fields
+
+Kafka module
+
+
+
+[float]
+== kafka fields
+
+
+
+
+[float]
+== log fields
+
+Kafka log lines.
+
+
+
+[float]
+=== `kafka.log.timestamp`
+
+The timestamp from the log line.
+
+
+[float]
+=== `kafka.log.level`
+
+example: WARN
+
+The log level.
+
+
+[float]
+=== `kafka.log.message`
+
+type: text
+
+The logged message.
+
+
+[float]
+=== `kafka.log.component`
+
+type: keyword
+
+Component the log is coming from.
+
+
+[float]
+=== `kafka.log.class`
+
+type: text
+
+Java class the log is coming from.
+
+
+[float]
+== trace fields
+
+Trace in the log line.
+
+
+
+[float]
+=== `kafka.log.trace.class`
+
+type: keyword
+
+Java class the trace is coming from.
+
+
+[float]
+=== `kafka.log.trace.message`
+
+type: text
+
+Message part of the trace.
+
+
+[float]
+=== `kafka.log.trace.full`
+
+type: text
+
+The full trace in the log line.
+
+
 [[exported-fields-kubernetes-processor]]
 == Kubernetes fields
 

filebeat/docs/modules/kafka.asciidoc

@@ -0,0 +1,37 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[filebeat-module-kafka]]
+== Kafka module
+
+This module collects and parses the logs created by https://kafka.apache.org/[Kafka].
+
+[float]
+=== Compatibility
+
+The Kafka module was tested with logs from versions 2.11.
+
+[float]
+=== Dashboard
+
+This module comes with a sample dashboard to see Kafka logs and stacktraces.
+
+image::./images/filebeat-kafka-logs-overview.png[]
+
+[float]
+=== Logs fileset settings
+
+[float]
+==== var.paths
+
+An array of paths where to look for the log files. If left empty, Filebeat
+will choose the paths depending on your operating systems.
+
+
+[float]
+=== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-kafka,exported fields>> section.
+

filebeat/docs/modules_list.asciidoc

@@ -6,6 +6,7 @@ This file is generated! See scripts/docs_collector.py
   * <<filebeat-module-apache2>>
   * <<filebeat-module-auditd>>
   * <<filebeat-module-icinga>>
+  * <<filebeat-module-kafka>>
   * <<filebeat-module-mysql>>
   * <<filebeat-module-nginx>>
   * <<filebeat-module-postgresql>>
@@ -19,6 +20,7 @@ include::modules-overview.asciidoc[]
 include::modules/apache2.asciidoc[]
 include::modules/auditd.asciidoc[]
 include::modules/icinga.asciidoc[]
+include::modules/kafka.asciidoc[]
 include::modules/mysql.asciidoc[]
 include::modules/nginx.asciidoc[]
 include::modules/postgresql.asciidoc[]

filebeat/filebeat.reference.yml

@@ -114,6 +114,21 @@ filebeat.modules:
     # can be added under this section.
     #prospector:
 
+#-------------------------------- Kafka Module -------------------------------
+- module: kafka
+  # All logs
+  log:
+    enabled: true
+
+    # Set custom paths for Kafka. If left empty,
+    # Filebeat will look under /opt.
+    #var.kafka_home:
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
+
 #-------------------------------- MySQL Module -------------------------------
 #- module: mysql
   # Error logs

filebeat/module/kafka/_meta/config.yml

@@ -0,0 +1,13 @@
+- module: kafka
+  # All logs
+  log:
+    enabled: true
+
+    # Set custom paths for Kafka. If left empty,
+    # Filebeat will look under /opt.
+    #var.kafka_home:
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+

filebeat/module/kafka/_meta/docs.asciidoc

@@ -0,0 +1,24 @@
+== Kafka module
+
+This module collects and parses the logs created by https://kafka.apache.org/[Kafka].
+
+[float]
+=== Compatibility
+
+The Kafka module was tested with logs from versions 2.11.
+
+[float]
+=== Dashboard
+
+This module comes with a sample dashboard to see Kafka logs and stacktraces.
+
+image::./images/filebeat-kafka-logs-overview.png[]
+
+[float]
+=== Logs fileset settings
+
+[float]
+==== var.paths
+
+An array of paths where to look for the log files. If left empty, Filebeat
+will choose the paths depending on your operating systems.

filebeat/module/kafka/_meta/fields.yml

@@ -0,0 +1,9 @@
+- key: kafka
+  title: "Kafka"
+  description: >
+    Kafka module
+  fields:
+    - name: kafka
+      type: group
+      description: >
+      fields:

filebeat/module/kafka/_meta/kibana/default/dashboard/Filebeat-Kafka-overview.json

@@ -0,0 +1,85 @@
+{
+  "objects": [
+    {
+      "attributes": {
+        "description": "",
+        "kibanaSavedObjectMeta": {
+          "searchSourceJSON": "{\"filter\":[]}"
+        },
+        "savedSearchId": "Kafka stacktraces",
+        "title": "Number of Kafka stracktraces by class",
+        "uiStateJSON": "{}",
+        "version": 1,
+        "visState": "{\"title\":\"Number of Kafka stracktraces by class\",\"type\":\"histogram\",\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{\"text\":\"@timestamp per 30 minutes\"}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"kafka.log.trace.class\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}}]}"
+      },
+      "id": "Number of Kafka stracktraces by class",
+      "type": "visualization",
+      "version": 1
+    },
+    {
+      "attributes": {
+        "columns": [
+          "kafka.log.class",
+          "kafka.log.trace.class",
+          "kafka.log.trace.full"
+        ],
+        "description": "",
+        "hits": 0,
+        "kibanaSavedObjectMeta": {
+          "searchSourceJSON": "{\"index\":\"filebeat-*\",\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"_exists_:kafka.log.trace.class\",\"language\":\"lucene\"},\"filter\":[]}"
+        },
+        "sort": [
+          "@timestamp",
+          "desc"
+        ],
+        "title": "Kafka stacktraces",
+        "version": 1
+      },
+      "id": "Kafka stacktraces",
+      "type": "search",
+      "version": 1
+    },
+    {
+      "attributes": {
+        "columns": [
+          "kafka.log.level",
+          "kafka.log.component",
+          "kafka.log.message"
+        ],
+        "description": "",
+        "hits": 0,
+        "kibanaSavedObjectMeta": {
+          "searchSourceJSON": "{\"index\":\"filebeat-*\",\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"kafka.log.level:*\",\"language\":\"lucene\"},\"filter\":[]}"
+        },
+        "sort": [
+          "@timestamp",
+          "desc"
+        ],
+        "title": "All Kafka logs",
+        "version": 1
+      },
+      "id": "All Kafka logs",
+      "type": "search",
+      "version": 1
+    },
+    {
+      "attributes": {
+        "description": "",
+        "hits": 0,
+        "kibanaSavedObjectMeta": {
+          "searchSourceJSON": "{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[],\"highlightAll\":true,\"version\":true}"
+        },
+        "optionsJSON": "{\"darkTheme\":false}",
+        "panelsJSON": "[{\"col\":1,\"id\":\"Number of Kafka stracktraces by class\",\"panelIndex\":1,\"row\":1,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"columns\":[\"kafka.log.class\",\"kafka.log.trace.class\",\"kafka.log.trace.full\"],\"id\":\"Kafka stacktraces\",\"panelIndex\":2,\"row\":1,\"size_x\":6,\"size_y\":3,\"sort\":[\"@timestamp\",\"desc\"],\"type\":\"search\"},{\"size_x\":12,\"size_y\":3,\"panelIndex\":3,\"type\":\"search\",\"id\":\"All Kafka logs\",\"col\":1,\"row\":4,\"columns\":[\"kafka.log.level\",\"kafka.log.component\",\"kafka.log.message\"],\"sort\":[\"@timestamp\",\"desc\"]}]",
+        "timeRestore": false,
+        "title": "Kafka logs overview",
+        "uiStateJSON": "{}",
+        "version": 1
+      },
+      "id": "Kafka logs overview",
+      "type": "dashboard",
+      "version": 1
+    }
+  ],
+  "version": "6.0.0-beta1-SNAPSHOT"
+}

filebeat/module/kafka/log/_meta/fields.yml

@@ -0,0 +1,41 @@
+- name: log
+  type: group
+  description: >
+    Kafka log lines.
+  fields:
+    - name: timestamp
+      description: >
+        The timestamp from the log line.
+    - name: level
+      example: "WARN"
+      description: >
+        The log level.
+    - name: message
+      type: text
+      description: >
+        The logged message.
+    - name: component
+      type: keyword
+      description: >
+        Component the log is coming from.
+    - name: class
+      type: text
+      description: >
+        Java class the log is coming from.
+    - name: trace
+      type: group
+      description: >
+          Trace in the log line.
+      fields:
+        - name: class
+          type: keyword
+          description: >
+            Java class the trace is coming from.
+        - name: message
+          type: text
+          description: >
+              Message part of the trace.
+        - name: full
+          type: text
+          description: >
+              The full trace in the log line.

filebeat/module/kafka/log/config/log.yml

@@ -0,0 +1,10 @@
+type: log
+paths:
+{{ range $i, $path := .paths }}
+ - {{$path}}
+{{ end }}
+exclude_files: [".gz$"]
+multiline:
+  pattern: '^\['
+  negate: true
+  match: after