Redis module

Contains two filesets:

* One for collecting & parsing the logs written on disk by Redis
* One that uses the Redis prospector to interrogate the slow logs

This also moves beat.read_timestamp to read_timestamp to match the rest of the modules.

CHANGELOG.asciidoc

@@ -46,6 +46,7 @@ https://github.com/elastic/beats/compare/v6.0.0-alpha1...master[Check the HEAD d
 
 *Filebeat*
 
+- Add experimental Redis module. {pull}4441[4441]
 - Nginx module: use the first not-private IP address as the remote_ip. {pull}4417[4417]
 
 *Heartbeat*

filebeat/_meta/common.full.p2.yml

@@ -212,7 +212,7 @@ filebeat.prospectors:
 
 #------------------------- Redis slowlog prospector ---------------------------
 # Experimental: Config options for the redis slow log prospector
-#- input_type: redis
+#- type: redis
   #hosts: ["localhost:6379"]
   #username:
   #password:

filebeat/docs/fields.asciidoc

@@ -21,6 +21,7 @@ grouped in the following categories:
 * <<exported-fields-log>>
 * <<exported-fields-mysql>>
 * <<exported-fields-nginx>>
+* <<exported-fields-redis>>
 * <<exported-fields-system>>
 
 --
@@ -1234,6 +1235,105 @@ type: text
 The error message
 
 
+[[exported-fields-redis]]
+== redis Fields
+
+redis Module
+
+
+
+[float]
+== redis Fields
+
+
+
+
+[float]
+== log Fields
+
+Redis log files
+
+
+
+[float]
+=== redis.log.pid
+
+type: long
+
+The process ID of the Redis server.
+
+
+[float]
+=== redis.log.role
+
+type: keyword
+
+The role of the Redis instance. Can be one of `master`, `slave`, `child` (for RDF/AOF writing child), or `sentinel`.
+
+
+[float]
+=== redis.log.level
+
+type: keyword
+
+The log level. Can be one of `debug`, `verbose`, `notice`, or `warning`.
+
+
+[float]
+=== redis.log.message
+
+type: text
+
+The log message
+
+
+[float]
+== slowlog Fields
+
+Slow logs are retrieved from Redis via a network connection.
+
+
+
+[float]
+=== redis.slowlog.cmd
+
+type: keyword
+
+The command executed.
+
+
+[float]
+=== redis.slowlog.duration.us
+
+type: long
+
+How long it took to execute the command in microseconds.
+
+
+[float]
+=== redis.slowlog.id
+
+type: long
+
+The ID of the query.
+
+
+[float]
+=== redis.slowlog.key
+
+type: keyword
+
+The key on which the command was executed.
+
+
+[float]
+=== redis.slowlog.args
+
+type: keyword
+
+The arguments with which the command was called.
+
+
 [[exported-fields-system]]
 == System Fields
 

filebeat/docs/modules/redis.asciidoc

@@ -0,0 +1,67 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[filebeat-module-redis]]
+== Redis module
+
+experimental[]
+
+This module has two filesets:
+
+* The `log` fileset collects and parses the logs that Redis writes to disk.  The `slowlog` fileset
+* connects to Redis via the network and retrieves the slow logs
+  by using the `SLOWLOG` command.
+
+For the `log` fileset, make sure the `logfile` option is set in the Redis configuration file. For
+the `slowlog` fileset, make sure the `slowlog-log-slower-than` is set.
+
+[float]
+=== Compatibility
+
+The Redis `log` fileset was tested with logs from Redis versions 1.2.6, 2.4.6, and 3.0.2, so we
+expect compatibility with any version 1.x, 2.x, or 3.x.
+
+On Windows, the default paths assume that Redis was installed from the Chocolatey repository.
+
+The Redis `slowlog` fileset was tested with Redis 3.0.2 and 2.4.6. We expect compatibility with any
+Redis version newer than 2.2.12, when the SLOWLOG command was added.
+
+[float]
+=== Dashboard
+
+This module comes with a sample dashboard.
+
+image::./images/kibana-redis.png[]
+
+[float]
+=== Log fileset settings
+
+[float]
+==== var.paths
+
+An array of paths where to look for the log files. If left empty, Filebeat
+will choose the paths depending on your operating systems.
+
+[float]
+=== Slowlog fileset settings
+
+[float]
+==== var.hosts
+
+An array of hosts to which Filebeat should connect to retrieve the slow logs. If left empty,
+`localhost:6379` is assumed.
+
+[float]
+==== var.password
+
+The password to use to connect to Redis, in case Redis authentication is enabled (the `requirepass`
+option in the Redis configuration).
+
+
+[float]
+=== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-redis,exported fields>> section.
+

filebeat/docs/modules_list.asciidoc

@@ -8,6 +8,7 @@ This file is generated! See scripts/docs_collector.py
   * <<filebeat-module-icinga>>
   * <<filebeat-module-mysql>>
   * <<filebeat-module-nginx>>
+  * <<filebeat-module-redis>>
   * <<filebeat-module-system>>
 
 
@@ -19,4 +20,5 @@ include::modules/auditd.asciidoc[]
 include::modules/icinga.asciidoc[]
 include::modules/mysql.asciidoc[]
 include::modules/nginx.asciidoc[]
+include::modules/redis.asciidoc[]
 include::modules/system.asciidoc[]

filebeat/filebeat.full.yml

@@ -381,7 +381,7 @@ filebeat.prospectors:
 
 #------------------------- Redis slowlog prospector ---------------------------
 # Experimental: Config options for the redis slow log prospector
-#- input_type: redis
+#- type: redis
   #hosts: ["localhost:6379"]
   #username:
   #password:

filebeat/module/redis/_meta/docs.asciidoc

@@ -0,0 +1,54 @@
+== Redis module
+
+experimental[]
+
+This module has two filesets:
+
+* The `log` fileset collects and parses the logs that Redis writes to disk.  The `slowlog` fileset
+* connects to Redis via the network and retrieves the slow logs
+  by using the `SLOWLOG` command.
+
+For the `log` fileset, make sure the `logfile` option is set in the Redis configuration file. For
+the `slowlog` fileset, make sure the `slowlog-log-slower-than` is set.
+
+[float]
+=== Compatibility
+
+The Redis `log` fileset was tested with logs from Redis versions 1.2.6, 2.4.6, and 3.0.2, so we
+expect compatibility with any version 1.x, 2.x, or 3.x.
+
+On Windows, the default paths assume that Redis was installed from the Chocolatey repository.
+
+The Redis `slowlog` fileset was tested with Redis 3.0.2 and 2.4.6. We expect compatibility with any
+Redis version newer than 2.2.12, when the SLOWLOG command was added.
+
+[float]
+=== Dashboard
+
+This module comes with a sample dashboard.
+
+image::./images/kibana-redis.png[]
+
+[float]
+=== Log fileset settings
+
+[float]
+==== var.paths
+
+An array of paths where to look for the log files. If left empty, Filebeat
+will choose the paths depending on your operating systems.
+
+[float]
+=== Slowlog fileset settings
+
+[float]
+==== var.hosts
+
+An array of hosts to which Filebeat should connect to retrieve the slow logs. If left empty,
+`localhost:6379` is assumed.
+
+[float]
+==== var.password
+
+The password to use to connect to Redis, in case Redis authentication is enabled (the `requirepass`
+option in the Redis configuration).

filebeat/module/redis/_meta/fields.yml

@@ -0,0 +1,9 @@
+- key: redis
+  title: "redis"
+  description: >
+    redis Module
+  fields:
+    - name: redis
+      type: group
+      description: >
+      fields:

filebeat/module/redis/_meta/kibana/dashboard/7fea2930-478e-11e7-b1f0-cb29bac6bf8b.json

@@ -0,0 +1,13 @@
+{
+  "hits": 0, 
+  "timeRestore": false, 
+  "description": "", 
+  "title": "Filebeat Redis", 
+  "uiStateJSON": "{\"P-5\":{\"vis\":{\"legendOpen\":false}}}", 
+  "panelsJSON": "[{\"col\":1,\"id\":\"78b9afe0-478f-11e7-b1f0-cb29bac6bf8b\",\"panelIndex\":2,\"row\":5,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"col\":4,\"id\":\"d2864600-478f-11e7-be88-2ddb32f3df97\",\"panelIndex\":3,\"row\":5,\"size_x\":9,\"size_y\":3,\"type\":\"visualization\"},{\"size_x\":12,\"size_y\":4,\"panelIndex\":4,\"type\":\"search\",\"id\":\"73613570-4791-11e7-be88-2ddb32f3df97\",\"col\":1,\"row\":8,\"columns\":[\"beat.name\",\"redis.log.level\",\"redis.log.role\",\"redis.log.message\"],\"sort\":[\"@timestamp\",\"desc\"]},{\"size_x\":6,\"size_y\":4,\"panelIndex\":5,\"type\":\"visualization\",\"id\":\"dcccaa80-4791-11e7-be88-2ddb32f3df97\",\"col\":7,\"row\":1},{\"size_x\":6,\"size_y\":4,\"panelIndex\":6,\"type\":\"search\",\"id\":\"0ab87b80-478e-11e7-b1f0-cb29bac6bf8b\",\"col\":1,\"row\":1,\"columns\":[\"beat.name\",\"message\",\"redis.slowlog.duration.us\",\"redis.slowlog.key\"],\"sort\":[\"@timestamp\",\"desc\"]}]", 
+  "optionsJSON": "{\"darkTheme\":false}", 
+  "version": 1, 
+  "kibanaSavedObjectMeta": {
+    "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}],\"highlightAll\":true,\"version\":true}"
+  }
+}

filebeat/module/redis/_meta/kibana/search/0ab87b80-478e-11e7-b1f0-cb29bac6bf8b.json

@@ -0,0 +1,19 @@
+{
+  "sort": [
+    "@timestamp", 
+    "desc"
+  ], 
+  "hits": 0, 
+  "description": "", 
+  "title": "Filebeat Redis slowlog", 
+  "version": 1, 
+  "kibanaSavedObjectMeta": {
+    "searchSourceJSON": "{\"index\":\"filebeat-*\",\"highlightAll\":true,\"version\":true,\"query\":{\"query_string\":{\"query\":\"_exists_:redis.slowlog\",\"analyze_wildcard\":true}},\"filter\":[]}"
+  }, 
+  "columns": [
+    "beat.name", 
+    "message", 
+    "redis.slowlog.duration.us", 
+    "redis.slowlog.key"
+  ]
+}

filebeat/module/redis/_meta/kibana/search/73613570-4791-11e7-be88-2ddb32f3df97.json

@@ -0,0 +1,19 @@
+{
+  "sort": [
+    "@timestamp", 
+    "desc"
+  ], 
+  "hits": 0, 
+  "description": "", 
+  "title": "Filebeat Redis logs", 
+  "version": 1, 
+  "kibanaSavedObjectMeta": {
+    "searchSourceJSON": "{\"index\":\"filebeat-*\",\"highlightAll\":true,\"version\":true,\"query\":{\"query_string\":{\"query\":\"_exists_:redis.log\",\"analyze_wildcard\":true}},\"filter\":[]}"
+  }, 
+  "columns": [
+    "beat.name", 
+    "redis.log.level", 
+    "redis.log.role", 
+    "redis.log.message"
+  ]
+}

filebeat/module/redis/_meta/kibana/visualization/78b9afe0-478f-11e7-b1f0-cb29bac6bf8b.json

@@ -0,0 +1,10 @@
+{
+  "visState": "{\"title\":\"Redis log levels and roles\",\"type\":\"pie\",\"params\":{\"addLegend\":true,\"addTooltip\":true,\"isDonut\":false,\"legendPosition\":\"bottom\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"redis.log.role\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"redis.log.level\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Log level\"}}],\"listeners\":{}}", 
+  "description": "", 
+  "title": "Redis log levels and roles", 
+  "uiStateJSON": "{}", 
+  "version": 1, 
+  "kibanaSavedObjectMeta": {
+    "searchSourceJSON": "{\"filter\":[],\"index\":\"filebeat-*\",\"highlightAll\":true,\"version\":true,\"query\":{\"query_string\":{\"query\":\"_exists_:redis.log\",\"analyze_wildcard\":true}}}"
+  }
+}

filebeat/module/redis/_meta/kibana/visualization/d2864600-478f-11e7-be88-2ddb32f3df97.json

@@ -0,0 +1,10 @@
+{
+  "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{},\"schema\":\"metric\",\"type\":\"count\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"customInterval\":\"2h\",\"extended_bounds\":{},\"field\":\"@timestamp\",\"interval\":\"auto\",\"min_doc_count\":1},\"schema\":\"segment\",\"type\":\"date_histogram\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"field\":\"redis.log.level\",\"order\":\"desc\",\"orderBy\":\"1\",\"size\":5},\"schema\":\"group\",\"type\":\"terms\"}],\"listeners\":{},\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"@timestamp per week\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"interpolate\":\"linear\",\"legendPosition\":\"right\",\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"mode\":\"stacked\",\"show\":\"true\",\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"value\"}]},\"title\":\"Redis logs over time\",\"type\":\"histogram\"}", 
+  "description": "", 
+  "title": "Redis logs over time", 
+  "uiStateJSON": "{\"vis\":{\"colors\":{\"notice\":\"#629E51\",\"warning\":\"#EF843C\"}}}", 
+  "version": 1, 
+  "kibanaSavedObjectMeta": {
+    "searchSourceJSON": "{\"filter\":[],\"index\":\"filebeat-*\",\"highlightAll\":true,\"version\":true,\"query\":{\"query_string\":{\"query\":\"_exists_:redis.log\",\"analyze_wildcard\":true}}}"
+  }
+}

filebeat/module/redis/_meta/kibana/visualization/dcccaa80-4791-11e7-be88-2ddb32f3df97.json

@@ -0,0 +1,11 @@
+{
+  "visState": "{\"title\":\"Redis slowest commands\",\"type\":\"histogram\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":200},\"position\":\"left\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Duration (microseconds)\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"interpolate\":\"linear\",\"legendPosition\":\"right\",\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Command\"},\"drawLinesBetweenPoints\":true,\"mode\":\"normal\",\"show\":true,\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":true,\"rotate\":75,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"bottom\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Command\"},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"redis.slowlog.duration.us\",\"customLabel\":\"Command\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"redis.slowlog.cmd\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Duration (microseconds)\"}}],\"listeners\":{}}", 
+  "description": "", 
+  "title": "Redis slowest commands", 
+  "uiStateJSON": "{}", 
+  "version": 1, 
+  "savedSearchId": "0ab87b80-478e-11e7-b1f0-cb29bac6bf8b", 
+  "kibanaSavedObjectMeta": {
+    "searchSourceJSON": "{\"filter\":[]}"
+  }
+}