Filebeat MySQL module (#3171)

* Contains slowlog and errors filesets * Test files for two mysql versions (5.5 and 5.7) * Add support for built-in variables (e.g. `builtin.hostname`) * Contains a sample Kibana dashboard Part of #3159.
elastic · Dec 13, 2016 · 227ea14 · 227ea14
1 parent 71f3939
commit 227ea14
Show file tree

Hide file tree

Showing 29 changed files with 1,119 additions and 4 deletions.
diff --git a/filebeat/_meta/fields.common.yml b/filebeat/_meta/fields.common.yml
@@ -33,3 +33,8 @@
       description: >
         The input type from which the event was generated. This field is set to the value specified for the `input_type` option in the prospector section of the Filebeat config file.
 
+    - name: error
+      description: >
+        Ingestion pipeline error message, added in case there are errors reported by
+        the Ingest Node in Elasticsearch.
+
diff --git a/filebeat/_meta/kibana/dashboard/Filebeat-MySQL-Dashboard.json b/filebeat/_meta/kibana/dashboard/Filebeat-MySQL-Dashboard.json
@@ -0,0 +1,13 @@
+{
+  "hits": 0, 
+  "timeRestore": false, 
+  "description": "", 
+  "title": "Filebeat MySQL Dashboard", 
+  "uiStateJSON": "{\"P-1\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}", 
+  "panelsJSON": "[{\"col\":1,\"id\":\"MySQL-slowest-queries\",\"panelIndex\":1,\"row\":8,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"MySQL-Slow-queries-over-time\",\"panelIndex\":2,\"row\":1,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"MySQL-error-logs\",\"panelIndex\":3,\"row\":1,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"columns\":[\"mysql.error.level\",\"mysql.error.message\"],\"id\":\"Filebeat-MySQL-error-log\",\"panelIndex\":4,\"row\":8,\"size_x\":6,\"size_y\":5,\"sort\":[\"@timestamp\",\"desc\"],\"type\":\"search\"},{\"col\":7,\"id\":\"MySQL-Error-logs-levels\",\"panelIndex\":5,\"row\":4,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"MySQL-Slow-logs-by-count\",\"panelIndex\":6,\"row\":4,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"}]", 
+  "optionsJSON": "{\"darkTheme\":false}", 
+  "version": 1, 
+  "kibanaSavedObjectMeta": {
+    "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}"
+  }
+}
diff --git a/filebeat/_meta/kibana/search/Filebeat-MySQL-Slow-log.json b/filebeat/_meta/kibana/search/Filebeat-MySQL-Slow-log.json
@@ -0,0 +1,16 @@
+{
+  "sort": [
+    "@timestamp", 
+    "desc"
+  ], 
+  "hits": 0, 
+  "description": "", 
+  "title": "Filebeat MySQL Slow log", 
+  "version": 1, 
+  "kibanaSavedObjectMeta": {
+    "searchSourceJSON": "{\"index\":\"filebeat-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"_exists_:mysql.slowlog\"}},\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647}}"
+  }, 
+  "columns": [
+    "_source"
+  ]
+}
diff --git a/filebeat/_meta/kibana/search/Filebeat-MySQL-error-log.json b/filebeat/_meta/kibana/search/Filebeat-MySQL-error-log.json
@@ -0,0 +1,17 @@
+{
+  "sort": [
+    "@timestamp", 
+    "desc"
+  ], 
+  "hits": 0, 
+  "description": "", 
+  "title": "Filebeat MySQL error log", 
+  "version": 1, 
+  "kibanaSavedObjectMeta": {
+    "searchSourceJSON": "{\"index\":\"filebeat-*\",\"query\":{\"query_string\":{\"query\":\"_exists_:mysql.error\",\"analyze_wildcard\":true}},\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647}}"
+  }, 
+  "columns": [
+    "mysql.error.level", 
+    "mysql.error.message"
+  ]
+}
diff --git a/filebeat/_meta/kibana/visualization/MySQL-Error-logs-levels.json b/filebeat/_meta/kibana/visualization/MySQL-Error-logs-levels.json
@@ -0,0 +1,11 @@
+{
+  "visState": "{\"title\":\"MySQL Error logs levels\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"mysql.error.level\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", 
+  "description": "", 
+  "title": "MySQL Error logs levels", 
+  "uiStateJSON": "{\"vis\":{\"colors\":{\"Note\":\"#9AC48A\",\"Warning\":\"#F9934E\",\"ERROR\":\"#E24D42\"}}}", 
+  "version": 1, 
+  "savedSearchId": "Filebeat-MySQL-error-log", 
+  "kibanaSavedObjectMeta": {
+    "searchSourceJSON": "{\"filter\":[]}"
+  }
+}
diff --git a/filebeat/_meta/kibana/visualization/MySQL-Slow-logs-by-count.json b/filebeat/_meta/kibana/visualization/MySQL-Slow-logs-by-count.json
@@ -0,0 +1,11 @@
+{
+  "visState": "{\"title\":\"MySQL Slow logs by count\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"mysql.slowlog.query\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", 
+  "description": "", 
+  "title": "MySQL Slow logs by count", 
+  "uiStateJSON": "{}", 
+  "version": 1, 
+  "savedSearchId": "Filebeat-MySQL-Slow-log", 
+  "kibanaSavedObjectMeta": {
+    "searchSourceJSON": "{\"filter\":[]}"
+  }
+}
diff --git a/filebeat/_meta/kibana/visualization/MySQL-Slow-queries-over-time.json b/filebeat/_meta/kibana/visualization/MySQL-Slow-queries-over-time.json
@@ -0,0 +1,11 @@
+{
+  "visState": "{\"title\":\"MySQL Slow queries over time\",\"type\":\"histogram\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"scale\":\"linear\",\"mode\":\"stacked\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Slow queries\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}}],\"listeners\":{}}", 
+  "description": "", 
+  "title": "MySQL Slow queries over time", 
+  "uiStateJSON": "{\"vis\":{\"colors\":{\"Slow queries\":\"#EF843C\"}}}", 
+  "version": 1, 
+  "savedSearchId": "Filebeat-MySQL-Slow-log", 
+  "kibanaSavedObjectMeta": {
+    "searchSourceJSON": "{\"filter\":[]}"
+  }
+}
diff --git a/filebeat/_meta/kibana/visualization/MySQL-error-logs.json b/filebeat/_meta/kibana/visualization/MySQL-error-logs.json
@@ -0,0 +1,11 @@
+{
+  "visState": "{\"title\":\"MySQL error logs\",\"type\":\"histogram\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"scale\":\"linear\",\"mode\":\"stacked\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Error logs\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}}],\"listeners\":{}}", 
+  "description": "", 
+  "title": "MySQL error logs", 
+  "uiStateJSON": "{\"vis\":{\"colors\":{\"Count\":\"#447EBC\",\"Error logs\":\"#1F78C1\"}}}", 
+  "version": 1, 
+  "savedSearchId": "Filebeat-MySQL-error-log", 
+  "kibanaSavedObjectMeta": {
+    "searchSourceJSON": "{\"filter\":[]}"
+  }
+}
diff --git a/filebeat/_meta/kibana/visualization/MySQL-slowest-queries.json b/filebeat/_meta/kibana/visualization/MySQL-slowest-queries.json
@@ -0,0 +1,11 @@
+{
+  "visState": "{\"title\":\"MySQL slowest queries\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"mysql.slowlog.query_time.sec\",\"customLabel\":\"Query time\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"mysql.slowlog.query\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Query\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"mysql.slowlog.user\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"User\"}}],\"listeners\":{}}", 
+  "description": "", 
+  "title": "MySQL slowest queries", 
+  "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", 
+  "version": 1, 
+  "savedSearchId": "Filebeat-MySQL-Slow-log", 
+  "kibanaSavedObjectMeta": {
+    "searchSourceJSON": "{\"filter\":[]}"
+  }
+}
diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -15,6 +15,7 @@ grouped in the following categories:
 * <<exported-fields-beat>>
 * <<exported-fields-cloud>>
 * <<exported-fields-log>>
+* <<exported-fields-mysql>>
 * <<exported-fields-nginx>>
 
 --
@@ -175,6 +176,141 @@ required: True
 The input type from which the event was generated. This field is set to the value specified for the `input_type` option in the prospector section of the Filebeat config file.
 
 
+[float]
+=== error
+
+Ingestion pipeline error message, added in case there are errors reported by the Ingest Node in Elasticsearch.
+
+
+[[exported-fields-mysql]]
+== MySQL Fields
+
+Module for parsing the MySQL log files.
+
+
+
+[float]
+== mysql Fields
+
+Fields from the MySQL log files.
+
+
+
+[float]
+== error Fields
+
+Contains fields from the MySQL error logs.
+
+
+
+[float]
+=== mysql.error.timestamp
+
+The timestamp from the log line.
+
+
+[float]
+=== mysql.error.thread_id
+
+type: long
+
+As of MySQL 5.7.2, this is the thread id. For MySQL versions prior to 5.7.2, this field contains the process id.
+
+
+[float]
+=== mysql.error.level
+
+example: Warning
+
+The log level.
+
+[float]
+=== mysql.error.message
+
+type: text
+
+The logged message.
+
+
+[float]
+== slowlog Fields
+
+Contains fields from the MySQL slow logs.
+
+
+
+[float]
+=== mysql.slowlog.user
+
+The MySQL user that created the query.
+
+
+[float]
+=== mysql.slowlog.host
+
+The host from where the user that created the query logged in.
+
+
+[float]
+=== mysql.slowlog.ip
+
+The IP address from where the user that created the query logged in.
+
+
+[float]
+=== mysql.slowlog.query_time.sec
+
+type: float
+
+The total time the query took, in seconds, as a floating point number.
+
+
+[float]
+=== mysql.slowlog.lock_time.sec
+
+type: float
+
+The amount of time the query waited for the lock to be available. The value is in seconds, as a floating point number.
+
+
+[float]
+=== mysql.slowlog.rows_sent
+
+type: long
+
+The number of rows returned by the query.
+
+
+[float]
+=== mysql.slowlog.rows_examined
+
+type: long
+
+The number of rows scanned by the query.
+
+
+[float]
+=== mysql.slowlog.timestamp
+
+type: long
+
+The unix timestamp taken from the `SET timestamp` query.
+
+
+[float]
+=== mysql.slowlog.query
+
+The slow query.
+
+
+[float]
+=== mysql.slowlog.id
+
+type: long
+
+The connection ID for the query.
+
+
 [[exported-fields-nginx]]
 == Nginx Fields
 

diff --git a/filebeat/filebeat.py b/filebeat/filebeat.py
@@ -6,6 +6,7 @@
 import requests
 import tempfile
 import subprocess
+import socket
 from jinja2 import Template
 
 
@@ -37,7 +38,7 @@ def main():
 
 def load_dashboards(args):
     cmd = ["../libbeat/dashboards/import_dashboards",
-           "-dir", "etc/kibana",
+           "-dir", "_meta/kibana",
            "-es", args.es]
     subprocess.Popen(cmd).wait()
 
@@ -77,7 +78,9 @@ def load_fileset(args, module, fileset, path):
 
 
 def evaluate_vars(args, var_in):
-    var = {}
+    var = {
+        "builtin": get_builtin_vars()
+    }
     for name, vals in var_in.items():
         var[name] = vals["default"]
 
@@ -86,6 +89,8 @@ def evaluate_vars(args, var_in):
         elif sys.platform == "windows" and "os.windows" in vals:
             var[name] = vals["os.windows"]
 
+        var[name] = Template(var[name]).render(var)
+
     # overrides
     if args.E is not None:
         for pair in args.E:
@@ -95,6 +100,16 @@ def evaluate_vars(args, var_in):
     return var
 
 
+def get_builtin_vars():
+    host = socket.gethostname()
+    hostname, _, domain = host.partition(".")
+    # separate the domain
+    return {
+        "hostname": hostname,
+        "domain": domain
+    }
+
+
 def load_pipeline(var, pipeline):
     path = os.path.join(var["beat"]["path"], Template(pipeline).render(var))
     print("Loading ingest pipeline: {}".format(path))
@@ -110,7 +125,8 @@ def load_pipeline(var, pipeline):
                              var["beat"]["pipeline_id"]),
                      data=contents)
     if r.status_code >= 300:
-        print("Error posting template: {}".format(r.text))
+        print("Error posting pipeline: {}".format(r.text))
+        sys.exit(1)
 
 
 def run_filebeat(args, prospectors):