[Filebeat] MSSQL module (#12079)

CHANGELOG.next.asciidoc

@@ -206,6 +206,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add new `container` input. {pull}12162[12162]
 - `container` and `docker` inputs now support reading of labels and env vars written by docker JSON file logging driver. {issue}8358[8358]
 - Add specific date processor to convert timezones so same pipeline can be used when convert_timezone is enabled or disabled. {pull}12253[12253]
+- Add MSSQL module {pull}12079[12079]
 
 *Heartbeat*
 

filebeat/docs/fields.asciidoc

@@ -34,6 +34,7 @@ grouped in the following categories:
 * <<exported-fields-log>>
 * <<exported-fields-logstash>>
 * <<exported-fields-mongodb>>
+* <<exported-fields-mssql>>
 * <<exported-fields-mysql>>
 * <<exported-fields-nats>>
 * <<exported-fields-netflow>>
@@ -7668,6 +7669,33 @@ alias to: message
 
 --
 
+[[exported-fields-mssql]]
+== mssql fields
+
+MS SQL Filebeat Module
+
+
+[float]
+== mssql fields
+
+Fields from the MSSQL log files
+
+
+[float]
+== log fields
+
+Common log fields
+
+
+*`mssql.log.origin`*::
++
+--
+type: keyword
+
+Origin of the message, usually the server but it can also be a recovery process
+
+--
+
 [[exported-fields-mysql]]
 == MySQL fields
 

filebeat/docs/modules/mssql.asciidoc

@@ -0,0 +1,63 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[filebeat-module-mssql]]
+:modulename: mssql
+:has-dashboards: false
+
+== MSSQL module
+
+The +{modulename}+ module parses error logs created by MSSQL.
+
+include::../include/what-happens.asciidoc[]
+
+[float]
+=== Compatibility
+
+include::../include/running-modules.asciidoc[]
+
+include::../include/configuring-intro.asciidoc[]
+
+The following example shows how to set paths in the +modules.d/{modulename}.yml+
+file to override the default paths for Træfik logs:
+
+["source","yaml",subs="attributes"]
+-----
+- module: mssql
+  access:
+    enabled: true
+    var.paths: ["/var/opt/mssql/log/error*"]
+-----
+
+
+To specify the same settings at the command line, you use:
+
+["source","sh",subs="attributes"]
+-----
+-M "mssql.access.var.paths=[/var/opt/mssql/log/error*]"
+-----
+
+//set the fileset name used in the included example
+:fileset_ex: log
+
+include::../include/config-option-intro.asciidoc[]
+
+[float]
+==== `log` fileset settings
+
+include::../include/var-paths.asciidoc[]
+
+:has-dashboards!:
+
+:fileset_ex!:
+
+:modulename!:
+
+
+[float]
+=== Fields
+
+For a description of each field in the module, see the
+<<exported-fields-mssql,exported fields>> section.
+

filebeat/docs/modules_list.asciidoc

@@ -17,6 +17,7 @@ This file is generated! See scripts/docs_collector.py
   * <<filebeat-module-kibana>>
   * <<filebeat-module-logstash>>
   * <<filebeat-module-mongodb>>
+  * <<filebeat-module-mssql>>
   * <<filebeat-module-mysql>>
   * <<filebeat-module-nats>>
   * <<filebeat-module-netflow>>
@@ -50,6 +51,7 @@ include::modules/kafka.asciidoc[]
 include::modules/kibana.asciidoc[]
 include::modules/logstash.asciidoc[]
 include::modules/mongodb.asciidoc[]
+include::modules/mssql.asciidoc[]
 include::modules/mysql.asciidoc[]
 include::modules/nats.asciidoc[]
 include::modules/netflow.asciidoc[]

x-pack/filebeat/filebeat.reference.yml

@@ -313,6 +313,16 @@ filebeat.modules:
     # can be added under this section.
     #input:
 
+#-------------------------------- Mssql Module --------------------------------
+- module: mssql
+  # Fileset for native deployment
+  log:
+    enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
 #-------------------------------- MySQL Module --------------------------------
 #- module: mysql
   # Error logs

x-pack/filebeat/module/mssql/_meta/config.yml

@@ -0,0 +1,8 @@
+- module: mssql
+  # Fileset for native deployment
+  log:
+    enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:

x-pack/filebeat/module/mssql/_meta/docs.asciidoc

@@ -0,0 +1,50 @@
+:modulename: mssql
+:has-dashboards: false
+
+== MSSQL module
+
+The +{modulename}+ module parses error logs created by MSSQL.
+
+include::../include/what-happens.asciidoc[]
+
+[float]
+=== Compatibility
+
+include::../include/running-modules.asciidoc[]
+
+include::../include/configuring-intro.asciidoc[]
+
+The following example shows how to set paths in the +modules.d/{modulename}.yml+
+file to override the default paths for Træfik logs:
+
+["source","yaml",subs="attributes"]
+-----
+- module: mssql
+  access:
+    enabled: true
+    var.paths: ["/var/opt/mssql/log/error*"]
+-----
+
+
+To specify the same settings at the command line, you use:
+
+["source","sh",subs="attributes"]
+-----
+-M "mssql.access.var.paths=[/var/opt/mssql/log/error*]"
+-----
+
+//set the fileset name used in the included example
+:fileset_ex: log
+
+include::../include/config-option-intro.asciidoc[]
+
+[float]
+==== `log` fileset settings
+
+include::../include/var-paths.asciidoc[]
+
+:has-dashboards!:
+
+:fileset_ex!:
+
+:modulename!:

x-pack/filebeat/module/mssql/_meta/fields.yml

@@ -0,0 +1,8 @@
+- key: mssql
+  title: "mssql"
+  description: MS SQL Filebeat Module
+  fields:
+    - name: mssql
+      type: group
+      description: Fields from the MSSQL log files
+      fields:

x-pack/filebeat/module/mssql/log/_meta/fields.yml

@@ -0,0 +1,7 @@
+- name: log
+  description: Common log fields
+  type: group
+  fields:
+    - name: origin
+      description: Origin of the message, usually the server but it can also be a recovery process
+      type: keyword

x-pack/filebeat/module/mssql/log/config/config.yml

@@ -0,0 +1,15 @@
+type: log
+paths:
+  {{ range $i, $path := .paths }}
+- {{$path}}
+  {{ end }}
+exclude_files: [".gz$"]
+
+multiline.pattern: '^\d\d'
+multiline.negate: true
+multiline.match: after
+
+{{ if .convert_timezone }}
+processors:
+- add_locale: ~
+{{ end }}

x-pack/filebeat/module/mssql/log/ingest/pipeline.json

@@ -0,0 +1,58 @@
+{
+    "description": "Pipeline to parse MSSQL logs",
+    "processors": [
+        {
+            "grok": {
+                "field": "message",
+                "patterns": ["%{MSSQL_DATE:date} %{DATA:mssql.log.origin} [ ]*%{GREEDYDATA:msg_temp}"],
+                "pattern_definitions": {
+                    "MSSQL_DATE":"%{DATA} %{DATA}"
+                }
+            }
+        },
+        {
+            "date": {
+                "field": "date",
+                "target_field": "@timestamp",
+                "formats": ["yyyy-MM-dd HH:mm:ss.SS"],
+                "ignore_failure": true
+            }
+        },
+        {
+            "date": {
+                "if": "ctx.event.timezone != null",
+                "field": "@timestamp",
+                "formats": ["ISO8601"],
+                "timezone": "{{ event.timezone }}",
+                "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
+            }
+        },
+        {
+            "remove": {
+                "field":"date",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "message",
+                "target_field": "log.original"
+            }
+        },
+        {
+            "rename": {
+                "field": "msg_temp",
+                "target_field": "message",
+                "ignore_missing": true
+            }
+        }
+    ],
+    "on_failure": [
+        {
+            "set": {
+                "field": "error.message",
+                "value": "{{ _ingest.on_failure_message }}"
+            }
+        }
+    ]
+}

x-pack/filebeat/module/mssql/log/manifest.yml

@@ -0,0 +1,22 @@
+module_version: 1.0
+
+var:
+  - name: paths
+    default:
+      - /var/opt/mssql/log/error*
+    os.darwin:
+      - /var/opt/mssql/log/error*
+    os.windows:
+      - c:\ProgramFiles\Microsoft SQL Server\MSSQL.1MSSQL\LOG\ERRORLOG*
+    os.linux:
+      - /var/opt/mssql/log/error*
+  - name: convert_timezone
+    default: true
+  # if ES < 6.1.0, this flag switches to false automatically when evaluating the
+  # pipeline
+    min_elasticsearch_version:
+      version: 6.1.0
+      value: false
+
+ingest_pipeline: ingest/pipeline.json
+input: config/config.yml

x-pack/filebeat/module/mssql/log/test/test.log

@@ -0,0 +1,21 @@
+2019-05-03 09:01:09.99 Server      Microsoft SQL Server 2017 (RTM-CU13) (KB4466404) - 14.0.3048.4 (X64)
+	Nov 30 2018 12:57:58
+	Copyright (C) 2017 Microsoft Corporation
+	Developer Edition (64-bit) on Linux (Ubuntu 16.04.5 LTS)
+2019-05-03 09:01:09.99 Server      UTC adjustment: 0:00
+2019-05-03 09:01:09.99 Server      (c) Microsoft Corporation.
+2019-05-03 09:01:09.99 Server      All rights reserved.
+2019-05-03 09:01:10.00 Server      Server process ID is 4124.
+2019-05-03 09:01:10.00 Server      Logging SQL Server messages in file '/var/opt/mssql/log/errorlog'.
+2019-05-03 09:01:10.00 Server      Registry startup parameters:
+	 -d /var/opt/mssql/data/master.mdf
+	 -l /var/opt/mssql/data/mastlog.ldf
+	 -e /var/opt/mssql/log/errorlog
+2019-05-03 09:01:10.00 Server      SQL Server detected 1 sockets with 6 cores per socket and 12 logical processors per socket, 12 total logical processors; using 12 logical processors based on SQL Server licensing. This is an informational message; no user action is required.
+2019-05-03 09:01:10.00 Server      SQL Server is starting at normal priority base (=7). This is an informational message only. No user action is required.
+2019-05-03 09:01:10.00 Server      Detected 25445 MB of RAM. This is an informational message; no user action is required.
+2019-05-03 09:01:10.00 Server      Using conventional memory in the memory manager.
+2019-05-03 09:01:10.01 Server      Large Page Allocated: 32MB
+2019-05-03 09:01:10.20 Server      Buffer pool extension is already disabled. No action is n
+2019-05-03 09:01:11.93 spid22s     Service Broker manager has started.
+2019-05-03 09:01:12.03 spid6s      Recovery is complete. This is an informational message only. No user action is required.