Deprecation fileset for Elasticsearch filebeat module (#7474)

CHANGELOG.asciidoc

@@ -206,6 +206,7 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff]
 - Add GC fileset to the Elasticsearch module. {pull}7305[7305]
 - Add Audit log fileset to the Elasticsearch module. {pull}7365[7365]
 - Add Slow log fileset to the Elasticsearch module. {pull}7473[7473]
+- Add deprecation fileset to the Elasticsearch module. {pull}7474[7474]
 
 *Heartbeat*
 

filebeat/docs/fields.asciidoc

@@ -916,6 +916,11 @@ The body of the request, if enabled
 
 --
 
+[float]
+== deprecation fields
+
+
+
 [float]
 == gc fields
 

filebeat/filebeat.reference.yml

@@ -110,6 +110,12 @@ filebeat.modules:
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
+  deprecation:
+    enabled: true
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
 #------------------------------- Icinga Module -------------------------------
 #- module: icinga
   # Main logs

filebeat/module/elasticsearch/_meta/config.yml

@@ -24,3 +24,9 @@
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
+
+  deprecation:
+    enabled: true
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:

filebeat/module/elasticsearch/deprecation/_meta/fields.yml

@@ -0,0 +1,4 @@
+- name: deprecation
+  type: group
+  description: >
+  fields:

filebeat/module/elasticsearch/deprecation/config/log.yml

@@ -0,0 +1,15 @@
+type: log
+paths:
+{{ range $i, $path := .paths }}
+ - {{$path}}
+{{ end }}
+exclude_files: [".gz$","_slowlog.log$","_access.log$"]
+multiline:
+  pattern: '^\[[0-9]{4}-[0-9]{2}-[0-9]{2}'
+  negate: true
+  match: after
+
+fields:
+  service.name: "elasticsearch"
+
+fields_under_root: true

filebeat/module/elasticsearch/deprecation/ingest/pipeline.json

@@ -0,0 +1,36 @@
+{
+    "description": "Pipeline for parsing the Elasticsearch deprecation log file.",
+    "on_failure": [
+        {
+            "set": {
+                "field": "error.message",
+                "value": "{{ _ingest.on_failure_message }}"
+            }
+        }
+    ],
+    "processors": [
+        {
+            "rename": {
+                "field": "@timestamp",
+                "target_field": "event.created"
+            }
+        },
+        {
+            "grok": {
+                "field": "message",
+                "pattern_definitions": {
+                    "GREEDYMULTILINE": "(.|\n)*"
+                },
+                "patterns": [
+                    "\\[%{TIMESTAMP_ISO8601:timestamp}\\]\\[%{LOGLEVEL:log.level}%{SPACE}*\\]\\[%{DATA:elasticsearch.server.component}%{SPACE}*\\] %{GREEDYMULTILINE:message}"
+                ]
+            }
+        },
+        {
+            "rename": {
+                "field": "timestamp",
+                "target_field": "@timestamp"
+            }
+        }
+    ]
+}

filebeat/module/elasticsearch/deprecation/manifest.yml

@@ -0,0 +1,13 @@
+module_version: 1.0
+
+var:
+  - name: paths
+    default:
+      - /var/log/elasticsearch/*_deprecation.log
+    os.darwin:
+      - /usr/local/elasticsearch/*_deprecation.log
+    os.windows:
+      - c:/ProgramData/Elastic/Elasticsearch/logs/*_deprecation.log
+
+ingest_pipeline: ingest/pipeline.json
+prospector: config/log.yml

filebeat/module/elasticsearch/deprecation/test/elasticsearch_deprecation.log

@@ -0,0 +1,4 @@
+[2018-04-23T16:40:13,737][WARN ][o.e.d.a.a.i.t.p.PutIndexTemplateRequest] Deprecated field [template] used, replaced by [index_patterns]
+[2018-04-23T16:40:13,862][WARN ][o.e.d.a.a.i.t.p.PutIndexTemplateRequest] Deprecated field [template] used, replaced by [index_patterns]
+[2018-04-23T16:40:14,792][WARN ][o.e.d.a.a.i.t.p.PutIndexTemplateRequest] Deprecated field [template] used, replaced by [index_patterns]
+[2018-04-23T16:40:15,127][WARN ][o.e.d.a.a.i.t.p.PutIndexTemplateRequest] Deprecated field [template] used, replaced by [index_patterns]

filebeat/module/elasticsearch/deprecation/test/other_elasticsearch_deprecation.log

@@ -0,0 +1,16 @@
+[2017-11-30T13:38:16,911][WARN ][o.e.d.c.ParseField       ] Deprecated field [inline] used, expected [source] instead
+[2017-11-30T13:38:16,941][WARN ][o.e.d.c.ParseField       ] Deprecated field [inline] used, expected [source] instead
+[2017-11-30T13:39:28,986][WARN ][o.e.d.i.m.UidFieldMapper ] Fielddata access on the _uid field is deprecated, use _id instead
+[2017-11-30T13:39:36,339][WARN ][o.e.d.i.m.UidFieldMapper ] Fielddata access on the _uid field is deprecated, use _id instead
+[2017-11-30T13:40:49,540][WARN ][o.e.d.i.m.UidFieldMapper ] Fielddata access on the _uid field is deprecated, use _id instead
+[2017-11-30T14:08:37,413][WARN ][o.e.d.i.m.UidFieldMapper ] Fielddata access on the _uid field is deprecated, use _id instead
+[2017-11-30T14:08:37,413][WARN ][o.e.d.i.m.UidFieldMapper ] Fielddata access on the _uid field is deprecated, use _id instead
+[2017-11-30T14:08:46,006][WARN ][o.e.d.i.m.UidFieldMapper ] Fielddata access on the _uid field is deprecated, use _id instead
+[2017-11-30T14:08:46,006][WARN ][o.e.d.i.m.UidFieldMapper ] Fielddata access on the _uid field is deprecated, use _id instead
+[2017-12-01T14:05:54,017][WARN ][o.e.d.i.m.AllFieldMapper ] [_all] is deprecated in 6.0+ and will be removed in 7.0. As a replacement, you can use [copy_to] on mapping fields to create your own catch all field.
+[2017-12-01T14:05:54,019][WARN ][o.e.d.i.m.AllFieldMapper ] [_all] is deprecated in 6.0+ and will be removed in 7.0. As a replacement, you can use [copy_to] on mapping fields to create your own catch all field.
+[2017-12-01T14:06:52,059][WARN ][o.e.d.i.m.AllFieldMapper ] [_all] is deprecated in 6.0+ and will be removed in 7.0. As a replacement, you can use [copy_to] on mapping fields to create your own catch all field.
+[2017-12-01T14:46:10,428][WARN ][o.e.d.s.a.InternalOrder$Parser] Deprecated aggregation order key [_term] used, replaced by [_key]
+[2017-12-04T16:17:18,271][WARN ][o.e.d.a.a.i.t.p.PutIndexTemplateRequest] Deprecated field [template] used, replaced by [index_patterns]
+[2017-12-04T16:17:18,282][WARN ][o.e.d.i.m.MapperService  ] [_default_] mapping is deprecated since it is not useful anymore now that indexes cannot have more than one type
+[2017-12-04T16:20:43,248][WARN ][o.e.d.i.m.MapperService  ] [_default_] mapping is deprecated since it is not useful anymore now that indexes cannot have more than one type

filebeat/modules.d/elasticsearch.yml.disabled

@@ -24,3 +24,9 @@
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
+
+  deprecation:
+    enabled: true
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths: