[Filebeat] Update handling of elasticsearch server logs

filebeat/docs/fields.asciidoc

@@ -49735,6 +49735,35 @@ example: 0
 
 --
 
+*`elasticsearch.elastic_product_origin`*::
++
+--
+Used by Elastic stack to identify which component of the stack sent the request
+
+type: keyword
+
+example: kibana
+
+--
+
+*`elasticsearch.http.request.x_opaque_id`*::
++
+--
+Used by Elasticsearch to throttle and deduplicate deprecation warnings
+
+example: v7app
+
+--
+
+*`elasticsearch.event.category`*::
++
+--
+Category of the deprecation event
+
+example: compatible_api
+
+--
+
 
 *`elasticsearch.audit.layer`*::
 +

filebeat/module/elasticsearch/_meta/fields.yml

@@ -40,3 +40,13 @@
           description: "Id of the shard"
           example: "0"
           type: keyword
+        - name: elastic_product_origin
+          type: keyword
+          description: "Used by Elastic stack to identify which component of the stack sent the request"
+          example: "kibana"
+        - name: http.request.x_opaque_id
+          description: "Used by Elasticsearch to throttle and deduplicate deprecation warnings"
+          example: "v7app"
+        - name: event.category
+          description: "Category of the deprecation event"
+          example: "compatible_api"

filebeat/module/elasticsearch/audit/ingest/pipeline.yml

@@ -3,9 +3,9 @@ processors:
 - set:
     field: event.ingested
     value: '{{_ingest.timestamp}}'
-- rename:
-    field: '@timestamp'
-    target_field: event.created
+- set:
+    copy_from: "@timestamp"
+    field: event.created
 - grok:
     field: message
     patterns:

filebeat/module/elasticsearch/deprecation/ingest/pipeline-json-7.yml

@@ -0,0 +1,97 @@
+description: Pipeline for parsing the Elasticsearch deprecation log file in JSON format.
+on_failure:
+- set:
+    field: error.message
+    value: '{{ _ingest.on_failure_message }}'
+processors:
+- json:
+    field: message
+    target_field: elasticsearch.deprecation
+- drop:
+    if: '!["deprecation", "deprecation.elasticsearch"].contains(ctx.elasticsearch.deprecation.type)'
+- remove:
+    field: elasticsearch.deprecation.type
+- dot_expander:
+    field: service.name
+    path: elasticsearch.deprecation
+- rename:
+    field: elasticsearch.deprecation.service.name
+    target_field: service.name
+    ignore_missing: true
+- rename:
+    field: elasticsearch.deprecation.level
+    target_field: log.level
+    ignore_missing: true
+- dot_expander:
+    field: log.level
+    path: elasticsearch.deprecation
+- rename:
+    field: elasticsearch.deprecation.log.level
+    target_field: log.level
+    ignore_missing: true
+- dot_expander:
+    field: log.logger
+    path: elasticsearch.deprecation
+- rename:
+    field: elasticsearch.deprecation.log.logger
+    target_field: log.logger
+    ignore_missing: true
+- dot_expander:
+    field: process.thread.name
+    path: elasticsearch.deprecation
+- rename:
+    field: elasticsearch.deprecation.process.thread.name
+    target_field: process.thread.name
+    ignore_missing: true
+- rename:
+    field: elasticsearch.deprecation.component
+    target_field: elasticsearch.component
+    ignore_missing: true
+- dot_expander:
+    field: cluster.name
+    path: elasticsearch.deprecation
+- rename:
+    field: elasticsearch.deprecation.cluster.name
+    target_field: elasticsearch.cluster.name
+- dot_expander:
+    field: node.name
+    path: elasticsearch.deprecation
+- rename:
+    field: elasticsearch.deprecation.node.name
+    target_field: elasticsearch.node.name
+- dot_expander:
+    field: cluster.uuid
+    path: elasticsearch.deprecation
+- rename:
+    field: elasticsearch.deprecation.cluster.uuid
+    target_field: elasticsearch.cluster.uuid
+    ignore_missing: true
+- dot_expander:
+    field: node.id
+    path: elasticsearch.deprecation
+- rename:
+    field: elasticsearch.deprecation.node.id
+    target_field: elasticsearch.node.id
+    ignore_missing: true
+- remove:
+    field: message
+- rename:
+    field: elasticsearch.deprecation.message
+    target_field: message
+- date:
+    field: 'elasticsearch.deprecation.@timestamp'
+    formats:
+    - ISO8601
+    ignore_failure: true
+    if: 'ctx.elasticsearch?.deprecation["@timestamp"] != null'
+- date:
+    field: 'elasticsearch.deprecation.timestamp'
+    formats:
+    - ISO8601
+    ignore_failure: true
+    if: 'ctx.elasticsearch?.deprecation?.timestamp != null'
+- remove:
+    field:
+      - elasticsearch.deprecation.timestamp
+      - elasticsearch.deprecation.@timestamp
+    ignore_missing: true

filebeat/module/elasticsearch/deprecation/ingest/pipeline-json-8.yml

@@ -0,0 +1,15 @@
+description: Pipeline for parsing the Elasticsearch deprecation log file in JSON format.
+on_failure:
+- set:
+    field: error.message
+    value: '{{ _ingest.on_failure_message }}'
+processors:
+- json:
+    field: message
+    add_to_root: true
+- dot_expander:
+    field: '*'
+    override: true
+- set:
+    field: event.dataset
+    value: elasticsearch.deprecation

filebeat/module/elasticsearch/deprecation/ingest/pipeline-json.yml

@@ -4,94 +4,9 @@ on_failure:
     field: error.message
     value: '{{ _ingest.on_failure_message }}'
 processors:
-- json:
-    field: message
-    target_field: elasticsearch.deprecation
-- drop:
-    if: '!["deprecation", "deprecation.elasticsearch"].contains(ctx.elasticsearch.deprecation.type)'
-- remove:
-    field: elasticsearch.deprecation.type
-- dot_expander:
-    field: service.name
-    path: elasticsearch.deprecation
-- rename:
-    field: elasticsearch.deprecation.service.name
-    target_field: service.name
-    ignore_missing: true
-- rename:
-    field: elasticsearch.deprecation.level
-    target_field: log.level
-    ignore_missing: true
-- dot_expander:
-    field: log.level
-    path: elasticsearch.deprecation
-- rename:
-    field: elasticsearch.deprecation.log.level
-    target_field: log.level
-    ignore_missing: true
-- dot_expander:
-    field: log.logger
-    path: elasticsearch.deprecation
-- rename:
-    field: elasticsearch.deprecation.log.logger
-    target_field: log.logger
-    ignore_missing: true
-- dot_expander:
-    field: process.thread.name
-    path: elasticsearch.deprecation
-- rename:
-    field: elasticsearch.deprecation.process.thread.name
-    target_field: process.thread.name
-    ignore_missing: true
-- rename:
-    field: elasticsearch.deprecation.component
-    target_field: elasticsearch.component
-    ignore_missing: true
-- dot_expander:
-    field: cluster.name
-    path: elasticsearch.deprecation
-- rename:
-    field: elasticsearch.deprecation.cluster.name
-    target_field: elasticsearch.cluster.name
-- dot_expander:
-    field: node.name
-    path: elasticsearch.deprecation
-- rename:
-    field: elasticsearch.deprecation.node.name
-    target_field: elasticsearch.node.name
-- dot_expander:
-    field: cluster.uuid
-    path: elasticsearch.deprecation
-- rename:
-    field: elasticsearch.deprecation.cluster.uuid
-    target_field: elasticsearch.cluster.uuid
-    ignore_missing: true
-- dot_expander:
-    field: node.id
-    path: elasticsearch.deprecation
-- rename:
-    field: elasticsearch.deprecation.node.id
-    target_field: elasticsearch.node.id
-    ignore_missing: true
-- remove:
-    field: message
-- rename:
-    field: elasticsearch.deprecation.message
-    target_field: message
-- date:
-    field: 'elasticsearch.deprecation.@timestamp'
-    formats:
-    - ISO8601
-    ignore_failure: true
-    if: 'ctx.elasticsearch?.deprecation["@timestamp"] != null'
-- date:
-    field: 'elasticsearch.deprecation.timestamp'
-    formats:
-    - ISO8601
-    ignore_failure: true
-    if: 'ctx.elasticsearch?.deprecation?.timestamp != null'
-- remove:
-    field:
-      - elasticsearch.deprecation.timestamp
-      - elasticsearch.deprecation.@timestamp
-    ignore_missing: true
+  - pipeline:
+      if: '!ctx.message.contains("ecs.version")'
+      name: '{< IngestPipeline "pipeline-json-7" >}'
+  - pipeline:
+      if: 'ctx.message.contains("ecs.version")'
+      name: '{< IngestPipeline "pipeline-json-8" >}'

filebeat/module/elasticsearch/deprecation/ingest/pipeline.yml

@@ -3,9 +3,9 @@ processors:
 - set:
     field: event.ingested
     value: '{{_ingest.timestamp}}'
-- rename:
-    field: '@timestamp'
-    target_field: event.created
+- set:
+    copy_from: "@timestamp"
+    field: event.created
 - grok:
     field: message
     patterns:

filebeat/module/elasticsearch/deprecation/manifest.yml

@@ -16,4 +16,6 @@ ingest_pipeline:
   - ingest/pipeline.yml
   - ingest/pipeline-plaintext.yml
   - ingest/pipeline-json.yml
+  - ingest/pipeline-json-7.yml
+  - ingest/pipeline-json-8.yml
 input: config/log.yml

filebeat/module/elasticsearch/deprecation/test/es_deprecation-json.800.log

@@ -1,15 +1,2 @@
-{"@timestamp":"2020-04-15T12:35:20.315Z", "log.level": "WARN",  "message":"Field parameter [precision] is deprecated and will be removed in a future version." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper","type":"deprecation","cluster.uuid":"a0P-i2H5R9-tJqwtF7BL0A","node.id":"FFMF7MVISuCWZMtxGmcGhg","node.name":"integTest-0","cluster.name":"integTest"}
-{"@timestamp":"2020-04-15T12:35:20.316Z", "log.level": "WARN",  "message":"Field parameter [tree] is deprecated and will be removed in a future version." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper","type":"deprecation","cluster.uuid":"a0P-i2H5R9-tJqwtF7BL0A","node.id":"FFMF7MVISuCWZMtxGmcGhg","node.name":"integTest-0","cluster.name":"integTest"}
-{"@timestamp":"2020-04-15T12:35:20.366Z", "log.level": "WARN",  "message":"Field parameter [precision] is deprecated and will be removed in a future version." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper","type":"deprecation","cluster.uuid":"a0P-i2H5R9-tJqwtF7BL0A","node.id":"FFMF7MVISuCWZMtxGmcGhg","node.name":"integTest-0","cluster.name":"integTest"}
-{"@timestamp":"2020-04-15T12:35:20.367Z", "log.level": "WARN",  "message":"Field parameter [strategy] is deprecated and will be removed in a future version." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper","type":"deprecation","cluster.uuid":"a0P-i2H5R9-tJqwtF7BL0A","node.id":"FFMF7MVISuCWZMtxGmcGhg","node.name":"integTest-0","cluster.name":"integTest"}
-{"@timestamp":"2020-04-15T12:35:20.479Z", "log.level": "WARN",  "message":"Field parameter [precision] is deprecated and will be removed in a future version." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][clusterApplierService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper","type":"deprecation","cluster.uuid":"a0P-i2H5R9-tJqwtF7BL0A","node.id":"FFMF7MVISuCWZMtxGmcGhg","node.name":"integTest-0","cluster.name":"integTest"}
-{"@timestamp":"2020-04-15T12:35:20.480Z", "log.level": "WARN",  "message":"Field parameter [strategy] is deprecated and will be removed in a future version." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][clusterApplierService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper","type":"deprecation","cluster.uuid":"a0P-i2H5R9-tJqwtF7BL0A","node.id":"FFMF7MVISuCWZMtxGmcGhg","node.name":"integTest-0","cluster.name":"integTest"}
-{"@timestamp":"2020-04-15T12:35:20.481Z", "log.level": "WARN",  "message":"Field parameter [precision] is deprecated and will be removed in a future version." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][clusterApplierService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper","type":"deprecation","cluster.uuid":"a0P-i2H5R9-tJqwtF7BL0A","node.id":"FFMF7MVISuCWZMtxGmcGhg","node.name":"integTest-0","cluster.name":"integTest"}
-{"@timestamp":"2020-04-15T12:35:20.487Z", "log.level": "WARN",  "message":"Field parameter [strategy] is deprecated and will be removed in a future version." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][clusterApplierService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper","type":"deprecation","cluster.uuid":"a0P-i2H5R9-tJqwtF7BL0A","node.id":"FFMF7MVISuCWZMtxGmcGhg","node.name":"integTest-0","cluster.name":"integTest"}
-{"@timestamp":"2020-04-16T13:46:33.582Z", "log.level": "WARN",  "message":"[PUT /_xpack/security/user/{username}/_password] is deprecated! Use [PUT /_security/user/{username}/_password] instead." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[n1][http_server_worker][T#3]","log.logger":"org.elasticsearch.deprecation.rest.RestController","type":"deprecation","cluster.uuid":"ZGYecRsDQPK_-ktRec3ZGQ","node.id":"Ni-9zbrZRm24wm7_zNtMTw","node.name":"n1","cluster.name":"es800"}
-{"@timestamp":"2020-04-16T13:46:34.219Z", "log.level": "WARN",  "message":"[PUT /_xpack/security/user/{username}/_password] is deprecated! Use [PUT /_security/user/{username}/_password] instead." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[n1][http_server_worker][T#4]","log.logger":"org.elasticsearch.deprecation.rest.RestController","type":"deprecation","cluster.uuid":"ZGYecRsDQPK_-ktRec3ZGQ","node.id":"Ni-9zbrZRm24wm7_zNtMTw","node.name":"n1","cluster.name":"es800"}
-{"@timestamp":"2020-04-16T13:46:34.339Z", "log.level": "WARN",  "message":"[PUT /_xpack/security/user/{username}/_password] is deprecated! Use [PUT /_security/user/{username}/_password] instead." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[n1][http_server_worker][T#5]","log.logger":"org.elasticsearch.deprecation.rest.RestController","type":"deprecation","cluster.uuid":"ZGYecRsDQPK_-ktRec3ZGQ","node.id":"Ni-9zbrZRm24wm7_zNtMTw","node.name":"n1","cluster.name":"es800"}
-{"@timestamp":"2020-04-16T13:46:34.455Z", "log.level": "WARN",  "message":"[PUT /_xpack/security/user/{username}/_password] is deprecated! Use [PUT /_security/user/{username}/_password] instead." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[n1][http_server_worker][T#6]","log.logger":"org.elasticsearch.deprecation.rest.RestController","type":"deprecation","cluster.uuid":"ZGYecRsDQPK_-ktRec3ZGQ","node.id":"Ni-9zbrZRm24wm7_zNtMTw","node.name":"n1","cluster.name":"es800"}
-{"@timestamp":"2020-04-16T13:47:36.309Z", "log.level": "WARN",  "message":"index name [.apm-custom-link] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" , "service.name":"ES_ECS","process.thread.name":"elasticsearch[n1][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.cluster.metadata.MetadataCreateIndexService","type":"deprecation","cluster.uuid":"ZGYecRsDQPK_-ktRec3ZGQ","node.id":"Ni-9zbrZRm24wm7_zNtMTw","node.name":"n1","cluster.name":"es800"}
-{"@timestamp":"2020-04-16T13:55:56.365Z", "log.level": "WARN",  "message":"index name [.monitoring-alerts-7] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" , "service.name":"ES_ECS","process.thread.name":"elasticsearch[n1][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.cluster.metadata.MetadataCreateIndexService","type":"deprecation","cluster.uuid":"ZGYecRsDQPK_-ktRec3ZGQ","node.id":"Ni-9zbrZRm24wm7_zNtMTw","node.name":"n1","cluster.name":"es800"}
-{"@timestamp":"2020-04-16T13:56:14.697Z", "log.level": "WARN",  "message":"[types removal] Using the _type field in queries and aggregations is deprecated, prefer to use a field instead." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[n1][search][T#7]","log.logger":"org.elasticsearch.deprecation.index.query.QueryShardContext","type":"deprecation","cluster.uuid":"ZGYecRsDQPK_-ktRec3ZGQ","node.id":"Ni-9zbrZRm24wm7_zNtMTw","node.name":"n1","cluster.name":"es800"}
+{"@timestamp":"2022-01-27T11:48:45.809Z", "log.level":"CRITICAL",  "data_stream.dataset":"deprecation.elasticsearch","data_stream.namespace":"default","data_stream.type":"logs","elasticsearch.elastic_product_origin":"","elasticsearch.event.category":"compatible_api","elasticsearch.http.request.x_opaque_id":"v7app","event.code":"create_index_with_types","message":"[types removal] Using include_type_name in create index requests is deprecated. The parameter will be removed in the next major version." , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"deprecation.elasticsearch","process.thread.name":"elasticsearch[runTask-0][transport_worker][T#8]","log.logger":"org.elasticsearch.deprecation.rest.action.admin.indices.RestCreateIndexAction","trace.id":"0af7651916cd43dd8448eb211c80319c","elasticsearch.cluster.uuid":"5alW33KLT16Lp1SevDqDSQ","elasticsearch.node.id":"tVLnAGLgQum5ca6z50aqbw","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"}
+{"@timestamp":"2022-01-27T11:52:39.882Z", "log.level":"CRITICAL",  "data_stream.dataset":"deprecation.elasticsearch","data_stream.namespace":"default","data_stream.type":"logs","elasticsearch.event.category":"compatible_api","event.code":"create_index_with_types","message":"[types removal] Using include_type_name in create index requests is deprecated. The parameter will be removed in the next major version." , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"deprecation.elasticsearch","process.thread.name":"elasticsearch[runTask-0][transport_worker][T#9]","log.logger":"org.elasticsearch.deprecation.rest.action.admin.indices.RestCreateIndexAction","elasticsearch.cluster.uuid":"5alW33KLT16Lp1SevDqDSQ","elasticsearch.node.id":"tVLnAGLgQum5ca6z50aqbw","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"}