Handle ECS-compatible deprecation logs emitted by ES 8.0.0+

CHANGELOG.next.asciidoc

@@ -256,6 +256,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Release Google Cloud module as GA. {pull}17511[17511]
 - Improve ECS categorization field mappings for nats module. {issue}16173[16173] {pull}17550[17550]
 - Enhance `elasticsearch/server` fileset to handle ECS-compatible logs emitted by Elasticsearch. {issue}17715[17715] {pull}17714[17714]
+- Enhance `elasticsearch/deprecation` fileset to handle ECS-compatible logs emitted by Elasticsearch. {issue}17715[17715] {pull}17728[17728]
 - Enhance `elasticsearch/slowlog` fileset to handle ECS-compatible logs emitted by Elasticsearch. {issue}17715[17715] {pull}17729[17729]
 
 *Heartbeat*

filebeat/module/elasticsearch/deprecation/ingest/pipeline-json.yml

@@ -11,12 +11,42 @@ processors:
     if: ctx.elasticsearch.deprecation.type != 'deprecation'
 - remove:
     field: elasticsearch.deprecation.type
+- dot_expander:
+    field: service.name
+    path: elasticsearch.deprecation
+- rename:
+    field: elasticsearch.deprecation.service.name
+    target_field: service.name
+    ignore_missing: true
 - rename:
     field: elasticsearch.deprecation.level
     target_field: log.level
+    ignore_missing: true
+- dot_expander:
+    field: log.level
+    path: elasticsearch.deprecation
+- rename:
+    field: elasticsearch.deprecation.log.level
+    target_field: log.level
+    ignore_missing: true
+- dot_expander:
+    field: log.logger
+    path: elasticsearch.deprecation
+- rename:
+    field: elasticsearch.deprecation.log.logger
+    target_field: log.logger
+    ignore_missing: true
+- dot_expander:
+    field: process.thread.name
+    path: elasticsearch.deprecation
+- rename:
+    field: elasticsearch.deprecation.process.thread.name
+    target_field: process.thread.name
+    ignore_missing: true
 - rename:
     field: elasticsearch.deprecation.component
     target_field: elasticsearch.component
+    ignore_missing: true
 - dot_expander:
     field: cluster.name
     path: elasticsearch.deprecation
@@ -48,9 +78,17 @@ processors:
 - rename:
     field: elasticsearch.deprecation.message
     target_field: message
-- date:
+- rename:
+    field: elasticsearch.deprecation.@timestamp
+    target_field: '@timestamp'
+    ignore_missing: true
+- rename:
     field: elasticsearch.deprecation.timestamp
     target_field: '@timestamp'
+    ignore_missing: true
+- date:
+    field: '@timestamp'
+    target_field: '@timestamp'
     formats:
     - ISO8601
     ignore_failure: true

filebeat/module/elasticsearch/deprecation/ingest/pipeline.yml

@@ -33,7 +33,10 @@ processors:
     value: "{{elasticsearch.node.name}}"
     if: "ctx?.elasticsearch?.node?.name != null"
 - remove:
-    field: elasticsearch.deprecation.timestamp
+    field:
+    - elasticsearch.deprecation.timestamp
+    - elasticsearch.deprecation.@timestamp
+    ignore_missing: true
 - remove:
     field:
     - first_char

filebeat/module/elasticsearch/deprecation/test/es_deprecation-json.800.log

@@ -0,0 +1,15 @@
+{"@timestamp":"2020-04-15T12:35:20.315Z", "log.level": "WARN",  "message":"Field parameter [precision] is deprecated and will be removed in a future version." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper","type":"deprecation","cluster.uuid":"a0P-i2H5R9-tJqwtF7BL0A","node.id":"FFMF7MVISuCWZMtxGmcGhg","node.name":"integTest-0","cluster.name":"integTest"}
+{"@timestamp":"2020-04-15T12:35:20.316Z", "log.level": "WARN",  "message":"Field parameter [tree] is deprecated and will be removed in a future version." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper","type":"deprecation","cluster.uuid":"a0P-i2H5R9-tJqwtF7BL0A","node.id":"FFMF7MVISuCWZMtxGmcGhg","node.name":"integTest-0","cluster.name":"integTest"}
+{"@timestamp":"2020-04-15T12:35:20.366Z", "log.level": "WARN",  "message":"Field parameter [precision] is deprecated and will be removed in a future version." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper","type":"deprecation","cluster.uuid":"a0P-i2H5R9-tJqwtF7BL0A","node.id":"FFMF7MVISuCWZMtxGmcGhg","node.name":"integTest-0","cluster.name":"integTest"}
+{"@timestamp":"2020-04-15T12:35:20.367Z", "log.level": "WARN",  "message":"Field parameter [strategy] is deprecated and will be removed in a future version." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper","type":"deprecation","cluster.uuid":"a0P-i2H5R9-tJqwtF7BL0A","node.id":"FFMF7MVISuCWZMtxGmcGhg","node.name":"integTest-0","cluster.name":"integTest"}
+{"@timestamp":"2020-04-15T12:35:20.479Z", "log.level": "WARN",  "message":"Field parameter [precision] is deprecated and will be removed in a future version." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][clusterApplierService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper","type":"deprecation","cluster.uuid":"a0P-i2H5R9-tJqwtF7BL0A","node.id":"FFMF7MVISuCWZMtxGmcGhg","node.name":"integTest-0","cluster.name":"integTest"}
+{"@timestamp":"2020-04-15T12:35:20.480Z", "log.level": "WARN",  "message":"Field parameter [strategy] is deprecated and will be removed in a future version." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][clusterApplierService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper","type":"deprecation","cluster.uuid":"a0P-i2H5R9-tJqwtF7BL0A","node.id":"FFMF7MVISuCWZMtxGmcGhg","node.name":"integTest-0","cluster.name":"integTest"}
+{"@timestamp":"2020-04-15T12:35:20.481Z", "log.level": "WARN",  "message":"Field parameter [precision] is deprecated and will be removed in a future version." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][clusterApplierService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper","type":"deprecation","cluster.uuid":"a0P-i2H5R9-tJqwtF7BL0A","node.id":"FFMF7MVISuCWZMtxGmcGhg","node.name":"integTest-0","cluster.name":"integTest"}
+{"@timestamp":"2020-04-15T12:35:20.487Z", "log.level": "WARN",  "message":"Field parameter [strategy] is deprecated and will be removed in a future version." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][clusterApplierService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper","type":"deprecation","cluster.uuid":"a0P-i2H5R9-tJqwtF7BL0A","node.id":"FFMF7MVISuCWZMtxGmcGhg","node.name":"integTest-0","cluster.name":"integTest"}
+{"@timestamp":"2020-04-16T13:46:33.582Z", "log.level": "WARN",  "message":"[PUT /_xpack/security/user/{username}/_password] is deprecated! Use [PUT /_security/user/{username}/_password] instead." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[n1][http_server_worker][T#3]","log.logger":"org.elasticsearch.deprecation.rest.RestController","type":"deprecation","cluster.uuid":"ZGYecRsDQPK_-ktRec3ZGQ","node.id":"Ni-9zbrZRm24wm7_zNtMTw","node.name":"n1","cluster.name":"es800"}
+{"@timestamp":"2020-04-16T13:46:34.219Z", "log.level": "WARN",  "message":"[PUT /_xpack/security/user/{username}/_password] is deprecated! Use [PUT /_security/user/{username}/_password] instead." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[n1][http_server_worker][T#4]","log.logger":"org.elasticsearch.deprecation.rest.RestController","type":"deprecation","cluster.uuid":"ZGYecRsDQPK_-ktRec3ZGQ","node.id":"Ni-9zbrZRm24wm7_zNtMTw","node.name":"n1","cluster.name":"es800"}
+{"@timestamp":"2020-04-16T13:46:34.339Z", "log.level": "WARN",  "message":"[PUT /_xpack/security/user/{username}/_password] is deprecated! Use [PUT /_security/user/{username}/_password] instead." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[n1][http_server_worker][T#5]","log.logger":"org.elasticsearch.deprecation.rest.RestController","type":"deprecation","cluster.uuid":"ZGYecRsDQPK_-ktRec3ZGQ","node.id":"Ni-9zbrZRm24wm7_zNtMTw","node.name":"n1","cluster.name":"es800"}
+{"@timestamp":"2020-04-16T13:46:34.455Z", "log.level": "WARN",  "message":"[PUT /_xpack/security/user/{username}/_password] is deprecated! Use [PUT /_security/user/{username}/_password] instead." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[n1][http_server_worker][T#6]","log.logger":"org.elasticsearch.deprecation.rest.RestController","type":"deprecation","cluster.uuid":"ZGYecRsDQPK_-ktRec3ZGQ","node.id":"Ni-9zbrZRm24wm7_zNtMTw","node.name":"n1","cluster.name":"es800"}
+{"@timestamp":"2020-04-16T13:47:36.309Z", "log.level": "WARN",  "message":"index name [.apm-custom-link] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" , "service.name":"ES_ECS","process.thread.name":"elasticsearch[n1][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.cluster.metadata.MetadataCreateIndexService","type":"deprecation","cluster.uuid":"ZGYecRsDQPK_-ktRec3ZGQ","node.id":"Ni-9zbrZRm24wm7_zNtMTw","node.name":"n1","cluster.name":"es800"}
+{"@timestamp":"2020-04-16T13:55:56.365Z", "log.level": "WARN",  "message":"index name [.monitoring-alerts-7] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" , "service.name":"ES_ECS","process.thread.name":"elasticsearch[n1][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.cluster.metadata.MetadataCreateIndexService","type":"deprecation","cluster.uuid":"ZGYecRsDQPK_-ktRec3ZGQ","node.id":"Ni-9zbrZRm24wm7_zNtMTw","node.name":"n1","cluster.name":"es800"}
+{"@timestamp":"2020-04-16T13:56:14.697Z", "log.level": "WARN",  "message":"[types removal] Using the _type field in queries and aggregations is deprecated, prefer to use a field instead." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[n1][search][T#7]","log.logger":"org.elasticsearch.deprecation.index.query.QueryShardContext","type":"deprecation","cluster.uuid":"ZGYecRsDQPK_-ktRec3ZGQ","node.id":"Ni-9zbrZRm24wm7_zNtMTw","node.name":"n1","cluster.name":"es800"}