From 3605c475a7b97fd696b2c710fa88bd09f5af0dba Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Wed, 13 May 2020 12:07:30 +0200 Subject: [PATCH] Cisco asa/ftd: Remove _temp_ fields on failure (#18392) The shared pipeline for Cisco ASA and FTD creates temporary fields under the _temp_ object. If a failure happens in the middle of the pipeline, all those fields would be indexed, causing the index mapping to grow too big. Fixes #18391 --- CHANGELOG.next.asciidoc | 1 + .../module/cisco/shared/ingest/asa-ftd-pipeline.yml | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index d189637976d..b89113002e6 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -128,6 +128,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fixed typo in log message. {pull}17897[17897] - Fix Cisco ASA ASA 3020** and 106023 messages {pull}17964[17964] - Unescape file name from SQS message. {pull}18370[18370] +- Improve cisco asa and ftd pipelines' failure handler to avoid mapping temporary fields. {issue}18391[18391] {pull}18392[18392] *Heartbeat* diff --git a/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml b/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml index babf697616b..d9066147953 100644 --- a/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml +++ b/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml @@ -1277,6 +1277,16 @@ processors: ignore_missing: true on_failure: + # Copy any fields under _temp_.cisco to its final destination. Those can help + # with diagnosing the failure. + - rename: + field: _temp_.cisco + target_field: "cisco.{< .internal_prefix >}" + ignore_failure: true + # Remove _temp_ to avoid adding a lot of unnecessary fields to the index. + - remove: + field: _temp_ + ignore_missing: true - append: field: "error.message" value: "{{ _ingest.on_failure_message }}"