From 7669c6c8f07f35f99a69cadf1a9128512cd059a0 Mon Sep 17 00:00:00 2001 From: SG Date: Tue, 6 Feb 2024 14:22:08 -0700 Subject: [PATCH] work in progress for idaholab/Malcolm#395, malcolm reporting capture statistics from zeek/suricata --- logstash/pipelines/beats/01_input_beats.conf | 11 ++++--- logstash/pipelines/beats/11_beats_logs.conf | 2 -- logstash/pipelines/beats/98_finalize.conf | 14 --------- .../pipelines/enrichment/98_finalize.conf | 24 --------------- logstash/pipelines/output/98_finalize.conf | 29 +++++++++++++++++++ 5 files changed, 34 insertions(+), 46 deletions(-) create mode 100644 logstash/pipelines/output/98_finalize.conf diff --git a/logstash/pipelines/beats/01_input_beats.conf b/logstash/pipelines/beats/01_input_beats.conf index 65a3e1abb..25f418ff9 100644 --- a/logstash/pipelines/beats/01_input_beats.conf +++ b/logstash/pipelines/beats/01_input_beats.conf @@ -8,13 +8,12 @@ filter { # this pipeline only needs to see logs from other misc. beats used by Malcolm if ("_malcolm_beats" in [tags]) or ("_zeekdiagnostic" in [tags]) { - mutate { id => "mutate_filebeat_malcolm_beats_forward_tag_remove" - remove_tag => [ "_malcolm_beats" ] } + + # rename message to event.original + mutate { id => "mutate_rename_beats_message" + rename => { "[message]" => "[event][original]" } } + } else { drop { id => "drop_not_malcolm_beats" } } - - # rename message to event.original - mutate { id => "mutate_rename_beats_message" - rename => { "[message]" => "[event][original]" } } } diff --git a/logstash/pipelines/beats/11_beats_logs.conf b/logstash/pipelines/beats/11_beats_logs.conf index a44c2dac7..3bb35c3d3 100644 --- a/logstash/pipelines/beats/11_beats_logs.conf +++ b/logstash/pipelines/beats/11_beats_logs.conf @@ -61,7 +61,6 @@ filter { id => "grok_beat_nginx_access" patterns_dir => "/usr/share/logstash/malcolm-patterns" match => { "[event][original]" => "%{NGINX_ACCESS}" } - # remove_tag => ["_grokparsefailure"] } } if ([event][dataset] == "nginx.error") { @@ -69,7 +68,6 @@ filter { id => "grok_beat_nginx_error" patterns_dir => "/usr/share/logstash/malcolm-patterns" match => { "[event][original]" => "%{NGINX_ERROR}" } - # remove_tag => ["_grokparsefailure"] } } diff --git a/logstash/pipelines/beats/98_finalize.conf b/logstash/pipelines/beats/98_finalize.conf index 36e5cd6ac..4db017bea 100644 --- a/logstash/pipelines/beats/98_finalize.conf +++ b/logstash/pipelines/beats/98_finalize.conf @@ -72,20 +72,6 @@ filter { ] } - # remove tags we'd rather not see - mutate { id => "mutate_beats_tags_remove" - remove_tag => [ "beats_input_codec_plain_applied", - "beats_input_raw_event", - "_malcolm_miscbeat", - "_dateparsefailure", - "_grokparsefailure", - "_jsonparsefailure", - "_dissectfailure", - "_ouilookupfailure", - "_geoip_lookup_failure", - "_zeekdiagnostic" ] } - - # event.provider if (![event][provider]) { mutate { id => "mutate_add_field_event_provider_beats" add_field => { "[event][provider]" => "malcolm" } } } diff --git a/logstash/pipelines/enrichment/98_finalize.conf b/logstash/pipelines/enrichment/98_finalize.conf index 0d979c734..2a8be2578 100644 --- a/logstash/pipelines/enrichment/98_finalize.conf +++ b/logstash/pipelines/enrichment/98_finalize.conf @@ -59,28 +59,4 @@ filter { "[message]" ] } - - # remove tags we'd rather not see - - mutate { id => "mutate_enrichment_tags_remove" - remove_tag => [ "beats_input_codec_plain_applied", - "beats_input_raw_event", - "_dateparsefailure", - "_grokparsefailure", - "_jsonparsefailure", - "_dissectfailure", - "_ouilookupfailure", - "_geoip_lookup_failure", - "_filebeat_suricata", - "_filebeat_suricata_hedgehog_live", - "_filebeat_suricata_live", - "_filebeat_suricata_malcolm_live", - "_filebeat_suricata_malcolm_upload", - "_filebeat_suricata_upload", - "_filebeat_zeek", - "_filebeat_zeek_hedgehog_live", - "_filebeat_zeek_live", - "_filebeat_zeek_malcolm_live", - "_filebeat_zeek_malcolm_upload", - "_filebeat_zeek_upload" ] } } \ No newline at end of file diff --git a/logstash/pipelines/output/98_finalize.conf b/logstash/pipelines/output/98_finalize.conf new file mode 100644 index 000000000..c383d8912 --- /dev/null +++ b/logstash/pipelines/output/98_finalize.conf @@ -0,0 +1,29 @@ +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. + +filter { + # remove tags we'd rather not see globally + mutate { id => "mutate_final_tags_remove" + remove_tag => [ "_dateparsefailure", + "_dissectfailure", + "_filebeat_suricata", + "_filebeat_suricata_hedgehog_live", + "_filebeat_suricata_live", + "_filebeat_suricata_malcolm_live", + "_filebeat_suricata_malcolm_upload", + "_filebeat_suricata_upload", + "_filebeat_zeek", + "_filebeat_zeek_hedgehog_live", + "_filebeat_zeek_live", + "_filebeat_zeek_malcolm_live", + "_filebeat_zeek_malcolm_upload", + "_filebeat_zeek_upload", + "_geoip_lookup_failure", + "_grokparsefailure", + "_jsonparsefailure", + "_malcolm_miscbeat", + "_ouilookupfailure", + "_zeekdiagnostic", + "beats_input_codec_plain_applied", + "beats_input_raw_event" ] } + +} \ No newline at end of file