work in progress for idaholab#395, malcolm reporting capture statisti…

…cs from zeek/suricata
mmguero-dev · Feb 6, 2024 · 7669c6c · 7669c6c
1 parent 50fb5c9
commit 7669c6c
Show file tree

Hide file tree

Showing 5 changed files with 34 additions and 46 deletions.
diff --git a/logstash/pipelines/beats/01_input_beats.conf b/logstash/pipelines/beats/01_input_beats.conf
@@ -8,13 +8,12 @@ filter {
   # this pipeline only needs to see logs from other misc. beats used by Malcolm
   if ("_malcolm_beats" in [tags]) or
      ("_zeekdiagnostic" in [tags]) {
-    mutate { id => "mutate_filebeat_malcolm_beats_forward_tag_remove"
-             remove_tag => [ "_malcolm_beats" ] }
+
+    # rename message to event.original
+    mutate { id => "mutate_rename_beats_message"
+             rename => { "[message]" => "[event][original]" } }
+
   } else {
     drop { id => "drop_not_malcolm_beats" }
   }
-
-  # rename message to event.original
-  mutate { id => "mutate_rename_beats_message"
-           rename => { "[message]" => "[event][original]" } }
 }
diff --git a/logstash/pipelines/beats/11_beats_logs.conf b/logstash/pipelines/beats/11_beats_logs.conf
@@ -61,15 +61,13 @@ filter {
         id => "grok_beat_nginx_access"
         patterns_dir => "/usr/share/logstash/malcolm-patterns"
         match => { "[event][original]" => "%{NGINX_ACCESS}" }
-        # remove_tag => ["_grokparsefailure"]
       }
     }
     if ([event][dataset] == "nginx.error") {
       grok {
         id => "grok_beat_nginx_error"
         patterns_dir => "/usr/share/logstash/malcolm-patterns"
         match => { "[event][original]" => "%{NGINX_ERROR}" }
-        # remove_tag => ["_grokparsefailure"]
       }
     }
 

diff --git a/logstash/pipelines/beats/98_finalize.conf b/logstash/pipelines/beats/98_finalize.conf
@@ -72,20 +72,6 @@ filter {
       ]
     }
 
-    # remove tags we'd rather not see
-    mutate { id => "mutate_beats_tags_remove"
-             remove_tag => [ "beats_input_codec_plain_applied",
-                             "beats_input_raw_event",
-                             "_malcolm_miscbeat",
-                             "_dateparsefailure",
-                             "_grokparsefailure",
-                             "_jsonparsefailure",
-                             "_dissectfailure",
-                             "_ouilookupfailure",
-                             "_geoip_lookup_failure",
-                             "_zeekdiagnostic" ] }
-
-
     # event.provider
     if (![event][provider]) { mutate { id => "mutate_add_field_event_provider_beats"
                                        add_field => { "[event][provider]" => "malcolm" } } }

diff --git a/logstash/pipelines/enrichment/98_finalize.conf b/logstash/pipelines/enrichment/98_finalize.conf
@@ -59,28 +59,4 @@ filter {
         "[message]"
       ]
     }
-
-    # remove tags we'd rather not see
-
-    mutate { id => "mutate_enrichment_tags_remove"
-             remove_tag => [ "beats_input_codec_plain_applied",
-                             "beats_input_raw_event",
-                             "_dateparsefailure",
-                             "_grokparsefailure",
-                             "_jsonparsefailure",
-                             "_dissectfailure",
-                             "_ouilookupfailure",
-                             "_geoip_lookup_failure",
-                             "_filebeat_suricata",
-                             "_filebeat_suricata_hedgehog_live",
-                             "_filebeat_suricata_live",
-                             "_filebeat_suricata_malcolm_live",
-                             "_filebeat_suricata_malcolm_upload",
-                             "_filebeat_suricata_upload",
-                             "_filebeat_zeek",
-                             "_filebeat_zeek_hedgehog_live",
-                             "_filebeat_zeek_live",
-                             "_filebeat_zeek_malcolm_live",
-                             "_filebeat_zeek_malcolm_upload",
-                             "_filebeat_zeek_upload" ] }
 }
diff --git a/logstash/pipelines/output/98_finalize.conf b/logstash/pipelines/output/98_finalize.conf
@@ -0,0 +1,29 @@
+# Copyright (c) 2024 Battelle Energy Alliance, LLC.  All rights reserved.
+
+filter {
+    # remove tags we'd rather not see globally
+    mutate { id => "mutate_final_tags_remove"
+             remove_tag => [ "_dateparsefailure",
+                             "_dissectfailure",
+                             "_filebeat_suricata",
+                             "_filebeat_suricata_hedgehog_live",
+                             "_filebeat_suricata_live",
+                             "_filebeat_suricata_malcolm_live",
+                             "_filebeat_suricata_malcolm_upload",
+                             "_filebeat_suricata_upload",
+                             "_filebeat_zeek",
+                             "_filebeat_zeek_hedgehog_live",
+                             "_filebeat_zeek_live",
+                             "_filebeat_zeek_malcolm_live",
+                             "_filebeat_zeek_malcolm_upload",
+                             "_filebeat_zeek_upload",
+                             "_geoip_lookup_failure",
+                             "_grokparsefailure",
+                             "_jsonparsefailure",
+                             "_malcolm_miscbeat",
+                             "_ouilookupfailure",
+                             "_zeekdiagnostic",
+                             "beats_input_codec_plain_applied",
+                             "beats_input_raw_event" ] }
+
+}