work in progress for idaholab#395, malcolm reporting capture statisti…

…cs from zeek/suricata
mmguero-dev · Feb 7, 2024 · 8a5d14a · 8a5d14a
1 parent d74deff
commit 8a5d14a
Show file tree

Hide file tree

Showing 4 changed files with 54 additions and 5 deletions.
diff --git a/dashboards/templates/composable/component/suricata_stats.json b/dashboards/templates/composable/component/suricata_stats.json
@@ -113,10 +113,15 @@
                 "detect.alert": { "type": "long" },
                 "detect.alert_queue_overflow": { "type": "long" },
                 "detect.alerts_suppressed": { "type": "long" },
-                "detect.engines.id": { "type": "long" },
-                "detect.engines.last_reload": { "type": "date" },
-                "detect.engines.rules_failed": { "type": "long" },
-                "detect.engines.rules_loaded": { "type": "long" },
+                "detect.engines": {
+                  "type": "nested",
+                  "properties": {
+                    "id": { "type": "long" },
+                    "last_reload": { "type": "date" },
+                    "rules_failed": { "type": "long" },
+                    "rules_loaded": { "type": "long" }
+                  }
+                },
                 "file_store.open_files": { "type": "long" },
                 "flow.emerg_mode_entered": { "type": "long" },
                 "flow.emerg_mode_over": { "type": "long" },

diff --git a/logstash/pipelines/beats/11_beats_logs.conf b/logstash/pipelines/beats/11_beats_logs.conf
@@ -781,9 +781,25 @@ filter {
     } # [miscbeat][winstat]
 
   } else if ("_zeekdiagnostic" in [tags]) {
+    #-------------------------------------------------
+    # Zeek diagnostic logs
+    # https://docs.zeek.org/en/master/script-reference/log-files.html#zeek-diagnostics
+
+    if ([zeek][stats]) {
+      # remove zero values from zeek stats
+      ruby {
+        id => "ruby_zeek_remove_zero_stats"
+        path => "/usr/share/logstash/malcolm-ruby/compact_event_hash.rb"
+        script_params => {
+          "field" => "[zeek][stats]"
+          "discard_zeroes" => "true"
+        }
+      }
+    }
 
     mutate { id => "mutate_add_field_event_module_zeek_diagnostic"
              add_field => { "[event][module]" => "zeek" } }
+
     mutate { id => "mutate_remove_fields_zeek_diagnostic"
              remove_field => [ "[event][original]",
                                "[firstPacket]",
@@ -794,11 +810,28 @@ filter {
                                "[event][duration]" ] }
 
   } else if ("_suricatastats" in [tags]) {
+    #-------------------------------------------------
+    # Suricata statistics
+    # https://docs.suricata.io/en/suricata-6.0.2/configuration/suricata-yaml.html#stats
+
+    if ([suricata][stats]) {
+      # remove zero values from suricata stats
+      ruby {
+        id => "ruby_suricata_remove_zero_stats"
+        path => "/usr/share/logstash/malcolm-ruby/compact_event_hash.rb"
+        script_params => {
+          "field" => "[suricata][stats]"
+          "discard_zeroes" => "true"
+        }
+      }
+    }
 
     mutate { id => "mutate_add_field_event_module_suricata_stats"
              add_field => { "[event][module]" => "suricata" } }
+
     mutate { id => "mutate_remove_fields_suricata_stats"
-             remove_field => [ "[firstPacket]",
+             remove_field => [ "[event][original]",
+                               "[firstPacket]",
                                "[lastPacket]",
                                "[timestamp]",
                                "[suricata][timestamp]",

diff --git a/logstash/ruby/compact_event.rb b/logstash/ruby/compact_event.rb
@@ -2,6 +2,11 @@ def concurrency
   :shared
 end
 
+def register(params)
+  _discard_zeroes_str = params["discard_zeroes"]
+  @discard_zeroes = [1, true, '1', 'true', 't', 'on', 'enabled'].include?(_discard_zeroes_str.to_s.downcase)
+end
+
 def compact(h)
   h.inject({}) do |result, (k, v)|
     case v
@@ -10,6 +15,8 @@ def compact(h)
       result[k] = c unless c.empty?
     when String
       result[k] = v unless (v.empty? || (v == "-") || (v == "?") || (v == "(empty)") || (v == "(none)") || (v == "(null)") || (v == "unset") || (v == "Nul"))
+    when Numeric
+      result[k] = v unless (@discard_zeroes && v.zero?)
     when Array
       c = v.delete_if{|e| e.nil? || (e.is_a?(String) && (e.empty? || (e == "-") || (e == "?") || (e == "(empty)") || (e == "(none)") || (e == "(null)") || (e == "unset") || (e == "Nul")))}
       result[k] = c unless c.empty?

diff --git a/logstash/ruby/compact_event_hash.rb b/logstash/ruby/compact_event_hash.rb
@@ -4,6 +4,8 @@ def concurrency
 
 def register(params)
   @field = params["field"]
+  _discard_zeroes_str = params["discard_zeroes"]
+  @discard_zeroes = [1, true, '1', 'true', 't', 'on', 'enabled'].include?(_discard_zeroes_str.to_s.downcase)
 end
 
 def compact(h)
@@ -14,6 +16,8 @@ def compact(h)
       result[k] = c unless c.empty?
     when String
       result[k] = v unless (v.empty? || (v == "-") || (v == "(empty)"))
+    when Numeric
+      result[k] = v unless (@discard_zeroes && v.zero?)
     when Array
       c = v.delete_if{|e| e.nil? || (e.is_a?(String) && (e.empty? || (e == "-") || (e == "(empty)")))}
       result[k] = c unless c.empty?