work on idaholab#19, severity tagging for GEO countries

mmguero-dev · Aug 19, 2021 · 6a44943 · 6a44943
1 parent f2dc744
commit 6a44943
Show file tree

Hide file tree

Showing 7 changed files with 16 additions and 6 deletions.
diff --git a/README.md b/README.md
@@ -498,7 +498,7 @@ Various other environment variables inside of `docker-compose.yml` can be tweake
 
 * `CONNECTION_SECONDS_SEVERITY_THRESHOLD` - when [severity scoring](#Severity) is enabled, this variable indicates the duration threshold (in seconds) for assigning severity to long connections (default `3600`)
 
-* `QUESTIONABLE_COUNTRY_CODES` - when [severity scoring](#Severity) is enabled, this variable defines a comma-separated list of countries of concern (using [ISO 3166-1 alpha-2 codes](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2#Current_codes)) (default `'CN,KP,KR,RU,UA'`)
+* `QUESTIONABLE_COUNTRY_CODES` - when [severity scoring](#Severity) is enabled, this variable defines a comma-separated list of countries of concern (using [ISO 3166-1 alpha-2 codes](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2#Current_codes)) (default `'CN,KP,RU,UA'`)
 
 * `ES_EXTERNAL_HOSTS` – if specified (in the format `'10.0.0.123:9200'`), logs received by Logstash will be forwarded on to another external Elasticsearch instance in addition to the one maintained locally by Malcolm
 

diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml
@@ -112,7 +112,7 @@ x-common-lookup-variables: &common-lookup-variables
   FREQ_SEVERITY_THRESHOLD : '2.0'
   TOTAL_MEGABYTES_SEVERITY_THRESHOLD : 1000
   CONNECTION_SECONDS_SEVERITY_THRESHOLD : 3600
-  QUESTIONABLE_COUNTRY_CODES : 'CN,KP,KR,RU,UA'
+  QUESTIONABLE_COUNTRY_CODES : 'CN,KP,RU,UA'
 
 x-common-beats-variables: &common-beats-variables
   BEATS_SSL : 'false'

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -112,7 +112,7 @@ x-common-lookup-variables: &common-lookup-variables
   FREQ_SEVERITY_THRESHOLD : '2.0'
   TOTAL_MEGABYTES_SEVERITY_THRESHOLD : 1000
   CONNECTION_SECONDS_SEVERITY_THRESHOLD : 3600
-  QUESTIONABLE_COUNTRY_CODES : 'CN,KP,KR,RU,UA'
+  QUESTIONABLE_COUNTRY_CODES : 'CN,KP,RU,UA'
 
 x-common-beats-variables: &common-beats-variables
   BEATS_SSL : 'false'

diff --git a/kibana/zeek_template.json b/kibana/zeek_template.json
@@ -39,6 +39,8 @@
         "event.kind": { "type": "keyword" },
         "event.outcome": { "type": "keyword" },
         "event.provider": { "type": "keyword" },
+        "event.risk_score": { "type": "float" },
+        "event.risk_score_norm": { "type": "float" },
         "event.start": { "type": "date" },
         "event.severity": { "type": "integer" },
         "event.severity_tags": { "type": "keyword" },

diff --git a/logstash/pipelines/enrichment/19_severity.conf b/logstash/pipelines/enrichment/19_severity.conf
@@ -37,7 +37,7 @@ filter {
     if ([srcGEO]) or ([dstGEO]) or ([dns][GEO]) {
       ruby {
         id => "ruby_add_field_severity_geo"
-        init => "countriesStr = ENV['QUESTIONABLE_COUNTRY_CODES'] || 'CN,KP,KR,RU,UA' ; $countries = countriesStr.gsub(/\s+/, '').upcase.split(',')"
+        init => "countriesStr = ENV['QUESTIONABLE_COUNTRY_CODES'] || 'CN,KP,RU,UA' ; $countries = countriesStr.gsub(/\s+/, '').upcase.split(',')"
         code => "
           srcGEOs = event.get('[srcGEO]')
           dstGEOs = event.get('[dstGEO]')
@@ -398,9 +398,12 @@ filter {
         end
         if !sevtags.nil? and (sevtags.length > 0) and ($severityMappings.length > 0) then
           sevhash = Hash[sevtags.select{|key| ($severityMappings[key].nil? ? 0 : $severityMappings[key]) > 0}.map{|key| [key, $severityMappings[key]]}]
-          sevnum = [sevhash.values.sum, 100].min
+          sevnum = sevhash.values.sum
           if (sevnum > 0) then
-            event.set('[event][severity]', sevnum)
+            sevnumNorm = [sevnum, 100].min
+            event.set('[event][risk_score]', Float(sevnum))
+            event.set('[event][risk_score_norm]', Float(sevnumNorm))
+            event.set('[event][severity]', sevnumNorm)
             event.set('[event][severity_tags]', sevhash.keys)
           else
             event.remove('[event][severity_tags]')

diff --git a/moloch/etc/config.ini b/moloch/etc/config.ini
@@ -73,6 +73,8 @@ zeek.ts=db:zeek.ts;kind:termfield;friendly:Timestamp;help:Zeek Timestamp
 host.name=db:host.name;kind:termfield;friendly:Zeek Node;help:Zeek Node
 event.severity=db:event.severity;kind:integer;friendly:Severity;help:Severity
 event.severity_tags=db:event.severity_tags;kind:termfield;friendly:Severity Tags;help:Severity Tags
+event.risk_score=db:event.risk_score;kind:termfield;friendly:Risk Score;help:Risk Score
+event.risk_score_norm=db:event.risk_score_norm;kind:termfield;friendly:Risk Score (Normalized);help:Risk Score (Normalized)
 
 # basic connection information
 zeek.orig_h=db:zeek.orig_h;kind:termfield;friendly:Originating host;help:Originating Host

diff --git a/moloch/wise/source.zeeklogs.js b/moloch/wise/source.zeeklogs.js
@@ -27,6 +27,8 @@ function ZeekLogs (api, section) {
   // add right-clicks
   var allFields = [
     "communityId",
+    "event.risk_score",
+    "event.risk_score_norm",
     "event.severity",
     "event.severity_tags",
     "host.name",
@@ -953,6 +955,7 @@ function ZeekLogs (api, section) {
     "      +arrayList(session.zeek, 'freq_score_v1', 'Freq Score v1', 'zeek.freq_score_v1')\n" +
     "      +arrayList(session.zeek, 'freq_score_v2', 'Freq Score v2', 'zeek.freq_score_v2')\n" +
     "      +arrayList(session.event, 'severity', 'Severity', 'event.severity')\n" +
+    "      +arrayList(session.event, 'risk_score', 'Risk Score', 'event.risk_score')\n" +
     "      +arrayList(session.event, 'severity_tags', 'Severity Tags', 'event.severity_tags')\n" +
 
     // file information