diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml index ec7e9d524..5e899c64f 100644 --- a/docker-compose-standalone.yml +++ b/docker-compose-standalone.yml @@ -89,8 +89,9 @@ x-kibana-helper-variables: &kibana-helper-variables ISM_SNAPSHOT_REPO : 'logs' x-logstash-variables: &logstash-variables - LOGSTASH_OUI_LOOKUP : 'true' - LOGSTASH_REVERSE_DNS : 'false' + LOGSTASH_OUI_LOOKUP : 'true' + LOGSTASH_SEVERITY_SCORING : 'true' + LOGSTASH_REVERSE_DNS : 'false' # ES_EXTERNAL_HOSTS : '10.0.0.123:9200' # ES_EXTERNAL_SSL : 'true' # ES_EXTERNAL_SSL_CERTIFICATE_VERIFICATION : 'false' @@ -108,6 +109,7 @@ x-common-upload-variables: &common-upload-variables x-common-lookup-variables: &common-lookup-variables FREQ_LOOKUP : 'false' + FREQ_SEVERITY_THRESHOLD : '2.0' x-common-beats-variables: &common-beats-variables BEATS_SSL : 'false' diff --git a/docker-compose.yml b/docker-compose.yml index a5a95ddc4..5b396cb59 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -89,8 +89,9 @@ x-kibana-helper-variables: &kibana-helper-variables ISM_SNAPSHOT_REPO : 'logs' x-logstash-variables: &logstash-variables - LOGSTASH_OUI_LOOKUP : 'true' - LOGSTASH_REVERSE_DNS : 'false' + LOGSTASH_OUI_LOOKUP : 'true' + LOGSTASH_SEVERITY_SCORING : 'true' + LOGSTASH_REVERSE_DNS : 'false' # ES_EXTERNAL_HOSTS : '10.0.0.123:9200' # ES_EXTERNAL_SSL : 'true' # ES_EXTERNAL_SSL_CERTIFICATE_VERIFICATION : 'false' @@ -108,6 +109,7 @@ x-common-upload-variables: &common-upload-variables x-common-lookup-variables: &common-lookup-variables FREQ_LOOKUP : 'false' + FREQ_SEVERITY_THRESHOLD : '2.0' x-common-beats-variables: &common-beats-variables BEATS_SSL : 'false' diff --git a/kibana/zeek_template.json b/kibana/zeek_template.json index 9c50bdcab..5baccf4a4 100644 --- a/kibana/zeek_template.json +++ b/kibana/zeek_template.json @@ -40,6 +40,8 @@ "event.outcome": { "type": "keyword" }, "event.provider": { "type": "keyword" }, "event.start": { "type": "date" }, + "event.severity": { "type": "integer" }, + "event.severity_tags": { "type": "keyword" }, "event.type": { "type": "keyword" }, "file.accessed": { "type": "date" }, "file.created": { "type": "date" }, diff --git a/logstash/maps/malcolm_severity.yaml b/logstash/maps/malcolm_severity.yaml index 7c6c5a5b7..f9133f09f 100644 --- a/logstash/maps/malcolm_severity.yaml +++ b/logstash/maps/malcolm_severity.yaml @@ -3,7 +3,8 @@ "CONNECTION_INBOUND": 50 "CONNECTION_INTERNAL": 0 "CONNECTION_OUTBOUND": 25 -"FILE_TYPE": 5 +"DOMAIN_HIGH_ENTROPY": 0 +"FILE_TYPE": 0 "FILE_TYPE_HIGH": 75 "FILE_TYPE_MEDIUM": 50 "NOTICE_VULN": 90 @@ -13,7 +14,6 @@ "NOTICE_SCAN": 60 "PASSWORD_CLEARTEXT": 90 "PROTOCOL_OUTDATED_OR_INSECURE": 60 -"PROTOCOL_REMOTE_CONTROL": 70 "SIGNATURES_CAPA": 50 "SIGNATURES_CLAMAV": 90 "SIGNATURES_MALASS": 90 diff --git a/logstash/pipelines/enrichment/19_severity.conf b/logstash/pipelines/enrichment/19_severity.conf index af38992ee..2c14c9daa 100644 --- a/logstash/pipelines/enrichment/19_severity.conf +++ b/logstash/pipelines/enrichment/19_severity.conf @@ -2,293 +2,327 @@ filter { # see malcolm_severity.yaml for mappings to severity scores - # identify cross-segment traffic based on previously-populated tag - if ("cross_segment" in [tags]) { - mutate { id => "mutate_add_field_severity_item_cross_segment" - add_field => { "[event][severity_tags]" => "CONNECTION_CROSS_SEGMENT" } } + mutate { + id => "mutate_add_field_env_logstash_severity_scoring" + add_field => { "[@metadata][ENV_LOGSTASH_SEVERITY_SCORING]" => "${LOGSTASH_SEVERITY_SCORING:false}" } } + if ([@metadata][ENV_LOGSTASH_SEVERITY_SCORING] == "true") { - # inbound/outbound/internal/external connection based on previously-populated tag - if ("internal_source" in [tags]) and ("internal_destination" in [tags]) { - mutate { id => "mutate_add_field_severity_item_internal" - add_field => { "[event][severity_tags]" => "CONNECTION_INTERNAL" } } - } else if ("external_source" in [tags]) and ("external_destination" in [tags]) { - mutate { id => "mutate_add_field_severity_item_external" - add_field => { "[event][severity_tags]" => "CONNECTION_EXTERNAL" } } - } else if ("internal_source" in [tags]) and ("external_destination" in [tags]) { - mutate { id => "mutate_add_field_severity_item_outbound" - add_field => { "[event][severity_tags]" => "CONNECTION_OUTBOUND" } } - } else if ("external_source" in [tags]) and ("internal_destination" in [tags]) { - mutate { id => "mutate_add_field_severity_item_inbound" - add_field => { "[event][severity_tags]" => "CONNECTION_INBOUND" } } - } + # identify cross-segment traffic based on previously-populated tag + if ("cross_segment" in [tags]) { + mutate { id => "mutate_add_field_severity_item_cross_segment" + add_field => { "[event][severity_tags]" => "CONNECTION_CROSS_SEGMENT" } } + } - # these (high/medium) file types pulled from extractor_override.interesting.zeek - # and probably ought to be modified if that file is modified - if ([zeek][filetype]) { + # inbound/outbound/internal/external connection based on previously-populated tag + if ("internal_source" in [tags]) and ("internal_destination" in [tags]) { + mutate { id => "mutate_add_field_severity_item_internal" + add_field => { "[event][severity_tags]" => "CONNECTION_INTERNAL" } } + } else if ("external_source" in [tags]) and ("external_destination" in [tags]) { + mutate { id => "mutate_add_field_severity_item_external" + add_field => { "[event][severity_tags]" => "CONNECTION_EXTERNAL" } } + } else if ("internal_source" in [tags]) and ("external_destination" in [tags]) { + mutate { id => "mutate_add_field_severity_item_outbound" + add_field => { "[event][severity_tags]" => "CONNECTION_OUTBOUND" } } + } else if ("external_source" in [tags]) and ("internal_destination" in [tags]) { + mutate { id => "mutate_add_field_severity_item_inbound" + add_field => { "[event][severity_tags]" => "CONNECTION_INBOUND" } } + } - # "high" severity files are executables and scripts (?) - if (("application/PowerShell" in [zeek][filetype]) or - ("application/vnd.microsoft.portable-executable" in [zeek][filetype]) or - ("application/x-csh" in [zeek][filetype]) or - ("application/x-dosexec" in [zeek][filetype]) or - ("application/x-elf" in [zeek][filetype]) or - ("application/x-executable" in [zeek][filetype]) or - ("application/x-ms-application" in [zeek][filetype]) or - ("application/x-msdos-program" in [zeek][filetype]) or - ("application/x-pe-app-32bit-i386" in [zeek][filetype]) or - ("application/x-perl" in [zeek][filetype]) or - ("application/x-python" in [zeek][filetype]) or - ("application/x-sh" in [zeek][filetype]) or - ("text/vbscript" in [zeek][filetype])) { - mutate { id => "mutate_add_field_severity_file_type_high" - add_field => { "[event][severity_tags]" => "FILE_TYPE_HIGH" } } + # these (high/medium) file types pulled from extractor_override.interesting.zeek + # and probably ought to be modified if that file is modified + if ([zeek][filetype]) { - # "medium" severity files are everything else in the "interesting" list - } else if (("application/binary" in [zeek][filetype]) or - ("application/ecmascript" in [zeek][filetype]) or - ("application/hta" in [zeek][filetype]) or - ("application/java-archive" in [zeek][filetype]) or - ("application/java-serialized-object" in [zeek][filetype]) or - ("application/java-vm" in [zeek][filetype]) or - ("application/javascript" in [zeek][filetype]) or - ("application/ms-vsi" in [zeek][filetype]) or - ("application/msaccess" in [zeek][filetype]) or - ("application/msaccess.addin" in [zeek][filetype]) or - ("application/msaccess.cab" in [zeek][filetype]) or - ("application/msaccess.ftemplate" in [zeek][filetype]) or - ("application/msaccess.runtime" in [zeek][filetype]) or - ("application/msaccess.webapplication" in [zeek][filetype]) or - ("application/msexcel" in [zeek][filetype]) or - ("application/mspowerpoint" in [zeek][filetype]) or - ("application/msword" in [zeek][filetype]) or - ("application/octet-stream" in [zeek][filetype]) or - ("application/pdf" in [zeek][filetype]) or - ("application/rtf" in [zeek][filetype]) or - ("application/vnd.apple.installer+xml" in [zeek][filetype]) or - ("application/vnd.ms-cab-compressed" in [zeek][filetype]) or - ("application/vnd.ms-excel" in [zeek][filetype]) or - ("application/vnd.ms-excel.addin.macroEnabled.12" in [zeek][filetype]) or - ("application/vnd.ms-excel.addin.macroenabled.12" in [zeek][filetype]) or - ("application/vnd.ms-excel.sheet.binary.macroEnabled.12" in [zeek][filetype]) or - ("application/vnd.ms-excel.sheet.binary.macroenabled.12" in [zeek][filetype]) or - ("application/vnd.ms-excel.sheet.macroEnabled.12" in [zeek][filetype]) or - ("application/vnd.ms-excel.sheet.macroenabled.12" in [zeek][filetype]) or - ("application/vnd.ms-excel.template.macroEnabled.12" in [zeek][filetype]) or - ("application/vnd.ms-excel.template.macroenabled.12" in [zeek][filetype]) or - ("application/vnd.ms-office.calx" in [zeek][filetype]) or - ("application/vnd.ms-officetheme" in [zeek][filetype]) or - ("application/vnd.ms-powerpoint" in [zeek][filetype]) or - ("application/vnd.ms-powerpoint.addin.macroEnabled.12" in [zeek][filetype]) or - ("application/vnd.ms-powerpoint.addin.macroenabled.12" in [zeek][filetype]) or - ("application/vnd.ms-powerpoint.presentation.macroEnabled.12" in [zeek][filetype]) or - ("application/vnd.ms-powerpoint.presentation.macroenabled.12" in [zeek][filetype]) or - ("application/vnd.ms-powerpoint.slide.macroEnabled.12" in [zeek][filetype]) or - ("application/vnd.ms-powerpoint.slide.macroenabled.12" in [zeek][filetype]) or - ("application/vnd.ms-powerpoint.slideshow.macroEnabled.12" in [zeek][filetype]) or - ("application/vnd.ms-powerpoint.slideshow.macroenabled.12" in [zeek][filetype]) or - ("application/vnd.ms-powerpoint.template.macroEnabled.12" in [zeek][filetype]) or - ("application/vnd.ms-powerpoint.template.macroenabled.12" in [zeek][filetype]) or - ("application/vnd.ms-word.document.macroEnabled.12" in [zeek][filetype]) or - ("application/vnd.ms-word.document.macroenabled.12" in [zeek][filetype]) or - ("application/vnd.ms-word.template.macroEnabled.12" in [zeek][filetype]) or - ("application/vnd.ms-word.template.macroenabled.12" in [zeek][filetype]) or - ("application/vnd.openofficeorg.extension" in [zeek][filetype]) or - ("application/vnd.openxmlformats-officedocument.presentationml.presentation" in [zeek][filetype]) or - ("application/vnd.openxmlformats-officedocument.presentationml.slide" in [zeek][filetype]) or - ("application/vnd.openxmlformats-officedocument.presentationml.slideshow" in [zeek][filetype]) or - ("application/vnd.openxmlformats-officedocument.presentationml.template" in [zeek][filetype]) or - ("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" in [zeek][filetype]) or - ("application/vnd.openxmlformats-officedocument.spreadsheetml.template" in [zeek][filetype]) or - ("application/vnd.openxmlformats-officedocument.wordprocessingml.document" in [zeek][filetype]) or - ("application/vnd.openxmlformats-officedocument.wordprocessingml.template" in [zeek][filetype]) or - ("application/windows-library+xml" in [zeek][filetype]) or - ("application/x-7z-compressed" in [zeek][filetype]) or - ("application/x-ace-compressed" in [zeek][filetype]) or - ("application/x-apple-diskimage" in [zeek][filetype]) or - ("application/x-bzip" in [zeek][filetype]) or - ("application/x-bzip2" in [zeek][filetype]) or - ("application/x-cfs-compressed" in [zeek][filetype]) or - ("application/x-compress" in [zeek][filetype]) or - ("application/x-compressed" in [zeek][filetype]) or - ("application/x-cpio" in [zeek][filetype]) or - ("application/x-dgc-compressed" in [zeek][filetype]) or - ("application/x-gca-compressed" in [zeek][filetype]) or - ("application/x-gtar" in [zeek][filetype]) or - ("application/x-gzip" in [zeek][filetype]) or - ("application/x-install-instructions" in [zeek][filetype]) or - ("application/x-lzh-compressed" in [zeek][filetype]) or - ("application/x-ms-installer" in [zeek][filetype]) or - ("application/x-ms-shortcut" in [zeek][filetype]) or - ("application/x-msdownload" in [zeek][filetype]) or - ("application/x-rar-compressed" in [zeek][filetype]) or - ("application/x-shockwave-flash" in [zeek][filetype]) or - ("application/x-zip-compressed" in [zeek][filetype]) or - ("application/zip" in [zeek][filetype]) or - ("text/jscript" in [zeek][filetype]) or - ("text/rtf" in [zeek][filetype])) { - mutate { id => "mutate_add_field_severity_file_type_medium" - add_field => { "[event][severity_tags]" => "FILE_TYPE_MEDIUM" } } + # "high" severity files are executables and scripts (?) + if (("application/PowerShell" in [zeek][filetype]) or + ("application/vnd.microsoft.portable-executable" in [zeek][filetype]) or + ("application/x-csh" in [zeek][filetype]) or + ("application/x-dosexec" in [zeek][filetype]) or + ("application/x-elf" in [zeek][filetype]) or + ("application/x-executable" in [zeek][filetype]) or + ("application/x-ms-application" in [zeek][filetype]) or + ("application/x-msdos-program" in [zeek][filetype]) or + ("application/x-pe-app-32bit-i386" in [zeek][filetype]) or + ("application/x-perl" in [zeek][filetype]) or + ("application/x-python" in [zeek][filetype]) or + ("application/x-sh" in [zeek][filetype]) or + ("text/vbscript" in [zeek][filetype])) { + mutate { id => "mutate_add_field_severity_file_type_high" + add_field => { "[event][severity_tags]" => "FILE_TYPE_HIGH" } } - # "low" severity files are other file transfers - } else { - mutate { id => "mutate_add_field_severity_file_type" - add_field => { "[event][severity_tags]" => "FILE_TYPE" } } + # "medium" severity files are everything else in the "interesting" list + } else if (("application/binary" in [zeek][filetype]) or + ("application/ecmascript" in [zeek][filetype]) or + ("application/hta" in [zeek][filetype]) or + ("application/java-archive" in [zeek][filetype]) or + ("application/java-serialized-object" in [zeek][filetype]) or + ("application/java-vm" in [zeek][filetype]) or + ("application/javascript" in [zeek][filetype]) or + ("application/ms-vsi" in [zeek][filetype]) or + ("application/msaccess" in [zeek][filetype]) or + ("application/msaccess.addin" in [zeek][filetype]) or + ("application/msaccess.cab" in [zeek][filetype]) or + ("application/msaccess.ftemplate" in [zeek][filetype]) or + ("application/msaccess.runtime" in [zeek][filetype]) or + ("application/msaccess.webapplication" in [zeek][filetype]) or + ("application/msexcel" in [zeek][filetype]) or + ("application/mspowerpoint" in [zeek][filetype]) or + ("application/msword" in [zeek][filetype]) or + ("application/octet-stream" in [zeek][filetype]) or + ("application/pdf" in [zeek][filetype]) or + ("application/rtf" in [zeek][filetype]) or + ("application/vnd.apple.installer+xml" in [zeek][filetype]) or + ("application/vnd.ms-cab-compressed" in [zeek][filetype]) or + ("application/vnd.ms-excel" in [zeek][filetype]) or + ("application/vnd.ms-excel.addin.macroEnabled.12" in [zeek][filetype]) or + ("application/vnd.ms-excel.addin.macroenabled.12" in [zeek][filetype]) or + ("application/vnd.ms-excel.sheet.binary.macroEnabled.12" in [zeek][filetype]) or + ("application/vnd.ms-excel.sheet.binary.macroenabled.12" in [zeek][filetype]) or + ("application/vnd.ms-excel.sheet.macroEnabled.12" in [zeek][filetype]) or + ("application/vnd.ms-excel.sheet.macroenabled.12" in [zeek][filetype]) or + ("application/vnd.ms-excel.template.macroEnabled.12" in [zeek][filetype]) or + ("application/vnd.ms-excel.template.macroenabled.12" in [zeek][filetype]) or + ("application/vnd.ms-office.calx" in [zeek][filetype]) or + ("application/vnd.ms-officetheme" in [zeek][filetype]) or + ("application/vnd.ms-powerpoint" in [zeek][filetype]) or + ("application/vnd.ms-powerpoint.addin.macroEnabled.12" in [zeek][filetype]) or + ("application/vnd.ms-powerpoint.addin.macroenabled.12" in [zeek][filetype]) or + ("application/vnd.ms-powerpoint.presentation.macroEnabled.12" in [zeek][filetype]) or + ("application/vnd.ms-powerpoint.presentation.macroenabled.12" in [zeek][filetype]) or + ("application/vnd.ms-powerpoint.slide.macroEnabled.12" in [zeek][filetype]) or + ("application/vnd.ms-powerpoint.slide.macroenabled.12" in [zeek][filetype]) or + ("application/vnd.ms-powerpoint.slideshow.macroEnabled.12" in [zeek][filetype]) or + ("application/vnd.ms-powerpoint.slideshow.macroenabled.12" in [zeek][filetype]) or + ("application/vnd.ms-powerpoint.template.macroEnabled.12" in [zeek][filetype]) or + ("application/vnd.ms-powerpoint.template.macroenabled.12" in [zeek][filetype]) or + ("application/vnd.ms-word.document.macroEnabled.12" in [zeek][filetype]) or + ("application/vnd.ms-word.document.macroenabled.12" in [zeek][filetype]) or + ("application/vnd.ms-word.template.macroEnabled.12" in [zeek][filetype]) or + ("application/vnd.ms-word.template.macroenabled.12" in [zeek][filetype]) or + ("application/vnd.openofficeorg.extension" in [zeek][filetype]) or + ("application/vnd.openxmlformats-officedocument.presentationml.presentation" in [zeek][filetype]) or + ("application/vnd.openxmlformats-officedocument.presentationml.slide" in [zeek][filetype]) or + ("application/vnd.openxmlformats-officedocument.presentationml.slideshow" in [zeek][filetype]) or + ("application/vnd.openxmlformats-officedocument.presentationml.template" in [zeek][filetype]) or + ("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" in [zeek][filetype]) or + ("application/vnd.openxmlformats-officedocument.spreadsheetml.template" in [zeek][filetype]) or + ("application/vnd.openxmlformats-officedocument.wordprocessingml.document" in [zeek][filetype]) or + ("application/vnd.openxmlformats-officedocument.wordprocessingml.template" in [zeek][filetype]) or + ("application/windows-library+xml" in [zeek][filetype]) or + ("application/x-7z-compressed" in [zeek][filetype]) or + ("application/x-ace-compressed" in [zeek][filetype]) or + ("application/x-apple-diskimage" in [zeek][filetype]) or + ("application/x-bzip" in [zeek][filetype]) or + ("application/x-bzip2" in [zeek][filetype]) or + ("application/x-cfs-compressed" in [zeek][filetype]) or + ("application/x-compress" in [zeek][filetype]) or + ("application/x-compressed" in [zeek][filetype]) or + ("application/x-cpio" in [zeek][filetype]) or + ("application/x-dgc-compressed" in [zeek][filetype]) or + ("application/x-gca-compressed" in [zeek][filetype]) or + ("application/x-gtar" in [zeek][filetype]) or + ("application/x-gzip" in [zeek][filetype]) or + ("application/x-install-instructions" in [zeek][filetype]) or + ("application/x-lzh-compressed" in [zeek][filetype]) or + ("application/x-ms-installer" in [zeek][filetype]) or + ("application/x-ms-shortcut" in [zeek][filetype]) or + ("application/x-msdownload" in [zeek][filetype]) or + ("application/x-rar-compressed" in [zeek][filetype]) or + ("application/x-shockwave-flash" in [zeek][filetype]) or + ("application/x-zip-compressed" in [zeek][filetype]) or + ("application/zip" in [zeek][filetype]) or + ("text/jscript" in [zeek][filetype]) or + ("text/rtf" in [zeek][filetype])) { + mutate { id => "mutate_add_field_severity_file_type_medium" + add_field => { "[event][severity_tags]" => "FILE_TYPE_MEDIUM" } } + + # "low" severity files are other file transfers + } else { + mutate { id => "mutate_add_field_severity_file_type" + add_field => { "[event][severity_tags]" => "FILE_TYPE" } } + } } - } - # assign severity to notice based on category - if ([zeek_notice]) { - if ([zeek_notice][category] == "ATTACK") { - mutate { id => "mutate_add_field_severity_notice_mitre_attack" - add_field => { "[event][severity_tags]" => "NOTICE_MITRE_ATTACK" } } - } else if ([zeek_notice][category] == "Scan") { - mutate { id => "mutate_add_field_severity_notice_scan" - add_field => { "[event][severity_tags]" => "NOTICE_SCAN" } } - } else if (([zeek_notice][category] == "FTP") or - ([zeek_notice][category] == "HTTP") or - ([zeek_notice][category] == "HTTPATTACKS") or - ([zeek_notice][category] == "SSL")) { - mutate { id => "mutate_add_field_severity_notice_protocol" - add_field => { "[event][severity_tags]" => "NOTICE_PROTOCOL" } } - } else if (([zeek_notice][category] =~ /^CVE/) or - ([zeek_notice][category] == "EternalSafety") or - ([zeek_notice][category] == "Ripple20") or - ([zeek_notice][category] == "Zerologon")) { - mutate { id => "mutate_add_field_severity_notice_vuln" - add_field => { "[event][severity_tags]" => "NOTICE_VULN" } } - } else { - mutate { id => "mutate_add_field_severity_notice_other" - add_field => { "[event][severity_tags]" => "NOTICE_OTHER" } } + # assign severity to notice based on category + if ([zeek_notice]) { + if ([zeek_notice][category] == "ATTACK") { + mutate { id => "mutate_add_field_severity_notice_mitre_attack" + add_field => { "[event][severity_tags]" => "NOTICE_MITRE_ATTACK" } } + } else if ([zeek_notice][category] == "Scan") { + mutate { id => "mutate_add_field_severity_notice_scan" + add_field => { "[event][severity_tags]" => "NOTICE_SCAN" } } + } else if (([zeek_notice][category] == "FTP") or + ([zeek_notice][category] == "HTTP") or + ([zeek_notice][category] == "HTTPATTACKS") or + ([zeek_notice][category] == "SSL")) { + mutate { id => "mutate_add_field_severity_notice_protocol" + add_field => { "[event][severity_tags]" => "NOTICE_PROTOCOL" } } + } else if (([zeek_notice][category] =~ /^CVE/) or + ([zeek_notice][category] == "EternalSafety") or + ([zeek_notice][category] == "Ripple20") or + ([zeek_notice][category] == "Zerologon")) { + mutate { id => "mutate_add_field_severity_notice_vuln" + add_field => { "[event][severity_tags]" => "NOTICE_VULN" } } + } else { + mutate { id => "mutate_add_field_severity_notice_other" + add_field => { "[event][severity_tags]" => "NOTICE_OTHER" } } + } } - } - # weird logs get one score at the moment - if ([zeek_weird]) { - mutate { id => "mutate_add_field_severity_weird_other" - add_field => { "[event][severity_tags]" => "WEIRD_OTHER" } } - } + # weird logs get one score at the moment + if ([zeek_weird]) { + mutate { id => "mutate_add_field_severity_weird_other" + add_field => { "[event][severity_tags]" => "WEIRD_OTHER" } } + } - # if zeek.password exists, it's assumed to be in cleartext - if ([zeek][password]) { - mutate { id => "mutate_add_field_severity_password_exists" - add_field => { "[event][severity_tags]" => "PASSWORD_CLEARTEXT" } } - } + # if zeek.password exists, it's assumed to be in cleartext + if ([zeek][password]) { + mutate { id => "mutate_add_field_severity_password_exists" + add_field => { "[event][severity_tags]" => "PASSWORD_CLEARTEXT" } } + } - # assign score to signatures based on engine - if ([zeek_signatures]) { - if ([zeek_signatures][engine] == 'capa') { - mutate { id => "mutate_add_field_severity_signatures_capa" - add_field => { "[event][severity_tags]" => "SIGNATURES_CAPA" } } - } else if ([zeek_signatures][engine] == 'clamav') { - mutate { id => "mutate_add_field_severity_signatures_clamav" - add_field => { "[event][severity_tags]" => "SIGNATURES_CLAMAV" } } - } else if ([zeek_signatures][engine] == 'malass') { - mutate { id => "mutate_add_field_severity_signatures_malass" - add_field => { "[event][severity_tags]" => "SIGNATURES_MALASS" } } - } else if ([zeek_signatures][engine] == 'virustotal') { - mutate { id => "mutate_add_field_severity_signatures_virustotal" - add_field => { "[event][severity_tags]" => "SIGNATURES_VIRUSTOTAL" } } - } else if ([zeek_signatures][engine] == 'yara') { - mutate { id => "mutate_add_field_severity_signatures_yara" - add_field => { "[event][severity_tags]" => "SIGNATURES_YARA" } } - } else { - mutate { id => "mutate_add_field_severity_signatures_other" - add_field => { "[event][severity_tags]" => "SIGNATURES_OTHER" } } + # check for high-entropy domain names for non-internal connections + if ([zeek][freq_score_v1]) or ([zeek][freq_score_v2]) { + ruby { + id => "ruby_add_field_severity_domain_entropy" + init => "$freqSeverityThreshold = ENV['FREQ_SEVERITY_THRESHOLD'] || '3.0'" + code => " + freqs1 = event.get('[zeek][freq_score_v1]') + freqs2 = event.get('[zeek][freq_score_v2]') + lowestFreqScore = [freqs1.nil? ? 100 : (freqs1.kind_of?(Array) ? freqs1.min : freqs1), + freqs2.nil? ? 100 : (freqs2.kind_of?(Array) ? freqs2.min : freqs2)].min + if (lowestFreqScore < Float($freqSeverityThreshold)) then + sevtags = Array.new unless (sevtags = event.get('[event][severity_tags]')) + if !sevtags.kind_of?(Array) then + newtags = Array.new + newtags.push(sevtags) + sevtags = newtags + end + sevtags.push('DOMAIN_HIGH_ENTROPY') + event.set('[event][severity_tags]', sevtags) + end" + } } - } - # insecure/outdated protocols requires a little bit more code than logstash can do - # so we'll do it in Ruby. at the moment these are mirroring what's in the - # Security Overview (95479950-41f2-11ea-88fa-7151df485405) dashboard, so they - # probably should be kept up-to-date with that. - # - # in addition to insecure/outdated protocols, append PROTOCOL_XXX to the - # severity_tags list, where XXX is the uppercased value of zeek.service, - # so that if the user wants to add PROTOCOL_SSH or whatever to their mapping - # for a custom severity value it will just work. if it's not in their - # mapping file, it'll return with a nil value and will be removed - # in ruby_calculate_final_severity_score - if (!([zeek][logType] =~ /^known/)) { - ruby { - id => "ruby_add_field_severity_insecure_protocols" - code => " - if (service = event.get('[zeek][service]')) then - found = 0 - if !service.kind_of?(Array) then - newsrv = Array.new - newsrv.push(service) - service = newsrv - end - version = event.get('[zeek][service_version]') - if !version.nil? and !version.kind_of?(Array) then - newver = Array.new - newver.push(version) - version = newver - end - service.each do |srv| - if found > 0 then - break + # assign score to signatures based on engine + if ([zeek_signatures]) { + if ([zeek_signatures][engine] == 'capa') { + mutate { id => "mutate_add_field_severity_signatures_capa" + add_field => { "[event][severity_tags]" => "SIGNATURES_CAPA" } } + } else if ([zeek_signatures][engine] == 'clamav') { + mutate { id => "mutate_add_field_severity_signatures_clamav" + add_field => { "[event][severity_tags]" => "SIGNATURES_CLAMAV" } } + } else if ([zeek_signatures][engine] == 'malass') { + mutate { id => "mutate_add_field_severity_signatures_malass" + add_field => { "[event][severity_tags]" => "SIGNATURES_MALASS" } } + } else if ([zeek_signatures][engine] == 'virustotal') { + mutate { id => "mutate_add_field_severity_signatures_virustotal" + add_field => { "[event][severity_tags]" => "SIGNATURES_VIRUSTOTAL" } } + } else if ([zeek_signatures][engine] == 'yara') { + mutate { id => "mutate_add_field_severity_signatures_yara" + add_field => { "[event][severity_tags]" => "SIGNATURES_YARA" } } + } else { + mutate { id => "mutate_add_field_severity_signatures_other" + add_field => { "[event][severity_tags]" => "SIGNATURES_OTHER" } } + } + } + + # insecure/outdated protocols requires a little bit more code than logstash can do + # so we'll do it in Ruby. at the moment these are mirroring what's in the + # Security Overview (95479950-41f2-11ea-88fa-7151df485405) dashboard, so they + # probably should be kept up-to-date with that. + # + # in addition to insecure/outdated protocols, append PROTOCOL_XXX to the + # severity_tags list, where XXX is the uppercased value of zeek.service, + # so that if the user wants to add PROTOCOL_SSH or whatever to their mapping + # for a custom severity value it will just work. if it's not in their + # mapping file, it'll return with a nil value and will be removed + # in ruby_calculate_final_severity_score + if (!([zeek][logType] =~ /^known/)) { + ruby { + id => "ruby_add_field_severity_insecure_protocols" + code => " + if (service = event.get('[zeek][service]')) then + found = 0 + if !service.kind_of?(Array) then + newsrv = Array.new + newsrv.push(service) + service = newsrv end - if ['ftp', 'tftp', 'telnet', 'rlogin', 'rsh'].include?(srv) then - found += 1 - elsif !version.nil? then - version.each do |ver| - if found > 0 then - break - end - if (srv.eql?('ssh') and (ver.to_f < 2.0)) or - (srv.eql?('smb') and (ver.to_f < 2.0)) or - (srv.eql?('ntp') and (ver.to_f < 4.0)) or - (srv.eql?('rfb') and (ver.to_f < 3.8)) or - (srv.eql?('rdp') and (ver.to_f < 6.0)) or - (srv.eql?('snmp') and (ver.to_f < 3.0)) or - (srv.eql?('ldap') and (ver.to_f < 3.0)) or - (srv.eql?('tls') and (!(ver.match(/TLS.*v?1\.?[23]/)))) then - found += 1 + version = event.get('[zeek][service_version]') + if !version.nil? and !version.kind_of?(Array) then + newver = Array.new + newver.push(version) + version = newver + end + service.each do |srv| + if found > 0 then + break + end + if ['ftp', 'tftp', 'telnet', 'rlogin', 'rsh'].include?(srv) then + found += 1 + elsif !version.nil? then + version.each do |ver| + if found > 0 then + break + end + if (srv.eql?('ssh') and (ver.to_f < 2.0)) or + (srv.eql?('smb') and (ver.to_f < 2.0)) or + (srv.eql?('ntp') and (ver.to_f < 4.0)) or + (srv.eql?('rfb') and (ver.to_f < 3.8)) or + (srv.eql?('rdp') and (ver.to_f < 6.0)) or + (srv.eql?('snmp') and (ver.to_f < 3.0)) or + (srv.eql?('ldap') and (ver.to_f < 3.0)) or + (srv.eql?('tls') and (!(ver.match(/TLS.*v?1\.?[23]/)))) then + found += 1 + end end end end + sevtags = Array.new unless (sevtags = event.get('[event][severity_tags]')) + if !sevtags.kind_of?(Array) then + newtags = Array.new + newtags.push(sevtags) + sevtags = newtags + end + if found > 0 then + sevtags.push('PROTOCOL_OUTDATED_OR_INSECURE') + end + sevtags.push(*service.map{|x| 'PROTOCOL_' + x.upcase}) + event.set('[event][severity_tags]', sevtags) + end" + } + } + + ruby { + id => "ruby_calculate_final_severity_score" + # pre-load severity score mapping in init outside of processing pipeline + init => " + require 'yaml'; $severityMappings = YAML.load(File.read('/etc/malcolm_severity.yaml')) + " + # to calculate severity: + # - look up list of severity_tags against severity score mapping (generate hash), ignoring <= 0 or missing (nil) values + # - set summed severity scores in event.severity/risk_score/risk_score_norm + # - set pruned severity_tags back to event.severity_tags + code => " + sevtags = event.get('[event][severity_tags]') + if !sevtags.nil? and !sevtags.kind_of?(Array) then + tagsarr = Array.new + tagsarr.push(sevtags) + sevtags = tagsarr + end + if !sevtags.nil? and (sevtags.length > 0) and ($severityMappings.length > 0) then + sevhash = Hash[sevtags.select{|key| ($severityMappings[key].nil? ? 0 : $severityMappings[key]) > 0}.map{|key| [key, $severityMappings[key]]}] + sevnum = [sevhash.values.sum, 100].min + if (sevnum > 0) then + event.set('[event][severity]', sevnum) + event.set('[event][severity_tags]', sevhash.keys) + else + event.remove('[event][severity_tags]') end - sevtags = Array.new unless (sevtags = event.get('[event][severity_tags]')) - if !sevtags.kind_of?(Array) then - newtags = Array.new - newtags.push(sevtags) - sevtags = newtags - end - if found > 0 then - sevtags.push('PROTOCOL_OUTDATED_OR_INSECURE') - end - sevtags.push(*service.map{|x| 'PROTOCOL_' + x.upcase}) - event.set('[event][severity_tags]', sevtags) end" } - } - ruby { - id => "ruby_calculate_final_severity_score" - # pre-load severity score mapping in init outside of processing pipeline - init => " - require 'yaml'; $severityMappings = YAML.load(File.read('/etc/malcolm_severity.yaml')) - " - # to calculate severity: - # - look up list of severity_tags against severity score mapping (generate hash), ignoring <= 0 or missing (nil) values - # - set summed severity scores in event.severity/risk_score/risk_score_norm - # - set pruned severity_tags back to event.severity_tags - code => " - sevtags = event.get('[event][severity_tags]') - if !sevtags.nil? and !sevtags.kind_of?(Array) then - tagsarr = Array.new - tagsarr.push(sevtags) - sevtags = tagsarr - end - if !sevtags.nil? and (sevtags.length > 0) and ($severityMappings.length > 0) then - sevhash = Hash[sevtags.select{|key| ($severityMappings[key].nil? ? 0 : $severityMappings[key]) > 0}.map{|key| [key, $severityMappings[key]]}] - sevnum = [sevhash.values.sum, 100].min - event.set('[event][severity_tags]', sevhash.keys) - event.set('[event][severity]', sevnum) - end" - } + } # LOGSTASH_SEVERITY_SCORING env check } # filter - diff --git a/moloch/etc/config.ini b/moloch/etc/config.ini index ec9545245..2cac798ad 100644 --- a/moloch/etc/config.ini +++ b/moloch/etc/config.ini @@ -71,6 +71,8 @@ zeek.community_id=db:zeek.community_id;kind:termfield;friendly:Zeek Connection C zeek.logType=db:zeek.logType;kind:termfield;friendly:Zeek Log Type;help:Zeek Log Type zeek.ts=db:zeek.ts;kind:termfield;friendly:Timestamp;help:Zeek Timestamp host.name=db:host.name;kind:termfield;friendly:Zeek Node;help:Zeek Node +event.severity=db:event.severity;kind:integer;friendly:Severity;help:Severity +event.severity_tags=db:event.severity_tags;kind:termfield;friendly:Severity Tags;help:Severity Tags # basic connection information zeek.orig_h=db:zeek.orig_h;kind:termfield;friendly:Originating host;help:Originating Host diff --git a/moloch/wise/source.zeeklogs.js b/moloch/wise/source.zeeklogs.js index 5d2c873fb..47b8c186c 100644 --- a/moloch/wise/source.zeeklogs.js +++ b/moloch/wise/source.zeeklogs.js @@ -27,6 +27,8 @@ function ZeekLogs (api, section) { // add right-clicks var allFields = [ "communityId", + "event.severity", + "event.severity_tags", "host.name", "ip.protocol", "mac.dst", @@ -950,6 +952,8 @@ function ZeekLogs (api, section) { " +arrayList(session.zeek, 'password', 'Password', 'zeek.password')\n" + " +arrayList(session.zeek, 'freq_score_v1', 'Freq Score v1', 'zeek.freq_score_v1')\n" + " +arrayList(session.zeek, 'freq_score_v2', 'Freq Score v2', 'zeek.freq_score_v2')\n" + + " +arrayList(session.event, 'severity', 'Severity', 'event.severity')\n" + + " +arrayList(session.event, 'severity_tags', 'Severity Tags', 'event.severity_tags')\n" + // file information " if (session.zeek.fuid || session.zeek.filename || session.zeek.filetype)\n" +