Cherry-pick #23772 to 7.x: [Filebeat] Add Zeek Signatures fileset

CHANGELOG.next.asciidoc

@@ -536,6 +536,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Move aws-s3 input to GA. {pull}23631[23631]
 - Populate `source.mac` and `destination.mac` for Suricata EVE events. {issue}23706[23706] {pull}23721[23721]
 - Added string splitting for httpjson input {pull}24022[24022]
+- Added Signatures fileset to Zeek module {pull}23772[23772]
 
 *Heartbeat*
 

filebeat/docs/fields.asciidoc

@@ -153290,6 +153290,73 @@ type: integer
 Height of the screen that is being shared.
 
 
+type: integer
+
+--
+
+[float]
+=== signature
+
+Fields exported by the Zeek Signature log.
+
+
+
+*`zeek.signature.note`*::
++
+--
+Notice associated with signature event.
+
+
+type: keyword
+
+--
+
+*`zeek.signature.sig_id`*::
++
+--
+The name of the signature that matched.
+
+
+type: keyword
+
+--
+
+*`zeek.signature.event_msg`*::
++
+--
+A more descriptive message of the signature-matching event.
+
+
+type: keyword
+
+--
+
+*`zeek.signature.sub_msg`*::
++
+--
+Extracted payload data or extra message.
+
+
+type: keyword
+
+--
+
+*`zeek.signature.sig_count`*::
++
+--
+Number of sigs, usually from summary count.
+
+
+type: integer
+
+--
+
+*`zeek.signature.host_count`*::
++
+--
+Number of hosts, from a summary count.
+
+
 type: integer
 
 --

x-pack/filebeat/filebeat.reference.yml

@@ -2138,7 +2138,7 @@ filebeat.modules:
   http:
     enabled: true
   intel:
-    enabled: true    
+    enabled: true
   irc:
     enabled: true
   kerberos:
@@ -2161,6 +2161,8 @@ filebeat.modules:
     enabled: true
   rfb:
     enabled: true
+  signature:
+    enabled: true
   sip:
     enabled: true
   smb_cmd:

x-pack/filebeat/module/zeek/_meta/config.yml

@@ -20,7 +20,7 @@
   http:
     enabled: true
   intel:
-    enabled: true    
+    enabled: true
   irc:
     enabled: true
   kerberos:
@@ -43,6 +43,8 @@
     enabled: true
   rfb:
     enabled: true
+  signature:
+    enabled: true
   sip:
     enabled: true
   smb_cmd:

x-pack/filebeat/module/zeek/signature/_meta/fields.yml

@@ -0,0 +1,35 @@
+- name: signature
+  type: group
+  default_field: false
+  description: >
+    Fields exported by the Zeek Signature log.
+  fields:
+    - name: note
+      type: keyword
+      description: >
+        Notice associated with signature event.
+
+    - name: sig_id
+      type: keyword
+      description: >
+        The name of the signature that matched.
+
+    - name: event_msg
+      type: keyword
+      description: >
+        A more descriptive message of the signature-matching event.
+
+    - name: sub_msg
+      type: keyword
+      description: >
+        Extracted payload data or extra message.
+
+    - name: sig_count
+      type: integer
+      description: >
+        Number of sigs, usually from summary count.
+
+    - name: host_count
+      type: integer
+      description: >
+        Number of hosts, from a summary count.

x-pack/filebeat/module/zeek/signature/config/signature.yml

@@ -0,0 +1,50 @@
+type: log
+paths:
+{{ range $i, $path := .paths }}
+ - {{$path}}
+{{ end }}
+exclude_files: [".gz$"]
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
+
+processors:
+  - rename:
+      fields:
+        - {from: message, to: event.original}
+  - decode_json_fields:
+      fields: [event.original]
+      target: zeek.signature
+  - convert:
+      ignore_missing: true
+      fields:
+        - {from: zeek.signature.src_addr, to: source.address}
+        - {from: zeek.signature.src_addr, to: source.ip, type: ip}
+        - {from: zeek.signature.src_port, to: source.port, type: long}
+        - {from: zeek.signature.dst_addr, to: destination.address}
+        - {from: zeek.signature.dst_addr, to: destination.ip, type: ip}
+        - {from: zeek.signature.dst_port, to: destination.port, type: long}
+  - rename:
+      ignore_missing: true
+      fields:
+        - from: zeek.signature.uid
+          to: zeek.session_id
+        - from: zeek.signature.sig_id
+          to: rule.id
+        - from: zeek.signature.event_msg
+          to: rule.description
+  - drop_fields:
+      ignore_missing: true
+      fields:
+        - zeek.signature.src_addr
+        - zeek.signature.src_port
+        - zeek.signature.dst_addr
+        - zeek.signature.dst_port
+  - add_fields:
+      target: event
+      fields:
+        kind: alert
+  - community_id:
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.7.0

x-pack/filebeat/module/zeek/signature/ingest/pipeline.yml

@@ -0,0 +1,89 @@
+---
+description: Pipeline for normalizing Zeek signature.log.
+processors:
+  - set:
+      field: event.ingested
+      value: '{{_ingest.timestamp}}'
+  - set:
+      field: event.created
+      value: '{{@timestamp}}'
+  - date:
+      field: zeek.signature.ts
+      formats:
+        - UNIX
+  - remove:
+      field: zeek.signature.ts
+  # IP Geolocation Lookup
+  - geoip:
+      if: ctx.source?.geo == null
+      field: source.ip
+      target_field: source.geo
+      ignore_missing: true
+      properties:
+        - city_name
+        - continent_name
+        - country_iso_code
+        - country_name
+        - location
+        - region_iso_code
+        - region_name
+  - geoip:
+      if: ctx.destination?.geo == null
+      field: destination.ip
+      target_field: destination.geo
+      ignore_missing: true
+      properties:
+        - city_name
+        - continent_name
+        - country_iso_code
+        - country_name
+        - location
+        - region_iso_code
+        - region_name
+
+  # IP Autonomous System (AS) Lookup
+  - geoip:
+      database_file: GeoLite2-ASN.mmdb
+      field: source.ip
+      target_field: source.as
+      properties:
+        - asn
+        - organization_name
+      ignore_missing: true
+  - geoip:
+      database_file: GeoLite2-ASN.mmdb
+      field: destination.ip
+      target_field: destination.as
+      properties:
+        - asn
+        - organization_name
+      ignore_missing: true
+  - rename:
+      field: source.as.asn
+      target_field: source.as.number
+      ignore_missing: true
+  - rename:
+      field: source.as.organization_name
+      target_field: source.as.organization.name
+      ignore_missing: true
+  - rename:
+      field: destination.as.asn
+      target_field: destination.as.number
+      ignore_missing: true
+  - rename:
+      field: destination.as.organization_name
+      target_field: destination.as.organization.name
+      ignore_missing: true
+  - append:
+      field: "related.ip"
+      value: "{{source.ip}}"
+      if: "ctx?.source?.ip != null"
+  - append:
+      field: "related.ip"
+      value: "{{destination.ip}}"
+      if: "ctx?.destination?.ip != null"
+
+on_failure:
+  - set:
+      field: error.message
+      value: "{{ _ingest.on_failure_message }}"

x-pack/filebeat/module/zeek/signature/manifest.yml

@@ -0,0 +1,19 @@
+module_version: 1.0
+
+var:
+  - name: paths
+    default:
+      - /var/log/bro/current/signatures.log
+    os.linux:
+      - /var/log/bro/current/signatures.log
+    os.darwin:
+      - /usr/local/var/logs/current/signatures.log
+  - name: tags
+    default: [zeek.signature]
+
+ingest_pipeline: ingest/pipeline.yml
+input: config/signature.yml
+
+requires.processors:
+- name: geoip
+  plugin: ingest-geoip

x-pack/filebeat/module/zeek/signature/test/signature-json.log

@@ -0,0 +1 @@
+{"ts": 1611852809.869245,"uid": "CbjAXE4CBxJ8W7VoJg","src_addr": "124.51.137.154","src_port": 51617,"dst_addr": "160.218.27.63","dst_port": 445,"note": "Signatures::Sensitive_Signature","sig_id": "my-second-sig","event_msg": "124.51.137.154: TCP traffic","sub_msg": ""}

x-pack/filebeat/module/zeek/signature/test/signature-json.log-expected.json

@@ -0,0 +1,48 @@
+[
+    {
+        "@timestamp": "2021-01-28T16:53:29.869Z",
+        "destination.address": "160.218.27.63",
+        "destination.as.number": 5610,
+        "destination.as.organization.name": "O2 Czech Republic, a.s.",
+        "destination.geo.continent_name": "Europe",
+        "destination.geo.country_iso_code": "CZ",
+        "destination.geo.country_name": "Czechia",
+        "destination.geo.location.lat": 50.0848,
+        "destination.geo.location.lon": 14.4112,
+        "destination.ip": "160.218.27.63",
+        "destination.port": 445,
+        "event.dataset": "zeek.signature",
+        "event.kind": "alert",
+        "event.module": "zeek",
+        "event.original": "{\"ts\": 1611852809.869245,\"uid\": \"CbjAXE4CBxJ8W7VoJg\",\"src_addr\": \"124.51.137.154\",\"src_port\": 51617,\"dst_addr\": \"160.218.27.63\",\"dst_port\": 445,\"note\": \"Signatures::Sensitive_Signature\",\"sig_id\": \"my-second-sig\",\"event_msg\": \"124.51.137.154: TCP traffic\",\"sub_msg\": \"\"}",
+        "fileset.name": "signature",
+        "input.type": "log",
+        "log.offset": 0,
+        "related.ip": [
+            "124.51.137.154",
+            "160.218.27.63"
+        ],
+        "rule.description": "124.51.137.154: TCP traffic",
+        "rule.id": "my-second-sig",
+        "service.type": "zeek",
+        "source.address": "124.51.137.154",
+        "source.as.number": 17858,
+        "source.as.organization.name": "LG POWERCOMM",
+        "source.geo.city_name": "Busan",
+        "source.geo.continent_name": "Asia",
+        "source.geo.country_iso_code": "KR",
+        "source.geo.country_name": "South Korea",
+        "source.geo.location.lat": 35.1003,
+        "source.geo.location.lon": 129.0442,
+        "source.geo.region_iso_code": "KR-26",
+        "source.geo.region_name": "Busan",
+        "source.ip": "124.51.137.154",
+        "source.port": 51617,
+        "tags": [
+            "zeek.signature"
+        ],
+        "zeek.session_id": "CbjAXE4CBxJ8W7VoJg",
+        "zeek.signature.note": "Signatures::Sensitive_Signature",
+        "zeek.signature.sub_msg": ""
+    }
+]

x-pack/filebeat/modules.d/zeek.yml.disabled

@@ -23,7 +23,7 @@
   http:
     enabled: true
   intel:
-    enabled: true    
+    enabled: true
   irc:
     enabled: true
   kerberos:
@@ -46,6 +46,8 @@
     enabled: true
   rfb:
     enabled: true
+  signature:
+    enabled: true
   sip:
     enabled: true
   smb_cmd: