Cherry-pick #23072 to 7.x: [Filebeat] Add fortinet/firewall network d…

…irection override based on interface (#23085) * [Filebeat] Add fortinet/firewall network direction override based on interface (#23072) * [Filebeat] Add fortinet/firewall network direction override based on interface * Add changelog entry * Don't override categorization if no interface name is set (cherry picked from commit 5a03647) * Fix up changelog
elastic · Dec 11, 2020 · a3f90fc · a3f90fc
1 parent 73cc6a0
commit a3f90fc
Show file tree

Hide file tree

Showing 10 changed files with 157 additions and 8 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -235,6 +235,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Drop aws.vpcflow.pkt_srcaddr and aws.vpcflow.pkt_dstaddr when equal to "-". {pull}22721[22721] {issue}22716[22716]
 - Fix cisco umbrella module config by adding input variable. {pull}22892[22892]
 - Fix network.direction logic in zeek connection fileset. {pull}22967[22967]
+- Fix bad `network.direction` values in Fortinet/firewall fileset. {pull}23072[23072]
 
 *Heartbeat*
 
@@ -505,6 +506,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Allow cisco/asa and cisco/ftd filesets to override network directionality based off of zones. {pull}23068[23068]
 - Allow cef and checkpoint modules to override network directionality based off of zones {pull}23066[23066]
 - Add `network.direction` to netflow/log fileset. {pull}23052[23052]
+- Add the ability to override `network.direction` based on interfaces in Fortinet/firewall fileset. {pull}23072[23072]
 
 *Heartbeat*
 

diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml
@@ -798,6 +798,16 @@ filebeat.modules:
     # The port to listen for syslog traffic. Defaults to 9004.
     #var.syslog_port: 9004
 
+    # Set internal interfaces. used to override parsed network.direction
+    # based on a tagged interface. Both internal and external interfaces must be
+    # set to leverage this functionality.
+    #var.internal_interfaces: [ "LAN" ]
+
+    # Set external interfaces. used to override parsed network.direction
+    # based on a tagged interface. Both internal and external interfaces must be
+    # set to leverage this functionality.
+    #var.external_interfaces: [ "WAN" ]
+
   clientendpoint:
     enabled: true
 

diff --git a/x-pack/filebeat/module/fortinet/_meta/config.yml b/x-pack/filebeat/module/fortinet/_meta/config.yml
@@ -12,6 +12,16 @@
     # The port to listen for syslog traffic. Defaults to 9004.
     #var.syslog_port: 9004
 
+    # Set internal interfaces. used to override parsed network.direction
+    # based on a tagged interface. Both internal and external interfaces must be
+    # set to leverage this functionality.
+    #var.internal_interfaces: [ "LAN" ]
+
+    # Set external interfaces. used to override parsed network.direction
+    # based on a tagged interface. Both internal and external interfaces must be
+    # set to leverage this functionality.
+    #var.external_interfaces: [ "WAN" ]
+
   clientendpoint:
     enabled: true
 

diff --git a/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml b/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml
@@ -28,3 +28,17 @@ processors:
       target: ''
       fields:
         ecs.version: 1.7.0
+
+{{ if .external_interfaces }}
+  - add_fields:
+      target: _temp
+      fields:
+        external_interfaces: {{ .external_interfaces | tojson }}
+{{ end }}
+
+{{ if .internal_interfaces }}
+  - add_fields:
+      target: _temp
+      fields:
+        internal_interfaces: {{ .internal_interfaces | tojson }}
+{{ end }}
diff --git a/x-pack/filebeat/module/fortinet/firewall/ingest/event.yml b/x-pack/filebeat/module/fortinet/firewall/ingest/event.yml
@@ -61,7 +61,7 @@ processors:
     field: fortinet.firewall.remip
     target_field: destination.ip
     ignore_missing: true
-    if: "ctx.destination?.ip == null"  
+    if: "ctx.destination?.ip == null"
 - convert:
     field: fortinet.firewall.dstport
     target_field: destination.port
@@ -183,11 +183,30 @@ processors:
     field: fortinet.firewall.dir
     target_field: network.direction
     ignore_missing: true
+    if: 'ctx.network?.direction == null'
 - rename:
     field: fortinet.firewall.direction
     target_field: network.direction
     ignore_missing: true
     if: "ctx.network?.direction == null"
+# Normalize the network direction
+- script:
+    lang: painless
+    ignore_failure: true
+    params:
+      outgoing: outbound
+      incoming: inbound
+    source: >-
+      if (ctx.network?.direction == null) {
+        return;
+      }
+      def k = ctx.network?.direction.toLowerCase();
+      def normalized = params.get(k);
+      if (normalized != null) {
+        ctx.network.direction = normalized;
+        return
+      }
+      ctx.network.direction = k;
 - rename:
     field: fortinet.firewall.service
     target_field: network.protocol
@@ -320,8 +339,10 @@ processors:
     - fortinet.firewall.locport
     - fortinet.firewall.filesize
     - fortinet.firewall.sess_duration
+    - fortinet.firewall.dir
+    - fortinet.firewall.direction
     ignore_missing: true
 on_failure:
 - set:
     field: error.message
-    value: '{{ _ingest.on_failure_message }}'
+    value: '{{ _ingest.on_failure_message }}'
diff --git a/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml b/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml
@@ -161,6 +161,65 @@ processors:
 - remove:
     field: fortinet.firewall.tunnelip
     if: "ctx.fortinet?.firewall?.tunnelip == 'N/A'"
+# Handle interface-based network directionality
+- set:
+    field: network.direction
+    value: inbound
+    if: >
+      ctx?._temp?.external_interfaces != null &&
+      ctx?._temp?.internal_interfaces != null &&
+      ctx?.observer?.ingress?.interface?.name != null &&
+      ctx?.observer?.egress?.interface?.name != null &&
+      ctx._temp.external_interfaces.contains(ctx.observer.ingress.interface.name) &&
+      ctx._temp.internal_interfaces.contains(ctx.observer.egress.interface.name)
+- set:
+    field: network.direction
+    value: outbound
+    if: >
+      ctx?._temp?.external_interfaces != null &&
+      ctx?._temp?.internal_interfaces != null &&
+      ctx?.observer?.ingress?.interface?.name != null &&
+      ctx?.observer?.egress?.interface?.name != null &&
+      ctx._temp.external_interfaces.contains(ctx.observer.egress.interface.name) &&
+      ctx._temp.internal_interfaces.contains(ctx.observer.ingress.interface.name)
+- set:
+    field: network.direction
+    value: internal
+    if: >
+      ctx?._temp?.external_interfaces != null &&
+      ctx?._temp?.internal_interfaces != null &&
+      ctx?.observer?.ingress?.interface?.name != null &&
+      ctx?.observer?.egress?.interface?.name != null &&
+      ctx._temp.internal_interfaces.contains(ctx.observer.egress.interface.name) &&
+      ctx._temp.internal_interfaces.contains(ctx.observer.ingress.interface.name)
+- set:
+    field: network.direction
+    value: external
+    if: >
+      ctx?._temp?.external_interfaces != null &&
+      ctx?._temp?.internal_interfaces != null &&
+      ctx?.observer?.ingress?.interface?.name != null &&
+      ctx?.observer?.egress?.interface?.name != null &&
+      ctx._temp.external_interfaces.contains(ctx.observer.egress.interface.name) &&
+      ctx._temp.external_interfaces.contains(ctx.observer.ingress.interface.name)
+- set:
+    field: network.direction
+    value: unknown
+    if: >
+      ctx?._temp?.external_interfaces != null &&
+      ctx?._temp?.internal_interfaces != null &&
+      ctx?.observer?.egress?.interface?.name != null &&
+      ctx?.observer?.ingress?.interface?.name != null &&
+      (
+        (
+          !ctx._temp.external_interfaces.contains(ctx.observer.egress.interface.name) &&
+          !ctx._temp.internal_interfaces.contains(ctx.observer.egress.interface.name)
+        ) ||
+        (
+          !ctx._temp.external_interfaces.contains(ctx.observer.ingress.interface.name) &&
+          !ctx._temp.internal_interfaces.contains(ctx.observer.ingress.interface.name)
+        )
+      )
 - remove:
     field:
     - _temp

diff --git a/x-pack/filebeat/module/fortinet/firewall/ingest/utm.yml b/x-pack/filebeat/module/fortinet/firewall/ingest/utm.yml
@@ -157,11 +157,30 @@ processors:
     field: fortinet.firewall.dir
     target_field: network.direction
     ignore_missing: true
+    if: 'ctx.network?.direction == null'
 - rename:
     field: fortinet.firewall.direction
     target_field: network.direction
     ignore_missing: true
     if: "ctx.network?.direction == null"
+# Normalize the network direction
+- script:
+    lang: painless
+    ignore_failure: true
+    params:
+      outgoing: outbound
+      incoming: inbound
+    source: >-
+      if (ctx.network?.direction == null) {
+        return;
+      }
+      def k = ctx.network?.direction.toLowerCase();
+      def normalized = params.get(k);
+      if (normalized != null) {
+        ctx.network.direction = normalized;
+        return
+      }
+      ctx.network.direction = k;
 - rename:
     field: fortinet.firewall.error
     target_field: event.message
@@ -431,6 +450,8 @@ processors:
     - fortinet.firewall.srcport
     - fortinet.firewall.sentbyte
     - fortinet.firewall.filesize
+    - fortinet.firewall.dir
+    - fortinet.firewall.direction
     ignore_missing: true
 on_failure:
 - set:

diff --git a/x-pack/filebeat/module/fortinet/firewall/manifest.yml b/x-pack/filebeat/module/fortinet/firewall/manifest.yml
@@ -9,6 +9,8 @@ var:
     default: 9004
   - name: input
     default: udp
+  - name: internal_interfaces
+  - name: external_interfaces
 
 ingest_pipeline:
   - ingest/pipeline.yml

diff --git a/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json b/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json
@@ -42,7 +42,7 @@
         "log.offset": 0,
         "message": "URL belongs to a denied category in policy",
         "network.bytes": 2282,
-        "network.direction": "outgoing",
+        "network.direction": "outbound",
         "network.iana_number": "6",
         "network.protocol": "https",
         "observer.egress.interface.name": "wan1",
@@ -189,7 +189,7 @@
         "log.offset": 1278,
         "message": "URL belongs to an allowed category in policy",
         "network.bytes": 10357,
-        "network.direction": "outgoing",
+        "network.direction": "outbound",
         "network.iana_number": "6",
         "network.protocol": "https",
         "observer.egress.interface.name": "wan1",
@@ -264,7 +264,7 @@
         "log.offset": 1980,
         "message": "Web.Client: HTTPS.BROWSER,",
         "network.application": "HTTPS.BROWSER",
-        "network.direction": "outgoing",
+        "network.direction": "outbound",
         "network.iana_number": "6",
         "network.protocol": "ssl",
         "observer.egress.interface.name": "wan1",
@@ -339,7 +339,7 @@
         "log.offset": 2683,
         "message": "Web.Client: HTTPS.BROWSER,",
         "network.application": "HTTPS.BROWSER",
-        "network.direction": "outgoing",
+        "network.direction": "outbound",
         "network.iana_number": "6",
         "network.protocol": "ssl",
         "observer.egress.interface.name": "wan1",
@@ -555,7 +555,7 @@
         "log.offset": 4486,
         "message": "Web.Client: HTTPS.BROWSER,",
         "network.application": "HTTPS.BROWSER",
-        "network.direction": "outgoing",
+        "network.direction": "outbound",
         "network.iana_number": "6",
         "network.protocol": "ssl",
         "observer.egress.interface.name": "wan1",
@@ -1941,7 +1941,7 @@
         "log.offset": 16463,
         "message": "Web.Client: HTTPS.BROWSER,",
         "network.application": "HTTPS.BROWSER",
-        "network.direction": "outgoing",
+        "network.direction": "outbound",
         "network.iana_number": "6",
         "network.protocol": "https",
         "observer.egress.interface.name": "port9",

diff --git a/x-pack/filebeat/modules.d/fortinet.yml.disabled b/x-pack/filebeat/modules.d/fortinet.yml.disabled
@@ -15,6 +15,16 @@
     # The port to listen for syslog traffic. Defaults to 9004.
     #var.syslog_port: 9004
 
+    # Set internal interfaces. used to override parsed network.direction
+    # based on a tagged interface. Both internal and external interfaces must be
+    # set to leverage this functionality.
+    #var.internal_interfaces: [ "LAN" ]
+
+    # Set external interfaces. used to override parsed network.direction
+    # based on a tagged interface. Both internal and external interfaces must be
+    # set to leverage this functionality.
+    #var.external_interfaces: [ "WAN" ]
+
   clientendpoint:
     enabled: true