[Filebeat][fortinet] Map x509 ecs fields for fortinet fw fileset (ela…

…stic#20983) (elastic#21014) * Map x509 ecs fields for fortinet fw fileset * Remove wrongly mapped field and bump ecs version (cherry picked from commit e11283e)
mtojek · Sep 9, 2020 · ee5d807 · ee5d807
1 parent 46f76cd
commit ee5d807
Show file tree

Hide file tree

Showing 7 changed files with 195 additions and 8 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -286,6 +286,7 @@ field. You can revert this change by configuring tags for the module and omittin
 - Fix long registry migration times. {pull}20717[20717] {issue}20705[20705]
 - Fix event types and categories in auditd module to comply with ECS {pull}20652[20652]
 - Update documentation in the azure module filebeat. {pull}20815[20815]
+- Remove wrongly mapped `tls.client.server_name` from `fortinet/firewall` fileset. {pull}20983[20983]
 
 *Heartbeat*
 
@@ -637,6 +638,7 @@ field. You can revert this change by configuring tags for the module and omittin
 - Improve Zeek x509 module with `x509` ECS mappings {pull}20867[20867]
 - Improve Zeek SSL module with `x509` ECS mappings {pull}20927[20927]
 - Improve Zeek Kerberos module with `x509` ECS mappings {pull}20958[20958]
+- Improve Fortinet firewall module with `x509` ECS mappings {pull}20983[20983]
 
 *Heartbeat*
 

diff --git a/x-pack/filebeat/module/fortinet/fields.go b/x-pack/filebeat/module/fortinet/fields.go
diff --git a/x-pack/filebeat/module/fortinet/firewall/_meta/fields.yml b/x-pack/filebeat/module/fortinet/firewall/_meta/fields.yml
@@ -3,6 +3,7 @@
   description: >
     Fields from fortinet FortiOS
   fields:
+
     - name: file.hash.crc32
       type: keyword
       description: >

diff --git a/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml b/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml
@@ -27,4 +27,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.5.0
+        ecs.version: 1.6.0
diff --git a/x-pack/filebeat/module/fortinet/firewall/ingest/utm.yml b/x-pack/filebeat/module/fortinet/firewall/ingest/utm.yml
@@ -30,7 +30,7 @@ processors:
     field: fortinet.firewall.remip
     target_field: destination.ip
     ignore_missing: true
-    if: "ctx.destination?.ip == null"  
+    if: "ctx.destination?.ip == null"
 - convert:
     field: fortinet.firewall.dst_port
     target_field: destination.port
@@ -295,16 +295,24 @@ processors:
     ignore_missing: true
 - rename:
     field: fortinet.firewall.scertcname
-    target_field: tls.client.server_name
+    target_field: tls.server.x509.subject.common_name
     ignore_missing: true
 - rename:
     field: fortinet.firewall.scertissuer
     target_field: tls.server.issuer
     ignore_missing: true
+- set:
+    field: tls.server.x509.issuer.common_name
+    value: "{{tls.server.issuer}}"
+    ignore_empty_value: true
 - rename:
     field: fortinet.firewall.ccertissuer
     target_field: tls.client.issuer
     ignore_missing: true
+- set:
+    field: tls.client.x509.issuer.common_name
+    value: "{{tls.client.issuer}}"
+    ignore_empty_value: true
 - rename:
     field: fortinet.firewall.sender
     target_field: tls.server.issuer
@@ -427,4 +435,4 @@ processors:
 on_failure:
 - set:
     field: error.message
-    value: '{{ _ingest.on_failure_message }}'
+    value: '{{ _ingest.on_failure_message }}'
diff --git a/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log b/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log
@@ -27,4 +27,5 @@
 <189>date=2020-04-23 time=12:11:48 devname="newfirewall" devid="newrouterid" logid="0001000014" type="traffic" subtype="local" level="notice" vd="root" eventtime=1587229908751434997 tz="-0500" srcip=2001:4860:4860::8888 identifier=0 srcintf="port1" srcintfrole="lan" dstip=2001:4860:4860::8888 dstintf="unknown0" dstintfrole="undefined" sessionid=6542345 proto=58 action="accept" policyid=0 policytype="someotherpolicy" service="icmp6/1/0" trandisp="noop" app="icmp6/25/0" duration=42 sentbyte=3014 rcvdbyte=20 sentpkt=4 rcvdpkt=0 appcat="unscanned"
 <189>date=2020-04-23 time=13:10:57 devname="newfirewall" devid="newrouterid" logid="0001000014" type="traffic" subtype="local" level="notice" vd="root" eventtime=1587229857509058693 tz="-0400" srcip=9.7.7.7 identifier=61 srcintf="wan1" srcintfrole="wan" dstip=8.8.8.8 dstintf="unknown0" dstintfrole="undefined" sessionid=123 proto=1 action="accept" policyid=0 policytype="rulepolicy" service="PING" dstcountry="Norway" srccountry="Netherlands" trandisp="noop" app="PING" duration=20 sentbyte=0 rcvdbyte=10 sentpkt=0 rcvdpkt=40 appcat="unscanned"
 <188>date=2020-04-23 time=12:14:39 devname="firewall3" devid="oldfwid" logid="0000000011" type="traffic" subtype="forward" level="warning" vd="root" eventtime=1587230079841464445 tz="-0500" srcip=192.168.1.1 srcport=62493 srcintf="port1" srcintfrole="lan" dstip=192.168.100.100 dstport=1235 dstintf="newinterface" dstintfrole="undefined" sessionid=54234 proto=17 action="ip-conn" policyid=49 policytype="policy" poluuid="654cc-b6542-53467u8-e45234-1566casd35f7836" policyname="oldpolicyname" user="elasticsuper" authserver="FSSO_newfsso" service="udp/12302" dstcountry="Reserved" srccountry="Reserved" appcat="unscanned" crscore=5 craction=63332144 crlevel="low"
-<189>date=2020-04-23 time=12:14:28 devname="firewall3" devid="oldfwid" logid="0000000013" type="traffic" subtype="forward" level="notice" vd="root" eventtime=1587230069291463928 tz="-0500" srcip=192.168.50.50 srcport=56603 srcintf="port1" srcintfrole="lan" dstip=8.8.8.8 dstport=442 dstintf="wan1" dstintfrole="wan" sessionid=2345 proto=6 action="close" policyid=2365 policytype="policy" poluuid="654644c-b064-fdgdf3425-f003-1234ghdf682e05f" policyname="someoldpolicyname" user="elasticuser" group="testgroup" authserver="FSSO_something" service="HTTPS" dstcountry="Netherlands" srccountry="Reserved" trandisp="snat" transip=23.23.23.23 transport=603 appid=43540 app="Skype.Portals" appcat="Collaboration" apprisk="elevated" applist="someapplist" appact="detected" duration=126 sentbyte=923 rcvdbyte=77654 sentpkt=113 rcvdpkt=70 vwlid=4 vwlquality="Seq_num(3), alive, selected" wanin=1130 wanout=6671 lanin=1406 lanout=146506 utmaction="block" countweb=1 countapp=1 crscore=5 craction=6144 crlevel="low"
+<189>date=2020-04-23 time=12:14:28 devname="firewall3" devid="oldfwid" logid="0000000013" type="traffic" subtype="forward" level="notice" vd="root" eventtime=1587230069291463928 tz="-0500" srcip=192.168.50.50 srcport=56603 srcintf="port1" srcintfrole="lan" dstip=8.8.8.8 dstport=442 dstintf="wan1" dstintfrole="wan" sessionid=2345 proto=6 action="close" policyid=2365 policytype="policy" poluuid="654644c-b064-fdgdf3425-f003-1234ghdf682e05f" policyname="someoldpolicyname" user="elasticuser" group="testgroup" authserver="FSSO_something" service="HTTPS" dstcountry="Netherlands" srccountry="Reserved" trandisp="snat" transip=23.23.23.23 transport=603 appid=43540 app="Skype.Portals" appcat="Collaboration" apprisk="elevated" applist="someapplist" appact="detected" duration=126 sentbyte=923 rcvdbyte=77654 sentpkt=113 rcvdpkt=70 vwlid=4 vwlquality="Seq_num(3), alive, selected" wanin=1130 wanout=6671 lanin=1406 lanout=146506 utmaction="block" countweb=1 countapp=1 crscore=5 craction=6144 crlevel="low"
+<190>date=2019-05-15 time=18:03:36 logid="1059028704" type="utm" subtype="app-ctrl" eventtype="app-ctrl-all" level="information" vd="root" eventtime=1557968615 appid=40568 srcip=10.1.100.22 dstip=195.8.215.136 srcport=50798 dstport=443 srcintf="port10" srcintfrole="lan" dstintf="port9" dstintfrole="wan" proto=6 service="HTTPS" direction="outgoing" policyid=1 sessionid=4414 applist="block-social.media" appcat="Web.Client" app="HTTPS.BROWSER" action="pass" hostname="www.dailymotion.com" incidentserialno=1962906680 url="/" msg="Web.Client: HTTPS.BROWSER," apprisk="medium" scertcname="*.dailymotion.com" scertissuer="DigiCert SHA2 High Assurance Server CA"
diff --git a/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json b/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json
@@ -289,7 +289,7 @@
             "fortinet-firewall",
             "forwarded"
         ],
-        "tls.client.server_name": "test.elastic.co",
+        "tls.server.x509.subject.common_name": "test.elastic.co",
         "url.domain": "elastic.co",
         "url.path": "/"
     },
@@ -363,7 +363,7 @@
             "fortinet-firewall",
             "forwarded"
         ],
-        "tls.client.server_name": "test.elastic.co",
+        "tls.server.x509.subject.common_name": "test.elastic.co",
         "url.domain": "elastic.co",
         "url.path": "/"
     },
@@ -1764,5 +1764,180 @@
             "fortinet-firewall",
             "forwarded"
         ]
+    },
+    {
+        "@timestamp": "2020-04-23T12:14:28.000-05:00",
+        "destination.as.number": 15169,
+        "destination.as.organization.name": "Google LLC",
+        "destination.bytes": 77654,
+        "destination.geo.continent_name": "North America",
+        "destination.geo.country_iso_code": "US",
+        "destination.geo.location.lat": 37.751,
+        "destination.geo.location.lon": -97.822,
+        "destination.ip": "8.8.8.8",
+        "destination.packets": 70,
+        "destination.port": 442,
+        "event.action": "close",
+        "event.category": [
+            "network"
+        ],
+        "event.code": "0000000013",
+        "event.dataset": "fortinet.firewall",
+        "event.duration": 126000000000,
+        "event.kind": "event",
+        "event.module": "fortinet",
+        "event.outcome": "success",
+        "event.start": "2020-04-18T12:14:29.291-05:00",
+        "event.timezone": "-0500",
+        "event.type": [
+            "connection",
+            "end",
+            "protocol",
+            "denied"
+        ],
+        "fileset.name": "firewall",
+        "fortinet.firewall.action": "close",
+        "fortinet.firewall.appact": "detected",
+        "fortinet.firewall.appid": "43540",
+        "fortinet.firewall.applist": "someapplist",
+        "fortinet.firewall.apprisk": "elevated",
+        "fortinet.firewall.authserver": "FSSO_something",
+        "fortinet.firewall.countapp": "1",
+        "fortinet.firewall.countweb": "1",
+        "fortinet.firewall.craction": "6144",
+        "fortinet.firewall.crlevel": "low",
+        "fortinet.firewall.crscore": "5",
+        "fortinet.firewall.dstcountry": "Netherlands",
+        "fortinet.firewall.dstintfrole": "wan",
+        "fortinet.firewall.lanin": "1406",
+        "fortinet.firewall.lanout": "146506",
+        "fortinet.firewall.sessionid": "2345",
+        "fortinet.firewall.srccountry": "Reserved",
+        "fortinet.firewall.srcintfrole": "lan",
+        "fortinet.firewall.subtype": "forward",
+        "fortinet.firewall.trandisp": "snat",
+        "fortinet.firewall.type": "traffic",
+        "fortinet.firewall.utmaction": "block",
+        "fortinet.firewall.vd": "root",
+        "fortinet.firewall.vwlid": "4",
+        "fortinet.firewall.vwlquality": "Seq_num(3), alive, selected",
+        "fortinet.firewall.wanin": "1130",
+        "fortinet.firewall.wanout": "6671",
+        "input.type": "log",
+        "log.level": "notice",
+        "log.offset": 15459,
+        "network.application": "Skype.Portals",
+        "network.bytes": 78577,
+        "network.iana_number": "6",
+        "network.packets": 183,
+        "network.protocol": "https",
+        "observer.egress.interface.name": "wan1",
+        "observer.ingress.interface.name": "port1",
+        "observer.name": "firewall3",
+        "observer.product": "Fortigate",
+        "observer.serial_number": "oldfwid",
+        "observer.type": "firewall",
+        "observer.vendor": "Fortinet",
+        "related.ip": [
+            "192.168.50.50",
+            "8.8.8.8"
+        ],
+        "related.user": [
+            "elasticuser"
+        ],
+        "rule.category": "Collaboration",
+        "rule.id": "2365",
+        "rule.name": "someoldpolicyname",
+        "rule.ruleset": "policy",
+        "rule.uuid": "654644c-b064-fdgdf3425-f003-1234ghdf682e05f",
+        "service.type": "fortinet",
+        "source.as.number": 14618,
+        "source.as.organization.name": "Amazon.com, Inc.",
+        "source.bytes": 923,
+        "source.geo.city_name": "Ashburn",
+        "source.geo.continent_name": "North America",
+        "source.geo.country_iso_code": "US",
+        "source.geo.location.lat": 39.0481,
+        "source.geo.location.lon": -77.4728,
+        "source.geo.region_iso_code": "US-VA",
+        "source.geo.region_name": "Virginia",
+        "source.ip": "192.168.50.50",
+        "source.nat.ip": "23.23.23.23",
+        "source.nat.port": 603,
+        "source.packets": 113,
+        "source.port": 56603,
+        "source.user.group.name": "testgroup",
+        "source.user.name": "elasticuser",
+        "tags": [
+            "fortinet-firewall",
+            "forwarded"
+        ]
+    },
+    {
+        "@timestamp": "2019-05-15T18:03:36.000Z",
+        "destination.as.number": 41690,
+        "destination.as.organization.name": "Dailymotion S.A.",
+        "destination.geo.continent_name": "Europe",
+        "destination.geo.country_iso_code": "FR",
+        "destination.geo.location.lat": 48.8582,
+        "destination.geo.location.lon": 2.3387,
+        "destination.ip": "195.8.215.136",
+        "destination.port": 443,
+        "event.action": "app-ctrl-all",
+        "event.category": [
+            "network"
+        ],
+        "event.code": "1059028704",
+        "event.dataset": "fortinet.firewall",
+        "event.kind": "event",
+        "event.module": "fortinet",
+        "event.outcome": "success",
+        "event.start": "2019-05-16T01:03:35.000Z",
+        "event.type": [
+            "allowed"
+        ],
+        "fileset.name": "firewall",
+        "fortinet.firewall.action": "pass",
+        "fortinet.firewall.appid": "40568",
+        "fortinet.firewall.apprisk": "medium",
+        "fortinet.firewall.dstintfrole": "wan",
+        "fortinet.firewall.incidentserialno": "1962906680",
+        "fortinet.firewall.sessionid": "4414",
+        "fortinet.firewall.srcintfrole": "lan",
+        "fortinet.firewall.subtype": "app-ctrl",
+        "fortinet.firewall.type": "utm",
+        "fortinet.firewall.vd": "root",
+        "input.type": "log",
+        "log.level": "information",
+        "log.offset": 16463,
+        "message": "Web.Client: HTTPS.BROWSER,",
+        "network.application": "HTTPS.BROWSER",
+        "network.direction": "outgoing",
+        "network.iana_number": "6",
+        "network.protocol": "https",
+        "observer.egress.interface.name": "port9",
+        "observer.ingress.interface.name": "port10",
+        "observer.product": "Fortigate",
+        "observer.type": "firewall",
+        "observer.vendor": "Fortinet",
+        "related.ip": [
+            "10.1.100.22",
+            "195.8.215.136"
+        ],
+        "rule.category": "Web-Client",
+        "rule.id": "1",
+        "rule.ruleset": "block-social.media",
+        "service.type": "fortinet",
+        "source.ip": "10.1.100.22",
+        "source.port": 50798,
+        "tags": [
+            "fortinet-firewall",
+            "forwarded"
+        ],
+        "tls.server.issuer": "DigiCert SHA2 High Assurance Server CA",
+        "tls.server.x509.issuer.common_name": "DigiCert SHA2 High Assurance Server CA",
+        "tls.server.x509.subject.common_name": "*.dailymotion.com",
+        "url.domain": "www.dailymotion.com",
+        "url.path": "/"
     }
 ]