From c3406c7f6f474c9cb03fcabb0a6d44dda31f8eae Mon Sep 17 00:00:00 2001 From: P1llus Date: Mon, 22 Jun 2020 07:31:13 +0200 Subject: [PATCH 1/3] added split for when dns is array, only fix eventtime if the time is not in seconds, added new timezone format option. New testlogs has been added --- .../fortinet/firewall/ingest/pipeline.yml | 3 + .../module/fortinet/firewall/ingest/utm.yml | 4 + .../fortinet/firewall/test/fortinet.log | 2 + .../firewall/test/fortinet.log-expected.json | 198 +++++++++++++++--- 4 files changed, 182 insertions(+), 25 deletions(-) diff --git a/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml b/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml index 46f3f5c86e3..3fbc69896f9 100644 --- a/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml +++ b/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml @@ -45,6 +45,7 @@ processors: formats: - yyyy-MM-dd HH:mm:ss - yyyy-MM-dd HH:mm:ss Z + - yyyy-MM-dd HH:mm:ss z - ISO8601 timezone: "{{fortinet.firewall.tz}}" if: "ctx.fortinet?.firewall?.tz != null" @@ -54,12 +55,14 @@ processors: formats: - yyyy-MM-dd HH:mm:ss - yyyy-MM-dd HH:mm:ss Z + - yyyy-MM-dd HH:mm:ss z - ISO8601 if: "ctx.fortinet?.firewall?.tz == null" - gsub: field: fortinet.firewall.eventtime pattern: "\\d{6}$" replacement: "" + if: "(ctx.fortinet?.firewall?.eventtime).length() > 18" - date: field: fortinet.firewall.eventtime target_field: event.start diff --git a/x-pack/filebeat/module/fortinet/firewall/ingest/utm.yml b/x-pack/filebeat/module/fortinet/firewall/ingest/utm.yml index a85f09c332a..91e6726932b 100644 --- a/x-pack/filebeat/module/fortinet/firewall/ingest/utm.yml +++ b/x-pack/filebeat/module/fortinet/firewall/ingest/utm.yml @@ -230,6 +230,10 @@ processors: field: fortinet.firewall.ipaddr target_field: dns.resolved_ip ignore_missing: true +- split: + field: dns.resolved_ip + separator: ', ' + ignore_missing: true - rename: field: fortinet.firewall.level target_field: log.level diff --git a/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log b/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log index 78921e79db8..6ef348cdae8 100644 --- a/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log +++ b/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log @@ -1,7 +1,9 @@ <188>date=2020-04-23 time=12:17:48 devname="testswitch1" devid="somerouterid" logid="0316013056" type="utm" subtype="webfilter" eventtype="ftgd_blk" level="warning" vd="root" eventtime=1587230269052907555 tz="-0500" policyid=100602 sessionid=1234 user="elasticuser" group="elasticgroup" authserver="elasticauth" srcip=192.168.2.1 srcport=61930 srcintf="port1" srcintfrole="lan" dstip=8.8.8.8 dstport=443 dstintf="wan1" dstintfrole="wan" proto=6 service="HTTPS" hostname="elastic.co" profile="elasticruleset" action="blocked" reqtype="direct" url="/config/" sentbyte=1152 rcvdbyte=1130 direction="outgoing" msg="URL belongs to a denied category in policy" method="domain" cat=76 catdesc="Internet Telephony" <189>date=2020-04-23 time=12:17:45 devname="testswitch1" devid="somerouterid" logid="0317013312" type="utm" subtype="webfilter" eventtype="ftgd_allow" level="notice" vd="root" eventtime=1587230266314799756 tz="-0500" policyid=38 sessionid=543234 user="elasticuser" group="elasticgroup" authserver="elasticauth" srcip=192.168.2.1 srcport=65236 srcintf="port1" srcintfrole="lan" dstip=8.8.8.8 dstport=443 dstintf="wan1" dstintfrole="wan" proto=6 service="HTTPS" hostname="elastic.co" profile="elasticruleset" action="passthrough" reqtype="direct" url="/" sentbyte=3545 rcvdbyte=6812 direction="outgoing" msg="URL belongs to an allowed category in policy" method="domain" cat=23 catdesc="Web-based Email" <190>date=2020-04-23 time=13:17:35 devname="testswitch1" devid="somerouterid" logid="1059028704" type="utm" subtype="app-ctrl" eventtype="signature" level="information" vd="root" eventtime=1587230255061492894 tz="-0400" appid=40568 user="elasticuser" group="elasticgroup" authserver="elasticauth" srcip=192.168.2.1 dstip=8.8.8.8 srcport=59790 dstport=443 srcintf="LAN" srcintfrole="lan" dstintf="wan1" dstintfrole="wan" proto=6 service="SSL" direction="outgoing" policyid=12 sessionid=453234 applist="elasticruleset" action="pass" appcat="Web.Client" app="HTTPS.BROWSER" hostname="elastic.co" incidentserialno=23465 url="/" msg="Web.Client: HTTPS.BROWSER," apprisk="medium" scertcname="test.elastic.co" +<190>date=2020-04-23 time=13:17:35 devname="testswitch1" devid="somerouterid" logid="1059028704" type="utm" subtype="app-ctrl" eventtype="signature" level="information" vd="root" eventtime=1591788391 tz="-0400" appid=40568 user="elasticuser" group="elasticgroup" authserver="elasticauth" srcip=192.168.2.1 dstip=8.8.8.8 srcport=59790 dstport=443 srcintf="LAN" srcintfrole="lan" dstintf="wan1" dstintfrole="wan" proto=6 service="SSL" direction="outgoing" policyid=12 sessionid=453234 applist="elasticruleset" action="pass" appcat="Web.Client" app="HTTPS.BROWSER" hostname="elastic.co" incidentserialno=23465 url="/" msg="Web.Client: HTTPS.BROWSER," apprisk="medium" scertcname="test.elastic.co" <189>date=2020-04-23 time=12:17:29 devname="testswitch1" devid="somerouterid" logid="1501054802" type="utm" subtype="dns" eventtype="dns-response" level="notice" vd="root" eventtime=1587230249360109339 tz="-0500" policyid=26 sessionid=543234 srcip=192.168.2.1 srcport=53430 srcintf="port1" srcintfrole="lan" dstip=8.8.8.8 dstport=53 dstintf="wan1" dstintfrole="wan" proto=17 profile="test" xid=2234 qname="elastic.example.com" qtype="A" qtypeval=1 qclass="IN" ipaddr="8.8.8.8" msg="Domain is monitored" action="pass" cat=23 catdesc="Web-based Email" +<189>date=2020-04-23 time=12:17:29 devname="testswitch1" devid="somerouterid" logid="1501054802" type="utm" subtype="dns" eventtype="dns-response" level="notice" vd="root" eventtime=1587230249360109339 tz="-0500" policyid=26 sessionid=543234 srcip=192.168.2.1 srcport=53430 srcintf="port1" srcintfrole="lan" dstip=8.8.8.8 dstport=53 dstintf="wan1" dstintfrole="wan" proto=17 profile="test" xid=2234 qname="elastic.example.com" qtype="A" qtypeval=1 qclass="IN" ipaddr="8.8.8.8, 8.8.4.4" msg="Domain is monitored" action="pass" cat=23 catdesc="Web-based Email" <190>date=2020-04-23 time=12:17:11 devname="testswitch1" devid="somerouterid" logid="1059028704" type="utm" subtype="app-ctrl" eventtype="signature" level="information" vd="root" eventtime=1587230232148674303 tz="-0500" appid=40568 user="elasticuser" group="elasticgroup" authserver="elasticauth" srcip=192.168.2.1 dstip=8.8.8.8 srcport=63012 dstport=443 srcintf="port1" srcintfrole="lan" dstintf="wan1" dstintfrole="wan" proto=6 service="SSL" direction="outgoing" policyid=100602 sessionid=543234 applist="elasticruleset" action="pass" appcat="Web.Client" app="HTTPS.BROWSER" hostname="elastic.no" incidentserialno=54323 url="/" msg="Web.Client: HTTPS.BROWSER," apprisk="medium" <189>date=2020-04-23 time=12:17:04 devname="testswitch1" devid="somerouterid" logid="1501054802" type="utm" subtype="dns" eventtype="dns-response" level="notice" vd="root" eventtime=1587230224712900694 tz="-0500" policyid=26 sessionid=5432 srcip=192.168.2.1 srcport=54438 srcintf="port1" srcintfrole="lan" dstip=8.8.8.8 dstport=53 dstintf="wan1" dstintfrole="wan" proto=17 profile="elastictest" xid=2352 qname="elastic.co" qtype="A" qtypeval=1 qclass="IN" ipaddr="8.8.8.8" msg="Domain is monitored" action="pass" cat=93 catdesc="Remote Access" <190>date=2020-04-23 time=12:17:12 devname="testswitch1" devid="somerouterid" logid="1500054000" type="utm" subtype="dns" eventtype="dns-query" level="information" vd="root" eventtime=1587230232658642672 tz="-0500" policyid=26 sessionid=543234 srcip=192.168.2.1 srcport=54788 srcintf="port1" srcintfrole="lan" dstip=8.8.8.8 dstport=53 dstintf="wan1" dstintfrole="wan" proto=17 profile="elastictest" xid=235 qname="elastic.co" qtype="A" qtypeval=1 qclass="IN" diff --git a/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json b/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json index aee5f237edf..180c1d67903 100644 --- a/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json +++ b/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json @@ -223,6 +223,80 @@ "url.domain": "elastic.co", "url.path": "/" }, + { + "@timestamp": "2020-04-23T13:17:35.000-04:00", + "destination.as.number": 15169, + "destination.as.organization.name": "Google LLC", + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.location.lat": 37.751, + "destination.geo.location.lon": -97.822, + "destination.ip": "8.8.8.8", + "destination.port": 443, + "event.action": "signature", + "event.category": [ + "network" + ], + "event.code": "1059028704", + "event.dataset": "fortinet.firewall", + "event.kind": "event", + "event.module": "fortinet", + "event.outcome": "success", + "event.start": "1970-01-19T06:09:48.391-04:00", + "event.timezone": "-0400", + "event.type": [ + "allowed" + ], + "fileset.name": "firewall", + "fortinet.firewall.action": "pass", + "fortinet.firewall.appid": "40568", + "fortinet.firewall.apprisk": "medium", + "fortinet.firewall.authserver": "elasticauth", + "fortinet.firewall.dstintfrole": "wan", + "fortinet.firewall.incidentserialno": "23465", + "fortinet.firewall.sessionid": "453234", + "fortinet.firewall.srcintfrole": "lan", + "fortinet.firewall.subtype": "app-ctrl", + "fortinet.firewall.type": "utm", + "fortinet.firewall.vd": "root", + "input.type": "log", + "log.level": "information", + "log.offset": 2112, + "message": "Web.Client: HTTPS.BROWSER,", + "network.application": "HTTPS.BROWSER", + "network.direction": "outgoing", + "network.iana_number": "6", + "network.protocol": "ssl", + "observer.egress.interface.name": "wan1", + "observer.ingress.interface.name": "LAN", + "observer.name": "testswitch1", + "observer.product": "Fortigate", + "observer.serial_number": "somerouterid", + "observer.type": "firewall", + "observer.vendor": "Fortinet", + "related.ip": [ + "192.168.2.1", + "8.8.8.8" + ], + "related.user": [ + "elasticuser" + ], + "rule.category": "Web-Client", + "rule.id": "12", + "rule.ruleset": "elasticruleset", + "service.type": "fortinet", + "source.ip": "192.168.2.1", + "source.port": 59790, + "source.user.group.name": "elasticgroup", + "source.user.name": "elasticuser", + "tags": [ + "fortinet-firewall", + "forwarded" + ], + "tls.client.server_name": "test.elastic.co", + "url.domain": "elastic.co", + "url.path": "/" + }, { "@timestamp": "2020-04-23T12:17:29.000-05:00", "destination.as.number": 15169, @@ -237,7 +311,9 @@ "dns.question.class": "IN", "dns.question.name": "elastic.example.com", "dns.question.type": "A", - "dns.resolved_ip": "8.8.8.8", + "dns.resolved_ip": [ + "8.8.8.8" + ], "event.action": "dns-response", "event.category": [ "network" @@ -265,7 +341,77 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "notice", - "log.offset": 2112, + "log.offset": 2806, + "message": "Domain is monitored", + "network.iana_number": "17", + "observer.egress.interface.name": "wan1", + "observer.ingress.interface.name": "port1", + "observer.name": "testswitch1", + "observer.product": "Fortigate", + "observer.serial_number": "somerouterid", + "observer.type": "firewall", + "observer.vendor": "Fortinet", + "related.ip": [ + "192.168.2.1", + "8.8.8.8" + ], + "rule.category": "Web-based Email", + "rule.id": "26", + "rule.ruleset": "test", + "service.type": "fortinet", + "source.ip": "192.168.2.1", + "source.port": 53430, + "tags": [ + "fortinet-firewall", + "forwarded" + ] + }, + { + "@timestamp": "2020-04-23T12:17:29.000-05:00", + "destination.as.number": 15169, + "destination.as.organization.name": "Google LLC", + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.location.lat": 37.751, + "destination.geo.location.lon": -97.822, + "destination.ip": "8.8.8.8", + "destination.port": 53, + "dns.id": "2234", + "dns.question.class": "IN", + "dns.question.name": "elastic.example.com", + "dns.question.type": "A", + "dns.resolved_ip": [ + "8.8.8.8", + "8.8.4.4" + ], + "event.action": "dns-response", + "event.category": [ + "network" + ], + "event.code": "1501054802", + "event.dataset": "fortinet.firewall", + "event.kind": "event", + "event.module": "fortinet", + "event.outcome": "success", + "event.start": "2020-04-18T12:17:29.360-05:00", + "event.timezone": "-0500", + "event.type": [ + "info", + "allowed" + ], + "fileset.name": "firewall", + "fortinet.firewall.action": "pass", + "fortinet.firewall.cat": "23", + "fortinet.firewall.dstintfrole": "wan", + "fortinet.firewall.qtypeval": "1", + "fortinet.firewall.sessionid": "543234", + "fortinet.firewall.srcintfrole": "lan", + "fortinet.firewall.subtype": "dns", + "fortinet.firewall.type": "utm", + "fortinet.firewall.vd": "root", + "input.type": "log", + "log.level": "notice", + "log.offset": 3356, "message": "Domain is monitored", "network.iana_number": "17", "observer.egress.interface.name": "wan1", @@ -328,7 +474,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "information", - "log.offset": 2662, + "log.offset": 3915, "message": "Web.Client: HTTPS.BROWSER,", "network.application": "HTTPS.BROWSER", "network.direction": "outgoing", @@ -377,7 +523,9 @@ "dns.question.class": "IN", "dns.question.name": "elastic.co", "dns.question.type": "A", - "dns.resolved_ip": "8.8.8.8", + "dns.resolved_ip": [ + "8.8.8.8" + ], "event.action": "dns-response", "event.category": [ "network" @@ -405,7 +553,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "notice", - "log.offset": 3342, + "log.offset": 4595, "message": "Domain is monitored", "network.iana_number": "17", "observer.egress.interface.name": "wan1", @@ -467,7 +615,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "information", - "log.offset": 3886, + "log.offset": 5139, "network.iana_number": "17", "observer.egress.interface.name": "wan1", "observer.ingress.interface.name": "port1", @@ -525,7 +673,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "notice", - "log.offset": 4345, + "log.offset": 5598, "message": "Server certificate passed", "network.iana_number": "6", "network.protocol": "https", @@ -579,7 +727,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "notice", - "log.offset": 4875, + "log.offset": 6128, "message": "FSSO-logon event from FSSO_elasticserver: user elasticouser logged on 10.10.10.10", "observer.name": "testswitch3", "observer.product": "Fortigate", @@ -639,7 +787,7 @@ "fortinet.firewall.xauthuser": "N/A", "input.type": "log", "log.level": "error", - "log.offset": 5288, + "log.offset": 6541, "message": "IPsec phase 1 error", "observer.name": "testswitch3", "observer.product": "Fortigate", @@ -706,7 +854,7 @@ "fortinet.firewall.xauthuser": "N/A", "input.type": "log", "log.level": "notice", - "log.offset": 5856, + "log.offset": 7109, "message": "progress IPsec phase 1", "network.direction": "outbound", "observer.name": "testswitch3", @@ -764,7 +912,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "notice", - "log.offset": 6430, + "log.offset": 7683, "message": "Performance statistics: average CPU: 0, memory: 23, concurrent sessions: 20, setup-rate: 0", "observer.name": "testswitch3", "observer.product": "Fortigate", @@ -803,7 +951,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "notice", - "log.offset": 6920, + "log.offset": 8173, "message": "User elastiiiuser added to auth logon", "observer.name": "testswitch3", "observer.product": "Fortigate", @@ -866,7 +1014,7 @@ "fortinet.firewall.xauthuser": "N/A", "input.type": "log", "log.level": "notice", - "log.offset": 7298, + "log.offset": 8551, "message": "progress IPsec phase 1", "network.direction": "outbound", "observer.name": "testswitch3", @@ -906,7 +1054,7 @@ "fortinet.firewall.version": "1.522479", "input.type": "log", "log.level": "notice", - "log.offset": 7868, + "log.offset": 9121, "message": "FortiSandbox AV database updated", "observer.name": "testswitch3", "observer.product": "Fortigate", @@ -943,7 +1091,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "information", - "log.offset": 8172, + "log.offset": 9425, "message": "Add a FortiClient Connection.", "observer.name": "testswitch3", "observer.product": "Fortigate", @@ -993,7 +1141,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "information", - "log.offset": 8642, + "log.offset": 9895, "message": "SSL new connection", "observer.name": "testswitch3", "observer.product": "Fortigate", @@ -1043,7 +1191,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "information", - "log.offset": 9019, + "log.offset": 10272, "message": "SSL tunnel established", "observer.name": "testswitch3", "observer.product": "Fortigate", @@ -1089,7 +1237,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "notice", - "log.offset": 9450, + "log.offset": 10703, "message": "FSSO-logoff event from FSSO_somefssoserver: user elasticuser logged off 1192.168.1.1", "observer.name": "testswitch3", "observer.product": "Fortigate", @@ -1127,7 +1275,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "notice", - "log.offset": 9875, + "log.offset": 11128, "message": "FortiCloud 9.9.9.9 server is connected", "observer.name": "testswitch3", "observer.product": "Fortigate", @@ -1158,7 +1306,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "notice", - "log.offset": 10195, + "log.offset": 11448, "message": "FortiCloud 4.4.4.4 server is disconnected", "observer.name": "testswitch3", "observer.product": "Fortigate", @@ -1213,7 +1361,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "warning", - "log.offset": 10550, + "log.offset": 11803, "network.iana_number": "17", "network.protocol": "dns", "observer.egress.interface.name": "wan1", @@ -1286,7 +1434,7 @@ "fortinet.firewall.vwlid": "0", "input.type": "log", "log.level": "notice", - "log.offset": 11142, + "log.offset": 12395, "network.bytes": 504096, "network.iana_number": "17", "network.packets": 1769018, @@ -1369,7 +1517,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "notice", - "log.offset": 11876, + "log.offset": 13129, "network.application": "icmp6/25/0", "network.bytes": 3034, "network.iana_number": "58", @@ -1447,7 +1595,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "notice", - "log.offset": 12426, + "log.offset": 13679, "network.application": "PING", "network.bytes": 10, "network.iana_number": "1", @@ -1516,7 +1664,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "warning", - "log.offset": 12972, + "log.offset": 14225, "network.iana_number": "17", "network.protocol": "udp/12302", "observer.egress.interface.name": "newinterface", From ac29e6c42dc05a2fe98c5395f57bfc0d790166c6 Mon Sep 17 00:00:00 2001 From: P1llus Date: Tue, 23 Jun 2020 15:13:52 +0200 Subject: [PATCH 2/3] updated changelog --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 2ee62f329a6..447a7fab694 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -194,6 +194,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix `o365` module ignoring `var.api` settings. {pull}18948[18948] - Fix `netflow` module to support 7 bytepad for IPFIX template. {issue}18098[18098] - Fix Cisco ASA dissect pattern for 313008 & 313009 messages. {pull}19149[19149] +- Fix date and timestamp formats for fortigate module {pull}19316[19316] *Heartbeat* From e84fa1988da246c79cb2079186c2ddf7dbb9968a Mon Sep 17 00:00:00 2001 From: P1llus Date: Wed, 24 Jun 2020 17:20:33 +0200 Subject: [PATCH 3/3] Fixing format set for events with non timezones --- .../fortinet/firewall/ingest/pipeline.yml | 9 +- .../fortinet/firewall/test/fortinet.log | 1 + .../firewall/test/fortinet.log-expected.json | 125 ++++++++++++++---- 3 files changed, 107 insertions(+), 28 deletions(-) diff --git a/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml b/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml index 3fbc69896f9..8fe8171ecd5 100644 --- a/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml +++ b/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml @@ -37,7 +37,7 @@ processors: if: "ctx.fortinet?.firewall?.tz != null" - set: field: _temp.time - value: "{{fortinet.firewall.date}} {{fortinet.firewall.time}}Z" + value: "{{fortinet.firewall.date}} {{fortinet.firewall.time}}" if: "ctx.fortinet?.firewall?.tz == null" - date: field: _temp.time @@ -69,6 +69,13 @@ processors: formats: - UNIX_MS timezone: "{{fortinet.firewall.tz}}" + if: "ctx.fortinet?.firewall?.tz != null" +- date: + field: fortinet.firewall.eventtime + target_field: event.start + formats: + - UNIX_MS + if: "ctx.fortinet?.firewall?.tz == null" - rename: field: fortinet.firewall.devname target_field: observer.name diff --git a/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log b/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log index 6ef348cdae8..01fc2444606 100644 --- a/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log +++ b/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log @@ -1,4 +1,5 @@ <188>date=2020-04-23 time=12:17:48 devname="testswitch1" devid="somerouterid" logid="0316013056" type="utm" subtype="webfilter" eventtype="ftgd_blk" level="warning" vd="root" eventtime=1587230269052907555 tz="-0500" policyid=100602 sessionid=1234 user="elasticuser" group="elasticgroup" authserver="elasticauth" srcip=192.168.2.1 srcport=61930 srcintf="port1" srcintfrole="lan" dstip=8.8.8.8 dstport=443 dstintf="wan1" dstintfrole="wan" proto=6 service="HTTPS" hostname="elastic.co" profile="elasticruleset" action="blocked" reqtype="direct" url="/config/" sentbyte=1152 rcvdbyte=1130 direction="outgoing" msg="URL belongs to a denied category in policy" method="domain" cat=76 catdesc="Internet Telephony" +<189>date=2020-04-23 time=01:16:08 devname="testswitch1" devid="somerouterid" logid="0000000013" type="traffic" subtype="forward" level="notice" vd="OPERATIONAL" eventtime=1592961368 srcip=10.10.10.10 srcport=60899 srcintf="srcintfname" srcintfrole="lan" dstip=8.8.8.8 dstport=161 dstintf="dstintfname" dstintfrole="lan" sessionid=155313 proto=17 action="deny" policyid=0 policytype="policy" service="SNMP" dstcountry="Reserved" srccountry="Reserved" trandisp="noop" duration=0 sentbyte=0 rcvdbyte=0 sentpkt=0 appcat="unscanned" crscore=30 craction=131072 crlevel="high" <189>date=2020-04-23 time=12:17:45 devname="testswitch1" devid="somerouterid" logid="0317013312" type="utm" subtype="webfilter" eventtype="ftgd_allow" level="notice" vd="root" eventtime=1587230266314799756 tz="-0500" policyid=38 sessionid=543234 user="elasticuser" group="elasticgroup" authserver="elasticauth" srcip=192.168.2.1 srcport=65236 srcintf="port1" srcintfrole="lan" dstip=8.8.8.8 dstport=443 dstintf="wan1" dstintfrole="wan" proto=6 service="HTTPS" hostname="elastic.co" profile="elasticruleset" action="passthrough" reqtype="direct" url="/" sentbyte=3545 rcvdbyte=6812 direction="outgoing" msg="URL belongs to an allowed category in policy" method="domain" cat=23 catdesc="Web-based Email" <190>date=2020-04-23 time=13:17:35 devname="testswitch1" devid="somerouterid" logid="1059028704" type="utm" subtype="app-ctrl" eventtype="signature" level="information" vd="root" eventtime=1587230255061492894 tz="-0400" appid=40568 user="elasticuser" group="elasticgroup" authserver="elasticauth" srcip=192.168.2.1 dstip=8.8.8.8 srcport=59790 dstport=443 srcintf="LAN" srcintfrole="lan" dstintf="wan1" dstintfrole="wan" proto=6 service="SSL" direction="outgoing" policyid=12 sessionid=453234 applist="elasticruleset" action="pass" appcat="Web.Client" app="HTTPS.BROWSER" hostname="elastic.co" incidentserialno=23465 url="/" msg="Web.Client: HTTPS.BROWSER," apprisk="medium" scertcname="test.elastic.co" <190>date=2020-04-23 time=13:17:35 devname="testswitch1" devid="somerouterid" logid="1059028704" type="utm" subtype="app-ctrl" eventtype="signature" level="information" vd="root" eventtime=1591788391 tz="-0400" appid=40568 user="elasticuser" group="elasticgroup" authserver="elasticauth" srcip=192.168.2.1 dstip=8.8.8.8 srcport=59790 dstport=443 srcintf="LAN" srcintfrole="lan" dstintf="wan1" dstintfrole="wan" proto=6 service="SSL" direction="outgoing" policyid=12 sessionid=453234 applist="elasticruleset" action="pass" appcat="Web.Client" app="HTTPS.BROWSER" hostname="elastic.co" incidentserialno=23465 url="/" msg="Web.Client: HTTPS.BROWSER," apprisk="medium" scertcname="test.elastic.co" diff --git a/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json b/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json index 180c1d67903..355c77d03be 100644 --- a/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json +++ b/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json @@ -74,6 +74,77 @@ "url.domain": "elastic.co", "url.path": "/config/" }, + { + "@timestamp": "2020-04-23T01:16:08.000Z", + "destination.as.number": 15169, + "destination.as.organization.name": "Google LLC", + "destination.bytes": 0, + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.location.lat": 37.751, + "destination.geo.location.lon": -97.822, + "destination.ip": "8.8.8.8", + "destination.port": 161, + "event.action": "deny", + "event.category": [ + "network" + ], + "event.code": "0000000013", + "event.dataset": "fortinet.firewall", + "event.duration": 0, + "event.kind": "event", + "event.module": "fortinet", + "event.outcome": "success", + "event.start": "1970-01-19T10:29:21.368Z", + "event.timezone": "-02:00", + "event.type": [ + "connection", + "end" + ], + "fileset.name": "firewall", + "fortinet.firewall.action": "deny", + "fortinet.firewall.craction": "131072", + "fortinet.firewall.crlevel": "high", + "fortinet.firewall.crscore": "30", + "fortinet.firewall.dstcountry": "Reserved", + "fortinet.firewall.dstintfrole": "lan", + "fortinet.firewall.sessionid": "155313", + "fortinet.firewall.srccountry": "Reserved", + "fortinet.firewall.srcintfrole": "lan", + "fortinet.firewall.subtype": "forward", + "fortinet.firewall.trandisp": "noop", + "fortinet.firewall.type": "traffic", + "fortinet.firewall.vd": "OPERATIONAL", + "input.type": "log", + "log.level": "notice", + "log.offset": 707, + "network.bytes": 0, + "network.iana_number": "17", + "network.protocol": "snmp", + "observer.egress.interface.name": "dstintfname", + "observer.ingress.interface.name": "srcintfname", + "observer.name": "testswitch1", + "observer.product": "Fortigate", + "observer.serial_number": "somerouterid", + "observer.type": "firewall", + "observer.vendor": "Fortinet", + "related.ip": [ + "10.10.10.10", + "8.8.8.8" + ], + "rule.category": "unscanned", + "rule.id": "0", + "rule.ruleset": "policy", + "service.type": "fortinet", + "source.bytes": 0, + "source.ip": "10.10.10.10", + "source.packets": 0, + "source.port": 60899, + "tags": [ + "fortinet-firewall", + "forwarded" + ] + }, { "@timestamp": "2020-04-23T12:17:45.000-05:00", "destination.as.number": 15169, @@ -113,7 +184,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "notice", - "log.offset": 707, + "log.offset": 1278, "message": "URL belongs to an allowed category in policy", "network.bytes": 10357, "network.direction": "outgoing", @@ -187,7 +258,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "information", - "log.offset": 1409, + "log.offset": 1980, "message": "Web.Client: HTTPS.BROWSER,", "network.application": "HTTPS.BROWSER", "network.direction": "outgoing", @@ -261,7 +332,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "information", - "log.offset": 2112, + "log.offset": 2683, "message": "Web.Client: HTTPS.BROWSER,", "network.application": "HTTPS.BROWSER", "network.direction": "outgoing", @@ -341,7 +412,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "notice", - "log.offset": 2806, + "log.offset": 3377, "message": "Domain is monitored", "network.iana_number": "17", "observer.egress.interface.name": "wan1", @@ -411,7 +482,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "notice", - "log.offset": 3356, + "log.offset": 3927, "message": "Domain is monitored", "network.iana_number": "17", "observer.egress.interface.name": "wan1", @@ -474,7 +545,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "information", - "log.offset": 3915, + "log.offset": 4486, "message": "Web.Client: HTTPS.BROWSER,", "network.application": "HTTPS.BROWSER", "network.direction": "outgoing", @@ -553,7 +624,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "notice", - "log.offset": 4595, + "log.offset": 5166, "message": "Domain is monitored", "network.iana_number": "17", "observer.egress.interface.name": "wan1", @@ -615,7 +686,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "information", - "log.offset": 5139, + "log.offset": 5710, "network.iana_number": "17", "observer.egress.interface.name": "wan1", "observer.ingress.interface.name": "port1", @@ -673,7 +744,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "notice", - "log.offset": 5598, + "log.offset": 6169, "message": "Server certificate passed", "network.iana_number": "6", "network.protocol": "https", @@ -727,7 +798,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "notice", - "log.offset": 6128, + "log.offset": 6699, "message": "FSSO-logon event from FSSO_elasticserver: user elasticouser logged on 10.10.10.10", "observer.name": "testswitch3", "observer.product": "Fortigate", @@ -787,7 +858,7 @@ "fortinet.firewall.xauthuser": "N/A", "input.type": "log", "log.level": "error", - "log.offset": 6541, + "log.offset": 7112, "message": "IPsec phase 1 error", "observer.name": "testswitch3", "observer.product": "Fortigate", @@ -854,7 +925,7 @@ "fortinet.firewall.xauthuser": "N/A", "input.type": "log", "log.level": "notice", - "log.offset": 7109, + "log.offset": 7680, "message": "progress IPsec phase 1", "network.direction": "outbound", "observer.name": "testswitch3", @@ -912,7 +983,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "notice", - "log.offset": 7683, + "log.offset": 8254, "message": "Performance statistics: average CPU: 0, memory: 23, concurrent sessions: 20, setup-rate: 0", "observer.name": "testswitch3", "observer.product": "Fortigate", @@ -951,7 +1022,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "notice", - "log.offset": 8173, + "log.offset": 8744, "message": "User elastiiiuser added to auth logon", "observer.name": "testswitch3", "observer.product": "Fortigate", @@ -1014,7 +1085,7 @@ "fortinet.firewall.xauthuser": "N/A", "input.type": "log", "log.level": "notice", - "log.offset": 8551, + "log.offset": 9122, "message": "progress IPsec phase 1", "network.direction": "outbound", "observer.name": "testswitch3", @@ -1054,7 +1125,7 @@ "fortinet.firewall.version": "1.522479", "input.type": "log", "log.level": "notice", - "log.offset": 9121, + "log.offset": 9692, "message": "FortiSandbox AV database updated", "observer.name": "testswitch3", "observer.product": "Fortigate", @@ -1091,7 +1162,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "information", - "log.offset": 9425, + "log.offset": 9996, "message": "Add a FortiClient Connection.", "observer.name": "testswitch3", "observer.product": "Fortigate", @@ -1141,7 +1212,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "information", - "log.offset": 9895, + "log.offset": 10466, "message": "SSL new connection", "observer.name": "testswitch3", "observer.product": "Fortigate", @@ -1191,7 +1262,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "information", - "log.offset": 10272, + "log.offset": 10843, "message": "SSL tunnel established", "observer.name": "testswitch3", "observer.product": "Fortigate", @@ -1237,7 +1308,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "notice", - "log.offset": 10703, + "log.offset": 11274, "message": "FSSO-logoff event from FSSO_somefssoserver: user elasticuser logged off 1192.168.1.1", "observer.name": "testswitch3", "observer.product": "Fortigate", @@ -1275,7 +1346,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "notice", - "log.offset": 11128, + "log.offset": 11699, "message": "FortiCloud 9.9.9.9 server is connected", "observer.name": "testswitch3", "observer.product": "Fortigate", @@ -1306,7 +1377,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "notice", - "log.offset": 11448, + "log.offset": 12019, "message": "FortiCloud 4.4.4.4 server is disconnected", "observer.name": "testswitch3", "observer.product": "Fortigate", @@ -1361,7 +1432,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "warning", - "log.offset": 11803, + "log.offset": 12374, "network.iana_number": "17", "network.protocol": "dns", "observer.egress.interface.name": "wan1", @@ -1434,7 +1505,7 @@ "fortinet.firewall.vwlid": "0", "input.type": "log", "log.level": "notice", - "log.offset": 12395, + "log.offset": 12966, "network.bytes": 504096, "network.iana_number": "17", "network.packets": 1769018, @@ -1517,7 +1588,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "notice", - "log.offset": 13129, + "log.offset": 13700, "network.application": "icmp6/25/0", "network.bytes": 3034, "network.iana_number": "58", @@ -1595,7 +1666,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "notice", - "log.offset": 13679, + "log.offset": 14250, "network.application": "PING", "network.bytes": 10, "network.iana_number": "1", @@ -1664,7 +1735,7 @@ "fortinet.firewall.vd": "root", "input.type": "log", "log.level": "warning", - "log.offset": 14225, + "log.offset": 14796, "network.iana_number": "17", "network.protocol": "udp/12302", "observer.egress.interface.name": "newinterface",