From a8061299a519e261802d7d10dc00df912d58d0d7 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Wed, 30 Jun 2021 13:54:50 +0200 Subject: [PATCH] [filebeat][fortinet] Use default add_locale for fortinet.firewall (#26524) * Use default add_locale for fortinet.firewall * Use event.timezone in all date processors (cherry picked from commit 5d258c897ab14422072626b218f3bb09996ab189) --- CHANGELOG.next.asciidoc | 32 +++++++++++++++++++ filebeat/docs/modules/fortinet.asciidoc | 4 ++- .../module/fortinet/_meta/docs.asciidoc | 4 ++- .../fortinet/firewall/config/firewall.yml | 1 + .../fortinet/firewall/ingest/pipeline.yml | 24 +++++++------- .../firewall/test/fortinet.log-expected.json | 13 +++++--- 6 files changed, 59 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 0e1aeec8868..beb7f3f693d 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -558,6 +558,38 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add `awsfargate` module to collect container logs from Amazon ECS on Fargate. {pull}25041[25041] - New module `cyberarkpas` for CyberArk Privileged Access Security audit logs. {pull}24803[24803] - Add `uri_parts` processor to Apache, Nginx, IIS, Traefik, S3Access, Cisco, F5, Fortinet, Google Workspace, Imperva, Microsoft, Netscout, O365, Sophos, Squid, Suricata, Zeek, Zia, Zoom, and ZScaler modules ingest pipelines. {issue}19088[19088] {pull}24699[24699] +- Update PanOS module to parse Global Protect & User ID logs. {issue}24722[24722] {issue}24724[24724] {pull}24927[24927] +- Add HMAC signature validation support for http_endpoint input. {pull}24918[24918] +- Add new grok pattern for iptables module for Ubiquiti UDM {issue}25615[25615] {pull}25616[25616] +- Add multiline support to aws-s3 input. {issue}25249[25249] {pull}25710[25710] {pull}25873[25873] +- Add monitoring metrics to the `aws-s3` input. {pull}25711[25711] +- Added `network.direction` fields to Zeek and Suricata modules using the `add_network_direction` processor {pull}24620[24620] +- Add Content-Type override to aws-s3 input. {issue}25697[25697] {pull}25772[25772] +- In Cisco Umbrella fileset add users from cisco.umbrella.identities to related.user. {pull}25776[25776] +- Add fingerprint processor to generate fixed ids for `google_workspace` events. {pull}25841[25841] +- Update PanOS module to parse HIP Match logs. {issue}24350[24350] {pull}25686[25686] +- Support MongoDB 4.4 in filebeat's MongoDB module. {issue}20501[20501] {pull}24774[24774] +- Enhance GCP module to populate orchestrator.* fields for GKE / K8S logs {pull}25368[25368] +- Add log_group_name_prefix config into aws-cloudwatch input. {pull}26187[26187] +- Move Filebeat azure module to GA. {pull}26114[26114] {pull}26168[26168] +- http_endpoint: Support multiple documents in a single request by POSTing an array or NDJSON format. {pull}25764[25764] +- Make `filestream` input GA. {pull}26127[26127] +- Add new `parser` to `filestream` input: `container`. {pull}26115[26115] +- Add support for ISO8601 timestamps in Zeek fileset {pull}25564[25564] +- Add possibility to include headers in resulting docs and preserve the original event in http_endpoint input {pull}26279[26279] +- Add `preserve_original_event` option to `o365audit` input. {pull}26273[26273] +- Add `log.flags` to events created by the `aws-s3` input. {pull}26267[26267] +- Add `include_s3_metadata` config option to the `aws-s3` input for including object metadata in events. {pull}26267[26267] +- RFC 5424 and UNIX socket support in the Syslog input are now GA {pull}26293[26293] +- Update grok patterns for HA Proxy module {issue}25827[25827] {pull}25835[25835] +- Update PanOS module's date processor formats to parse `strict_date_optional_time_nanos`. {issue}26033[26033] {pull}26158[26158] +- Update Okta module to parse additional fields to `okta.debug_context.debug_data`. {issue}25689[25689] {pull}25818[25818] +- Added dataset `anomalithreatstream` to the `threatintel` module to ingest indicators from Anomali ThreatStream {pull}26350[26350] +- Add support for `copytruncate` method when rotating input logs with an external tool in `filestream` input. {pull}23457[23457] +- Add `uri_parts` and `user_agent` ingest processors to `aws.elb` module. {issue}26435[26435] {pull}26441[26441] +- Added dataset `recordedfuture` to the `threatintel` module to ingest indicators from Recorded Future Connect API {pull}26481[26481] +- Update `fortinet` ingest pipelines. {issue}22136[22136] {issue}25254[25254] {pull}24816[24816] +- Use default add_locale for fortinet.firewall {issue}20300[20300] {pull}26524[26524] *Heartbeat* diff --git a/filebeat/docs/modules/fortinet.asciidoc b/filebeat/docs/modules/fortinet.asciidoc index 78c8825bdd1..eb6e6a3f014 100644 --- a/filebeat/docs/modules/fortinet.asciidoc +++ b/filebeat/docs/modules/fortinet.asciidoc @@ -27,7 +27,7 @@ include::../include/gs-link.asciidoc[] [float] === Compatibility -This module has been tested against FortiOS version 6.0.x and 6.2.x. +This module has been tested against FortiOS version 6.0.x and 6.2.x. Versions above this are expected to work but have not been tested. include::../include/configuring-intro.asciidoc[] @@ -51,6 +51,8 @@ include::../include/config-option-intro.asciidoc[] include::../include/var-paths.asciidoc[] +include::../include/timezone-support.asciidoc[] + *`var.input`*:: The input to use, can be either the value `tcp`, `udp` or `file`. diff --git a/x-pack/filebeat/module/fortinet/_meta/docs.asciidoc b/x-pack/filebeat/module/fortinet/_meta/docs.asciidoc index acf35ecacc9..b504fc1cbf4 100644 --- a/x-pack/filebeat/module/fortinet/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/fortinet/_meta/docs.asciidoc @@ -22,7 +22,7 @@ include::../include/gs-link.asciidoc[] [float] === Compatibility -This module has been tested against FortiOS version 6.0.x and 6.2.x. +This module has been tested against FortiOS version 6.0.x and 6.2.x. Versions above this are expected to work but have not been tested. include::../include/configuring-intro.asciidoc[] @@ -46,6 +46,8 @@ include::../include/config-option-intro.asciidoc[] include::../include/var-paths.asciidoc[] +include::../include/timezone-support.asciidoc[] + *`var.input`*:: The input to use, can be either the value `tcp`, `udp` or `file`. diff --git a/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml b/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml index f83d8bfe67d..525064cbacc 100644 --- a/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml +++ b/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml @@ -25,6 +25,7 @@ tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} processors: + - add_locale: ~ - add_fields: target: '' fields: diff --git a/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml b/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml index c103fd14700..3a8ed0a8b26 100644 --- a/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml +++ b/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml @@ -47,12 +47,12 @@ processors: ignore_empty_value: true - set: field: _temp.time - value: "{{fortinet.firewall.date}} {{fortinet.firewall.time}} {{fortinet.firewall.tz}}" - if: "ctx.fortinet?.firewall?.tz != null" + value: "{{fortinet.firewall.date}} {{fortinet.firewall.time}} {{event.timezone}}" + if: "ctx.event?.timezone != null" - set: field: _temp.time value: "{{fortinet.firewall.date}} {{fortinet.firewall.time}}" - if: "ctx.fortinet?.firewall?.tz == null" + if: "ctx.event?.timezone == null" - date: field: _temp.time target_field: "@timestamp" @@ -61,8 +61,8 @@ processors: - yyyy-MM-dd HH:mm:ss Z - yyyy-MM-dd HH:mm:ss z - ISO8601 - timezone: "{{fortinet.firewall.tz}}" - if: "ctx.fortinet?.firewall?.tz != null" + timezone: "{{event.timezone}}" + if: "ctx.event?.timezone != null" - date: field: _temp.time target_field: "@timestamp" @@ -71,7 +71,7 @@ processors: - yyyy-MM-dd HH:mm:ss Z - yyyy-MM-dd HH:mm:ss z - ISO8601 - if: "ctx.fortinet?.firewall?.tz == null" + if: "ctx.event?.timezone == null" - gsub: field: fortinet.firewall.eventtime pattern: "\\d{6}$" @@ -82,27 +82,27 @@ processors: target_field: event.start formats: - UNIX_MS - timezone: "{{fortinet.firewall.tz}}" - if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.fortinet?.firewall?.tz != null && (ctx.fortinet?.firewall?.eventtime).length() > 11" + timezone: "{{event.timezone}}" + if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.event?.timezone != null && (ctx.fortinet?.firewall?.eventtime).length() > 11" - date: field: fortinet.firewall.eventtime target_field: event.start formats: - UNIX - timezone: "{{fortinet.firewall.tz}}" - if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.fortinet?.firewall?.tz != null && (ctx.fortinet?.firewall?.eventtime).length() <= 11" + timezone: "{{event.timezone}}" + if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.event?.timezone != null && (ctx.fortinet?.firewall?.eventtime).length() <= 11" - date: field: fortinet.firewall.eventtime target_field: event.start formats: - UNIX_MS - if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.fortinet?.firewall?.tz == null && (ctx.fortinet?.firewall?.eventtime).length() > 11" + if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.event?.timezone == null && (ctx.fortinet?.firewall?.eventtime).length() > 11" - date: field: fortinet.firewall.eventtime target_field: event.start formats: - UNIX - if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.fortinet?.firewall?.tz == null && (ctx.fortinet?.firewall?.eventtime).length() <= 11" + if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.event?.timezone == null && (ctx.fortinet?.firewall?.eventtime).length() <= 11" - script: lang: painless source: "ctx.event.duration = Long.parseLong(ctx.fortinet.firewall.duration) * 1000000000" diff --git a/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json b/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json index 172748796d1..9302e52f727 100644 --- a/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json +++ b/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json @@ -76,7 +76,7 @@ "url.path": "/config/" }, { - "@timestamp": "2020-04-23T01:16:08.000Z", + "@timestamp": "2020-04-23T01:16:08.000-02:00", "destination.as.number": 15169, "destination.as.organization.name": "Google LLC", "destination.bytes": 0, @@ -97,7 +97,8 @@ "event.kind": "event", "event.module": "fortinet", "event.outcome": "success", - "event.start": "2020-06-24T01:16:08.000Z", + "event.start": "2020-06-23T23:16:08.000-02:00", + "event.timezone": "-02:00", "event.type": [ "connection", "end" @@ -1905,7 +1906,7 @@ ] }, { - "@timestamp": "2019-05-15T18:03:36.000Z", + "@timestamp": "2019-05-15T18:03:36.000-02:00", "destination.as.number": 41690, "destination.as.organization.name": "Dailymotion S.A.", "destination.geo.continent_name": "Europe", @@ -1924,7 +1925,8 @@ "event.kind": "event", "event.module": "fortinet", "event.outcome": "success", - "event.start": "2019-05-16T01:03:35.000Z", + "event.start": "2019-05-15T23:03:35.000-02:00", + "event.timezone": "-02:00", "event.type": [ "allowed" ], @@ -1973,7 +1975,7 @@ "url.path": "/" }, { - "@timestamp": "2020-11-02T08:11:38.000Z", + "@timestamp": "2020-11-02T08:11:38.000-02:00", "destination.as.number": 15169, "destination.as.organization.name": "Google LLC", "destination.geo.continent_name": "North America", @@ -1991,6 +1993,7 @@ "event.kind": "event", "event.module": "fortinet", "event.outcome": "success", + "event.timezone": "-02:00", "event.type": [ "connection" ],