From 1aa5c06761072641a7738d4b5a2136cbde4a94af Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 23 Mar 2021 19:32:54 -0400 Subject: [PATCH] [Filebeat] Fix date parsing in GSuite/Google Workspace modules (#24696) * Fix date parsing in GSuite/login fileset The format of Date#toUTCString was incompatible with the format accepted by Elasticsearch by default. By writing a Date object from the JS pipeline this becomes a time.Time in the event that is formatted by common.Time when going out as JSON. Fixes #24694 * Apply fix to google_workspace Fixes #24692 (cherry picked from commit a4a3ff0c8f85c9b4e8e00cda57af6036b3a66678) --- CHANGELOG.next.asciidoc | 4 +- .../google_workspace/login/config/pipeline.js | 2 +- .../test/login-test.json.log-expected.json | 225 ++++++++++++++++++ .../module/gsuite/login/config/pipeline.js | 2 +- .../gsuite-login-test.json.log-expected.json | 209 ++++++++++++++++ 5 files changed, 437 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 59d04abf10f..943fab85333 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -222,6 +222,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix `google_workspace` pagination. {pull}24668[24668] - Fix netflow module ignoring detect_sequence_reset flag. {issue}24268[24268] {pull}24270[24270] - Fix Cisco ASA parser for message 302022. {issue}24405[24405] {pull}24697[24697] +- Fix date parsing in GSuite/login and Google Workspace/login filesets. {issue}24694[24694] *Heartbeat* @@ -229,9 +230,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fixed scheduler shutdown issues which would in rare situations cause a panic due to semaphore misuse. {pull}16397[16397] - Fixed TCP TLS checks to properly validate hostnames, this broke in 7.x and only worked for IP SANs. {pull}17549[17549] -*Heartbeat* - - *Journalbeat* diff --git a/x-pack/filebeat/module/google_workspace/login/config/pipeline.js b/x-pack/filebeat/module/google_workspace/login/config/pipeline.js index 9f9610393f1..a7b54afd43e 100644 --- a/x-pack/filebeat/module/google_workspace/login/config/pipeline.js +++ b/x-pack/filebeat/module/google_workspace/login/config/pipeline.js @@ -64,7 +64,7 @@ var login = (function () { // this is a timestamp in microseconds case "timestamp": var millis = p.intValue / 1000; - evt.Put("event.start", new Date(millis).toUTCString()); + evt.Put("event.start", new Date(millis)); break; case "challenge_status": if (p.value === "Challenge Passed") { diff --git a/x-pack/filebeat/module/google_workspace/login/test/login-test.json.log-expected.json b/x-pack/filebeat/module/google_workspace/login/test/login-test.json.log-expected.json index 48f7038df80..a4e0f480040 100644 --- a/x-pack/filebeat/module/google_workspace/login/test/login-test.json.log-expected.json +++ b/x-pack/filebeat/module/google_workspace/login/test/login-test.json.log-expected.json @@ -55,6 +55,174 @@ "user.target.email": "foo@elastic.co", "user.target.name": "foo" }, + { + "@timestamp": "2020-10-02T15:00:00.000Z", + "event.action": "suspicious_login", + "event.category": [ + "authentication" + ], + "event.dataset": "google_workspace.login", + "event.id": "1", + "event.module": "google_workspace", + "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"login\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"account_warning\",\"name\":\"suspicious_login\",\"parameters\":[{\"name\":\"affected_email_address\",\"value\":\"foo@elastic.co\"},{\"name\":\"login_timestamp\",\"intValue\":1593695305123456}]}}", + "event.provider": "login", + "event.start": "2020-07-02T13:08:25.123Z", + "event.type": [ + "info" + ], + "fileset.name": "login", + "google_workspace.actor.type": "USER", + "google_workspace.event.type": "account_warning", + "google_workspace.kind": "admin#reports#activity", + "google_workspace.login.affected_email_address": "foo@elastic.co", + "google_workspace.organization.domain": "elastic.com", + "input.type": "log", + "log.offset": 406, + "organization.id": "1", + "related.ip": [ + "98.235.162.24" + ], + "related.user": [ + "foo" + ], + "service.type": "google_workspace", + "source.as.number": 7922, + "source.as.organization.name": "Comcast Cable Communications, LLC", + "source.geo.city_name": "State College", + "source.geo.continent_name": "North America", + "source.geo.country_iso_code": "US", + "source.geo.country_name": "United States", + "source.geo.location.lat": 40.7957, + "source.geo.location.lon": -77.8618, + "source.geo.region_iso_code": "US-PA", + "source.geo.region_name": "Pennsylvania", + "source.ip": "98.235.162.24", + "source.user.domain": "bar.com", + "source.user.email": "foo@bar.com", + "source.user.id": "1", + "source.user.name": "foo", + "tags": [ + "forwarded" + ], + "user.domain": "bar.com", + "user.id": "1", + "user.name": "foo", + "user.target.domain": "elastic.co", + "user.target.email": "foo@elastic.co", + "user.target.name": "foo" + }, + { + "@timestamp": "2020-10-02T15:00:00.000Z", + "event.action": "suspicious_login_less_secure_app", + "event.category": [ + "authentication" + ], + "event.dataset": "google_workspace.login", + "event.id": "1", + "event.module": "google_workspace", + "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"login\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"account_warning\",\"name\":\"suspicious_login_less_secure_app\",\"parameters\":[{\"name\":\"affected_email_address\",\"value\":\"foo@elastic.co\"},{\"name\":\"login_timestamp\",\"intValue\":1593695305123456}]}}", + "event.provider": "login", + "event.start": "2020-07-02T13:08:25.123Z", + "event.type": [ + "info" + ], + "fileset.name": "login", + "google_workspace.actor.type": "USER", + "google_workspace.event.type": "account_warning", + "google_workspace.kind": "admin#reports#activity", + "google_workspace.login.affected_email_address": "foo@elastic.co", + "google_workspace.organization.domain": "elastic.com", + "input.type": "log", + "log.offset": 853, + "organization.id": "1", + "related.ip": [ + "98.235.162.24" + ], + "related.user": [ + "foo" + ], + "service.type": "google_workspace", + "source.as.number": 7922, + "source.as.organization.name": "Comcast Cable Communications, LLC", + "source.geo.city_name": "State College", + "source.geo.continent_name": "North America", + "source.geo.country_iso_code": "US", + "source.geo.country_name": "United States", + "source.geo.location.lat": 40.7957, + "source.geo.location.lon": -77.8618, + "source.geo.region_iso_code": "US-PA", + "source.geo.region_name": "Pennsylvania", + "source.ip": "98.235.162.24", + "source.user.domain": "bar.com", + "source.user.email": "foo@bar.com", + "source.user.id": "1", + "source.user.name": "foo", + "tags": [ + "forwarded" + ], + "user.domain": "bar.com", + "user.id": "1", + "user.name": "foo", + "user.target.domain": "elastic.co", + "user.target.email": "foo@elastic.co", + "user.target.name": "foo" + }, + { + "@timestamp": "2020-10-02T15:00:00.000Z", + "event.action": "suspicious_programmatic_login", + "event.category": [ + "authentication" + ], + "event.dataset": "google_workspace.login", + "event.id": "1", + "event.module": "google_workspace", + "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"login\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"account_warning\",\"name\":\"suspicious_programmatic_login\",\"parameters\":[{\"name\":\"affected_email_address\",\"value\":\"foo@elastic.co\"},{\"name\":\"login_timestamp\",\"intValue\":1593695305123456}]}}", + "event.provider": "login", + "event.start": "2020-07-02T13:08:25.123Z", + "event.type": [ + "info" + ], + "fileset.name": "login", + "google_workspace.actor.type": "USER", + "google_workspace.event.type": "account_warning", + "google_workspace.kind": "admin#reports#activity", + "google_workspace.login.affected_email_address": "foo@elastic.co", + "google_workspace.organization.domain": "elastic.com", + "input.type": "log", + "log.offset": 1316, + "organization.id": "1", + "related.ip": [ + "98.235.162.24" + ], + "related.user": [ + "foo" + ], + "service.type": "google_workspace", + "source.as.number": 7922, + "source.as.organization.name": "Comcast Cable Communications, LLC", + "source.geo.city_name": "State College", + "source.geo.continent_name": "North America", + "source.geo.country_iso_code": "US", + "source.geo.country_name": "United States", + "source.geo.location.lat": 40.7957, + "source.geo.location.lon": -77.8618, + "source.geo.region_iso_code": "US-PA", + "source.geo.region_name": "Pennsylvania", + "source.ip": "98.235.162.24", + "source.user.domain": "bar.com", + "source.user.email": "foo@bar.com", + "source.user.id": "1", + "source.user.name": "foo", + "tags": [ + "forwarded" + ], + "user.domain": "bar.com", + "user.id": "1", + "user.name": "foo", + "user.target.domain": "elastic.co", + "user.target.email": "foo@elastic.co", + "user.target.name": "foo" + }, { "@timestamp": "2020-10-02T15:00:00.000Z", "event.action": "account_disabled_generic", @@ -223,6 +391,63 @@ "user.target.email": "foo@elastic.co", "user.target.name": "foo" }, + { + "@timestamp": "2020-10-02T15:00:00.000Z", + "event.action": "account_disabled_hijacked", + "event.category": [ + "authentication" + ], + "event.dataset": "google_workspace.login", + "event.id": "1", + "event.module": "google_workspace", + "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"login\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"account_warning\",\"name\":\"account_disabled_hijacked\",\"parameters\":[{\"name\":\"affected_email_address\",\"value\":\"foo@elastic.co\"},{\"name\":\"login_timestamp\",\"intValue\":1593695305123456}]}}", + "event.provider": "login", + "event.start": "2020-07-02T13:08:25.123Z", + "event.type": [ + "user", + "change" + ], + "fileset.name": "login", + "google_workspace.actor.type": "USER", + "google_workspace.event.type": "account_warning", + "google_workspace.kind": "admin#reports#activity", + "google_workspace.login.affected_email_address": "foo@elastic.co", + "google_workspace.organization.domain": "elastic.com", + "input.type": "log", + "log.offset": 2992, + "organization.id": "1", + "related.ip": [ + "98.235.162.24" + ], + "related.user": [ + "foo" + ], + "service.type": "google_workspace", + "source.as.number": 7922, + "source.as.organization.name": "Comcast Cable Communications, LLC", + "source.geo.city_name": "State College", + "source.geo.continent_name": "North America", + "source.geo.country_iso_code": "US", + "source.geo.country_name": "United States", + "source.geo.location.lat": 40.7957, + "source.geo.location.lon": -77.8618, + "source.geo.region_iso_code": "US-PA", + "source.geo.region_name": "Pennsylvania", + "source.ip": "98.235.162.24", + "source.user.domain": "bar.com", + "source.user.email": "foo@bar.com", + "source.user.id": "1", + "source.user.name": "foo", + "tags": [ + "forwarded" + ], + "user.domain": "bar.com", + "user.id": "1", + "user.name": "foo", + "user.target.domain": "elastic.co", + "user.target.email": "foo@elastic.co", + "user.target.name": "foo" + }, { "@timestamp": "2020-10-02T15:00:00.000Z", "event.action": "gov_attack_warning", diff --git a/x-pack/filebeat/module/gsuite/login/config/pipeline.js b/x-pack/filebeat/module/gsuite/login/config/pipeline.js index 0fb518b351d..2ad5d52f7de 100644 --- a/x-pack/filebeat/module/gsuite/login/config/pipeline.js +++ b/x-pack/filebeat/module/gsuite/login/config/pipeline.js @@ -64,7 +64,7 @@ var login = (function () { // this is a timestamp in microseconds case "timestamp": var millis = p.intValue / 1000; - evt.Put("event.start", new Date(millis).toUTCString()); + evt.Put("event.start", new Date(millis)); break; case "challenge_status": if (p.value === "Challenge Passed") { diff --git a/x-pack/filebeat/module/gsuite/login/test/gsuite-login-test.json.log-expected.json b/x-pack/filebeat/module/gsuite/login/test/gsuite-login-test.json.log-expected.json index 261bf54dbf6..9bc77dc7d03 100644 --- a/x-pack/filebeat/module/gsuite/login/test/gsuite-login-test.json.log-expected.json +++ b/x-pack/filebeat/module/gsuite/login/test/gsuite-login-test.json.log-expected.json @@ -51,6 +51,162 @@ "user.id": "1", "user.name": "foo" }, + { + "event.action": "suspicious_login", + "event.category": [ + "authentication" + ], + "event.dataset": "gsuite.login", + "event.id": "1", + "event.module": "gsuite", + "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"login\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"account_warning\",\"name\":\"suspicious_login\",\"parameters\":[{\"name\":\"affected_email_address\",\"value\":\"foo@elastic.co\"},{\"name\":\"login_timestamp\",\"intValue\":1593695305123456}]}}", + "event.provider": "login", + "event.start": "2020-07-02T13:08:25.123Z", + "event.type": [ + "info" + ], + "fileset.name": "login", + "gsuite.actor.type": "USER", + "gsuite.event.type": "account_warning", + "gsuite.kind": "admin#reports#activity", + "gsuite.login.affected_email_address": "foo@elastic.co", + "gsuite.organization.domain": "elastic.com", + "input.type": "log", + "log.offset": 406, + "organization.id": "1", + "related.ip": [ + "98.235.162.24" + ], + "related.user": [ + "foo" + ], + "service.type": "gsuite", + "source.as.number": 7922, + "source.as.organization.name": "Comcast Cable Communications, LLC", + "source.geo.city_name": "State College", + "source.geo.continent_name": "North America", + "source.geo.country_iso_code": "US", + "source.geo.country_name": "United States", + "source.geo.location.lat": 40.7957, + "source.geo.location.lon": -77.8618, + "source.geo.region_iso_code": "US-PA", + "source.geo.region_name": "Pennsylvania", + "source.ip": "98.235.162.24", + "source.user.domain": "bar.com", + "source.user.email": "foo@bar.com", + "source.user.id": "1", + "source.user.name": "foo", + "tags": [ + "forwarded" + ], + "user.domain": "bar.com", + "user.id": "1", + "user.name": "foo" + }, + { + "event.action": "suspicious_login_less_secure_app", + "event.category": [ + "authentication" + ], + "event.dataset": "gsuite.login", + "event.id": "1", + "event.module": "gsuite", + "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"login\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"account_warning\",\"name\":\"suspicious_login_less_secure_app\",\"parameters\":[{\"name\":\"affected_email_address\",\"value\":\"foo@elastic.co\"},{\"name\":\"login_timestamp\",\"intValue\":1593695305123456}]}}", + "event.provider": "login", + "event.start": "2020-07-02T13:08:25.123Z", + "event.type": [ + "info" + ], + "fileset.name": "login", + "gsuite.actor.type": "USER", + "gsuite.event.type": "account_warning", + "gsuite.kind": "admin#reports#activity", + "gsuite.login.affected_email_address": "foo@elastic.co", + "gsuite.organization.domain": "elastic.com", + "input.type": "log", + "log.offset": 853, + "organization.id": "1", + "related.ip": [ + "98.235.162.24" + ], + "related.user": [ + "foo" + ], + "service.type": "gsuite", + "source.as.number": 7922, + "source.as.organization.name": "Comcast Cable Communications, LLC", + "source.geo.city_name": "State College", + "source.geo.continent_name": "North America", + "source.geo.country_iso_code": "US", + "source.geo.country_name": "United States", + "source.geo.location.lat": 40.7957, + "source.geo.location.lon": -77.8618, + "source.geo.region_iso_code": "US-PA", + "source.geo.region_name": "Pennsylvania", + "source.ip": "98.235.162.24", + "source.user.domain": "bar.com", + "source.user.email": "foo@bar.com", + "source.user.id": "1", + "source.user.name": "foo", + "tags": [ + "forwarded" + ], + "user.domain": "bar.com", + "user.id": "1", + "user.name": "foo" + }, + { + "event.action": "suspicious_programmatic_login", + "event.category": [ + "authentication" + ], + "event.dataset": "gsuite.login", + "event.id": "1", + "event.module": "gsuite", + "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"login\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"account_warning\",\"name\":\"suspicious_programmatic_login\",\"parameters\":[{\"name\":\"affected_email_address\",\"value\":\"foo@elastic.co\"},{\"name\":\"login_timestamp\",\"intValue\":1593695305123456}]}}", + "event.provider": "login", + "event.start": "2020-07-02T13:08:25.123Z", + "event.type": [ + "info" + ], + "fileset.name": "login", + "gsuite.actor.type": "USER", + "gsuite.event.type": "account_warning", + "gsuite.kind": "admin#reports#activity", + "gsuite.login.affected_email_address": "foo@elastic.co", + "gsuite.organization.domain": "elastic.com", + "input.type": "log", + "log.offset": 1316, + "organization.id": "1", + "related.ip": [ + "98.235.162.24" + ], + "related.user": [ + "foo" + ], + "service.type": "gsuite", + "source.as.number": 7922, + "source.as.organization.name": "Comcast Cable Communications, LLC", + "source.geo.city_name": "State College", + "source.geo.continent_name": "North America", + "source.geo.country_iso_code": "US", + "source.geo.country_name": "United States", + "source.geo.location.lat": 40.7957, + "source.geo.location.lon": -77.8618, + "source.geo.region_iso_code": "US-PA", + "source.geo.region_name": "Pennsylvania", + "source.ip": "98.235.162.24", + "source.user.domain": "bar.com", + "source.user.email": "foo@bar.com", + "source.user.id": "1", + "source.user.name": "foo", + "tags": [ + "forwarded" + ], + "user.domain": "bar.com", + "user.id": "1", + "user.name": "foo" + }, { "event.action": "account_disabled_generic", "event.category": [ @@ -207,6 +363,59 @@ "user.id": "1", "user.name": "foo" }, + { + "event.action": "account_disabled_hijacked", + "event.category": [ + "authentication" + ], + "event.dataset": "gsuite.login", + "event.id": "1", + "event.module": "gsuite", + "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"login\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"account_warning\",\"name\":\"account_disabled_hijacked\",\"parameters\":[{\"name\":\"affected_email_address\",\"value\":\"foo@elastic.co\"},{\"name\":\"login_timestamp\",\"intValue\":1593695305123456}]}}", + "event.provider": "login", + "event.start": "2020-07-02T13:08:25.123Z", + "event.type": [ + "user", + "change" + ], + "fileset.name": "login", + "gsuite.actor.type": "USER", + "gsuite.event.type": "account_warning", + "gsuite.kind": "admin#reports#activity", + "gsuite.login.affected_email_address": "foo@elastic.co", + "gsuite.organization.domain": "elastic.com", + "input.type": "log", + "log.offset": 2992, + "organization.id": "1", + "related.ip": [ + "98.235.162.24" + ], + "related.user": [ + "foo" + ], + "service.type": "gsuite", + "source.as.number": 7922, + "source.as.organization.name": "Comcast Cable Communications, LLC", + "source.geo.city_name": "State College", + "source.geo.continent_name": "North America", + "source.geo.country_iso_code": "US", + "source.geo.country_name": "United States", + "source.geo.location.lat": 40.7957, + "source.geo.location.lon": -77.8618, + "source.geo.region_iso_code": "US-PA", + "source.geo.region_name": "Pennsylvania", + "source.ip": "98.235.162.24", + "source.user.domain": "bar.com", + "source.user.email": "foo@bar.com", + "source.user.id": "1", + "source.user.name": "foo", + "tags": [ + "forwarded" + ], + "user.domain": "bar.com", + "user.id": "1", + "user.name": "foo" + }, { "event.action": "gov_attack_warning", "event.category": [