[Filebeat] Fix date parsing in GSuite/Google Workspace modules (elast…

…ic#24696) (elastic#24736) * Fix date parsing in GSuite/login fileset The format of Date#toUTCString was incompatible with the format accepted by Elasticsearch by default. By writing a Date object from the JS pipeline this becomes a time.Time in the event that is formatted by common.Time when going out as JSON. Fixes elastic#24694 * Apply fix to google_workspace Fixes elastic#24692 (cherry picked from commit 2a63518)
leweafan · Mar 24, 2021 · 783e39d · 783e39d
1 parent 001aff6
commit 783e39d
Show file tree

Hide file tree

Showing 5 changed files with 437 additions and 5 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -201,6 +201,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix `google_workspace` pagination. {pull}24668[24668]
 - Fix netflow module ignoring detect_sequence_reset flag. {issue}24268[24268] {pull}24270[24270]
 - Fix Cisco ASA parser for message 302022. {issue}24405[24405] {pull}24697[24697]
+- Fix date parsing in GSuite/login and Google Workspace/login filesets. {issue}24694[24694]
 - Fix gcp/vpcflow module error where input type was defaulting to file. {pull}24719[24719]
 
 *Heartbeat*
@@ -209,9 +210,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fixed scheduler shutdown issues which would in rare situations cause a panic due to semaphore misuse. {pull}16397[16397]
 - Fixed TCP TLS checks to properly validate hostnames, this broke in 7.x and only worked for IP SANs. {pull}17549[17549]
 
-*Heartbeat*
-
-
 *Journalbeat*
 
 

diff --git a/x-pack/filebeat/module/google_workspace/login/config/pipeline.js b/x-pack/filebeat/module/google_workspace/login/config/pipeline.js
@@ -64,7 +64,7 @@ var login = (function () {
                 // this is a timestamp in microseconds
                 case "timestamp":
                     var millis = p.intValue / 1000;
-                    evt.Put("event.start", new Date(millis).toUTCString());
+                    evt.Put("event.start", new Date(millis));
                     break;
                 case "challenge_status":
                     if (p.value === "Challenge Passed") {

diff --git a/x-pack/filebeat/module/google_workspace/login/test/login-test.json.log-expected.json b/x-pack/filebeat/module/google_workspace/login/test/login-test.json.log-expected.json
@@ -55,6 +55,174 @@
         "user.target.email": "[email protected]",
         "user.target.name": "foo"
     },
+    {
+        "@timestamp": "2020-10-02T15:00:00.000Z",
+        "event.action": "suspicious_login",
+        "event.category": [
+            "authentication"
+        ],
+        "event.dataset": "google_workspace.login",
+        "event.id": "1",
+        "event.module": "google_workspace",
+        "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"login\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"[email protected]\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"account_warning\",\"name\":\"suspicious_login\",\"parameters\":[{\"name\":\"affected_email_address\",\"value\":\"[email protected]\"},{\"name\":\"login_timestamp\",\"intValue\":1593695305123456}]}}",
+        "event.provider": "login",
+        "event.start": "2020-07-02T13:08:25.123Z",
+        "event.type": [
+            "info"
+        ],
+        "fileset.name": "login",
+        "google_workspace.actor.type": "USER",
+        "google_workspace.event.type": "account_warning",
+        "google_workspace.kind": "admin#reports#activity",
+        "google_workspace.login.affected_email_address": "[email protected]",
+        "google_workspace.organization.domain": "elastic.com",
+        "input.type": "log",
+        "log.offset": 406,
+        "organization.id": "1",
+        "related.ip": [
+            "98.235.162.24"
+        ],
+        "related.user": [
+            "foo"
+        ],
+        "service.type": "google_workspace",
+        "source.as.number": 7922,
+        "source.as.organization.name": "Comcast Cable Communications, LLC",
+        "source.geo.city_name": "State College",
+        "source.geo.continent_name": "North America",
+        "source.geo.country_iso_code": "US",
+        "source.geo.country_name": "United States",
+        "source.geo.location.lat": 40.7957,
+        "source.geo.location.lon": -77.8618,
+        "source.geo.region_iso_code": "US-PA",
+        "source.geo.region_name": "Pennsylvania",
+        "source.ip": "98.235.162.24",
+        "source.user.domain": "bar.com",
+        "source.user.email": "[email protected]",
+        "source.user.id": "1",
+        "source.user.name": "foo",
+        "tags": [
+            "forwarded"
+        ],
+        "user.domain": "bar.com",
+        "user.id": "1",
+        "user.name": "foo",
+        "user.target.domain": "elastic.co",
+        "user.target.email": "[email protected]",
+        "user.target.name": "foo"
+    },
+    {
+        "@timestamp": "2020-10-02T15:00:00.000Z",
+        "event.action": "suspicious_login_less_secure_app",
+        "event.category": [
+            "authentication"
+        ],
+        "event.dataset": "google_workspace.login",
+        "event.id": "1",
+        "event.module": "google_workspace",
+        "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"login\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"[email protected]\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"account_warning\",\"name\":\"suspicious_login_less_secure_app\",\"parameters\":[{\"name\":\"affected_email_address\",\"value\":\"[email protected]\"},{\"name\":\"login_timestamp\",\"intValue\":1593695305123456}]}}",
+        "event.provider": "login",
+        "event.start": "2020-07-02T13:08:25.123Z",
+        "event.type": [
+            "info"
+        ],
+        "fileset.name": "login",
+        "google_workspace.actor.type": "USER",
+        "google_workspace.event.type": "account_warning",
+        "google_workspace.kind": "admin#reports#activity",
+        "google_workspace.login.affected_email_address": "[email protected]",
+        "google_workspace.organization.domain": "elastic.com",
+        "input.type": "log",
+        "log.offset": 853,
+        "organization.id": "1",
+        "related.ip": [
+            "98.235.162.24"
+        ],
+        "related.user": [
+            "foo"
+        ],
+        "service.type": "google_workspace",
+        "source.as.number": 7922,
+        "source.as.organization.name": "Comcast Cable Communications, LLC",
+        "source.geo.city_name": "State College",
+        "source.geo.continent_name": "North America",
+        "source.geo.country_iso_code": "US",
+        "source.geo.country_name": "United States",
+        "source.geo.location.lat": 40.7957,
+        "source.geo.location.lon": -77.8618,
+        "source.geo.region_iso_code": "US-PA",
+        "source.geo.region_name": "Pennsylvania",
+        "source.ip": "98.235.162.24",
+        "source.user.domain": "bar.com",
+        "source.user.email": "[email protected]",
+        "source.user.id": "1",
+        "source.user.name": "foo",
+        "tags": [
+            "forwarded"
+        ],
+        "user.domain": "bar.com",
+        "user.id": "1",
+        "user.name": "foo",
+        "user.target.domain": "elastic.co",
+        "user.target.email": "[email protected]",
+        "user.target.name": "foo"
+    },
+    {
+        "@timestamp": "2020-10-02T15:00:00.000Z",
+        "event.action": "suspicious_programmatic_login",
+        "event.category": [
+            "authentication"
+        ],
+        "event.dataset": "google_workspace.login",
+        "event.id": "1",
+        "event.module": "google_workspace",
+        "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"login\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"[email protected]\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"account_warning\",\"name\":\"suspicious_programmatic_login\",\"parameters\":[{\"name\":\"affected_email_address\",\"value\":\"[email protected]\"},{\"name\":\"login_timestamp\",\"intValue\":1593695305123456}]}}",
+        "event.provider": "login",
+        "event.start": "2020-07-02T13:08:25.123Z",
+        "event.type": [
+            "info"
+        ],
+        "fileset.name": "login",
+        "google_workspace.actor.type": "USER",
+        "google_workspace.event.type": "account_warning",
+        "google_workspace.kind": "admin#reports#activity",
+        "google_workspace.login.affected_email_address": "[email protected]",
+        "google_workspace.organization.domain": "elastic.com",
+        "input.type": "log",
+        "log.offset": 1316,
+        "organization.id": "1",
+        "related.ip": [
+            "98.235.162.24"
+        ],
+        "related.user": [
+            "foo"
+        ],
+        "service.type": "google_workspace",
+        "source.as.number": 7922,
+        "source.as.organization.name": "Comcast Cable Communications, LLC",
+        "source.geo.city_name": "State College",
+        "source.geo.continent_name": "North America",
+        "source.geo.country_iso_code": "US",
+        "source.geo.country_name": "United States",
+        "source.geo.location.lat": 40.7957,
+        "source.geo.location.lon": -77.8618,
+        "source.geo.region_iso_code": "US-PA",
+        "source.geo.region_name": "Pennsylvania",
+        "source.ip": "98.235.162.24",
+        "source.user.domain": "bar.com",
+        "source.user.email": "[email protected]",
+        "source.user.id": "1",
+        "source.user.name": "foo",
+        "tags": [
+            "forwarded"
+        ],
+        "user.domain": "bar.com",
+        "user.id": "1",
+        "user.name": "foo",
+        "user.target.domain": "elastic.co",
+        "user.target.email": "[email protected]",
+        "user.target.name": "foo"
+    },
     {
         "@timestamp": "2020-10-02T15:00:00.000Z",
         "event.action": "account_disabled_generic",
@@ -223,6 +391,63 @@
         "user.target.email": "[email protected]",
         "user.target.name": "foo"
     },
+    {
+        "@timestamp": "2020-10-02T15:00:00.000Z",
+        "event.action": "account_disabled_hijacked",
+        "event.category": [
+            "authentication"
+        ],
+        "event.dataset": "google_workspace.login",
+        "event.id": "1",
+        "event.module": "google_workspace",
+        "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"login\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"[email protected]\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"account_warning\",\"name\":\"account_disabled_hijacked\",\"parameters\":[{\"name\":\"affected_email_address\",\"value\":\"[email protected]\"},{\"name\":\"login_timestamp\",\"intValue\":1593695305123456}]}}",
+        "event.provider": "login",
+        "event.start": "2020-07-02T13:08:25.123Z",
+        "event.type": [
+            "user",
+            "change"
+        ],
+        "fileset.name": "login",
+        "google_workspace.actor.type": "USER",
+        "google_workspace.event.type": "account_warning",
+        "google_workspace.kind": "admin#reports#activity",
+        "google_workspace.login.affected_email_address": "[email protected]",
+        "google_workspace.organization.domain": "elastic.com",
+        "input.type": "log",
+        "log.offset": 2992,
+        "organization.id": "1",
+        "related.ip": [
+            "98.235.162.24"
+        ],
+        "related.user": [
+            "foo"
+        ],
+        "service.type": "google_workspace",
+        "source.as.number": 7922,
+        "source.as.organization.name": "Comcast Cable Communications, LLC",
+        "source.geo.city_name": "State College",
+        "source.geo.continent_name": "North America",
+        "source.geo.country_iso_code": "US",
+        "source.geo.country_name": "United States",
+        "source.geo.location.lat": 40.7957,
+        "source.geo.location.lon": -77.8618,
+        "source.geo.region_iso_code": "US-PA",
+        "source.geo.region_name": "Pennsylvania",
+        "source.ip": "98.235.162.24",
+        "source.user.domain": "bar.com",
+        "source.user.email": "[email protected]",
+        "source.user.id": "1",
+        "source.user.name": "foo",
+        "tags": [
+            "forwarded"
+        ],
+        "user.domain": "bar.com",
+        "user.id": "1",
+        "user.name": "foo",
+        "user.target.domain": "elastic.co",
+        "user.target.email": "[email protected]",
+        "user.target.name": "foo"
+    },
     {
         "@timestamp": "2020-10-02T15:00:00.000Z",
         "event.action": "gov_attack_warning",

diff --git a/x-pack/filebeat/module/gsuite/login/config/pipeline.js b/x-pack/filebeat/module/gsuite/login/config/pipeline.js
@@ -64,7 +64,7 @@ var login = (function () {
                 // this is a timestamp in microseconds
                 case "timestamp":
                     var millis = p.intValue / 1000;
-                    evt.Put("event.start", new Date(millis).toUTCString());
+                    evt.Put("event.start", new Date(millis));
                     break;
                 case "challenge_status":
                     if (p.value === "Challenge Passed") {