[Filebeat][Gsuite] Transform all dates to timestamp with processor (e…

…lastic#20308) (elastic#20319) * Transform all dates to timestamp with processor * Change parse date function to create a chain (cherry picked from commit 8eb5d14)
leweafan · Jul 29, 2020 · 07b60b8 · 07b60b8
1 parent 0df5d78
commit 07b60b8
Show file tree

Hide file tree

Showing 5 changed files with 80 additions and 56 deletions.
diff --git a/x-pack/filebeat/module/gsuite/admin/config/pipeline.js b/x-pack/filebeat/module/gsuite/admin/config/pipeline.js
@@ -421,11 +421,7 @@ var login = (function () {
             return;
         }
 
-        var millisToNano = 1e6;
-        var tsStart = Date.parse(start) * millisToNano;
-        var tsEnd = Date.parse(end) * millisToNano;
-
-        evt.Put("event.duration", tsEnd-tsStart);
+        evt.Put("event.duration", end.UnixNano() - start.UnixNano());
     };
 
     var setEventOutcome = function(evt) {
@@ -451,6 +447,34 @@ var login = (function () {
         evt.Delete("gsuite.admin.WHITELISTED_GROUPS");
     };
 
+    var deleteField = function(field) {
+        return function(evt) {
+            evt.Delete(field);
+        };
+    };
+
+    var parseDate = function(field, targetField) {
+        return new processor.Chain()
+            .Add(new processor.Timestamp({
+                field: field,
+                target_field: targetField,
+                timezone: "UTC",
+                layouts: [
+                    "2006-01-02T15:04:05Z",
+                    "2006-01-02T15:04:05.999Z",
+                    "2006/01/02 15:04:05 UTC",
+                ],
+                tests: [
+                    "2020-02-05T18:19:23Z",
+                    "2020-02-05T18:19:23.599Z",
+                    "2020/07/28 04:59:59 UTC",
+                ],
+                ignore_missing: true,
+            }))
+            .Add(deleteField(field))
+            .Build()
+    };
+
     var pipeline = new processor.Chain()
         .Add(categorizeEvent)
         .Add(flattenParams)
@@ -621,22 +645,6 @@ var login = (function () {
                     from: "gsuite.admin.PRIVILEGE_NAME",
                     to: "gsuite.admin.privilege.name",
                 },
-                {
-                    from: "gsuite.admin.BEGIN_DATE_TIME",
-                    to: "event.start",
-                },
-                {
-                    from: "gsuite.admin.END_DATE_TIME",
-                    to: "event.end",
-                },
-                {
-                    from: "gsuite.admin.START_DATE",
-                    to: "event.start",
-                },
-                {
-                    from: "gsuite.admin.END_DATE",
-                    to: "event.end",
-                },
                 {
                     from: "gsuite.admin.SITE_LOCATION",
                     to: "url.path",
@@ -685,10 +693,6 @@ var login = (function () {
                     from: "gsuite.admin.EMAIL_LOG_SEARCH_MSG_ID",
                     to: "gsuite.admin.email.log_search_filter.message_id",
                 },
-                {
-                    from: "gsuite.admin.EMAIL_LOG_SEARCH_END_DATE",
-                    to: "gsuite.admin.email.log_search_filter.end_date",
-                },
                 {
                     from: "gsuite.admin.EMAIL_LOG_SEARCH_RECIPIENT",
                     to: "gsuite.admin.email.log_search_filter.recipient.value",
@@ -707,10 +711,6 @@ var login = (function () {
                     to: "gsuite.admin.email.log_search_filter.sender.ip",
                     type: "ip",
                 },
-                {
-                    from: "gsuite.admin.EMAIL_LOG_SEARCH_START_DATE",
-                    to: "gsuite.admin.email.log_search_filter.start_date",
-                },
                 {
                     from: "gsuite.admin.QUARANTINE_NAME",
                     to: "gsuite.admin.email.quarantine_name",
@@ -847,10 +847,6 @@ var login = (function () {
                     from: "gsuite.admin.USER_NICKNAME",
                     to: "gsuite.admin.user.nickname",
                 },
-                {
-                    from: "gsuite.admin.BIRTHDATE",
-                    to: "gsuite.admin.user.birthdate",
-                },
                 {
                     from: "gsuite.admin.ACTION_ID",
                     to: "gsuite.admin.mobile.action.id",
@@ -905,6 +901,34 @@ var login = (function () {
             ignore_missing: true,
             fail_on_error: false,
         })
+        .Add(parseDate(
+            "gsuite.admin.EMAIL_LOG_SEARCH_END_DATE",
+            "gsuite.admin.email.log_search_filter.end_date"
+        ))
+        .Add(parseDate(
+            "gsuite.admin.EMAIL_LOG_SEARCH_START_DATE",
+            "gsuite.admin.email.log_search_filter.start_date"
+        ))
+        .Add(parseDate(
+            "gsuite.admin.BIRTHDATE",
+            "gsuite.admin.user.birthdate"
+        ))
+        .Add(parseDate(
+            "gsuite.admin.BEGIN_DATE_TIME",
+            "event.start"
+        ))
+        .Add(parseDate(
+            "gsuite.admin.START_DATE",
+            "event.start"
+        ))
+        .Add(parseDate(
+            "gsuite.admin.END_DATE",
+            "event.end"
+        ))
+        .Add(parseDate(
+            "gsuite.admin.END_DATE_TIME",
+            "event.end"
+        ))
         .Add(setGroupInfo)
         .Add(setRelatedUserInfo)
         .Add(setEventDuration)

diff --git a/x-pack/filebeat/module/gsuite/admin/test/gsuite-admin-docs-test.json.log-expected.json b/x-pack/filebeat/module/gsuite/admin/test/gsuite-admin-docs-test.json.log-expected.json
@@ -58,12 +58,12 @@
         ],
         "event.dataset": "gsuite.admin",
         "event.duration": 10800000000000,
-        "event.end": "2002-10-02T15:00:00Z",
+        "event.end": "2002-10-02T15:00:00.000Z",
         "event.id": "1",
         "event.module": "gsuite",
         "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"admin\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"[email protected]\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"DOCS_SETTINGS\",\"name\":\"DRIVE_DATA_RESTORE\",\"parameters\":[{\"name\":\"BEGIN_DATE_TIME\",\"value\":\"2002-10-02T12:00:00Z\"},{\"name\":\"END_DATE_TIME\",\"value\":\"2002-10-02T15:00:00Z\"},{\"name\":\"USER_EMAIL\",\"value\":\"[email protected]\"}]}}",
         "event.provider": "admin",
-        "event.start": "2002-10-02T12:00:00Z",
+        "event.start": "2002-10-02T12:00:00.000Z",
         "event.type": [
             "info"
         ],
@@ -157,4 +157,4 @@
             "forwarded"
         ]
     }
-]
+]
diff --git a/x-pack/filebeat/module/gsuite/admin/test/gsuite-admin-gmail-test.json.log b/x-pack/filebeat/module/gsuite/admin/test/gsuite-admin-gmail-test.json.log
@@ -1,5 +1,5 @@
 {"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"admin","customerId":"1"},"actor":{"callerType":"USER","email":"[email protected]","profileId":1},"ownerDomain":"elastic.com","ipAddress":"98.235.162.24","events":{"type":"EMAIL_SETTINGS","name":"DROP_FROM_QUARANTINE","parameters":[{"name":"EMAIL_LOG_SEARCH_MSG_ID","value":"id"},{"name":"QUARANTINE_NAME","value":"quarantine"}]}}
-{"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"admin","customerId":"1"},"actor":{"callerType":"USER","email":"[email protected]","profileId":1},"ownerDomain":"elastic.com","ipAddress":"98.235.162.24","events":{"type":"EMAIL_SETTINGS","name":"EMAIL_LOG_SEARCH","parameters":[{"name":"EMAIL_LOG_SEARCH_END_DATE","value":"2002-10-02T12:00:00Z"},{"name":"EMAIL_LOG_SEARCH_MSG_ID","value":"id"},{"name":"EMAIL_LOG_SEARCH_RECIPIENT","value":"recipient"},{"name":"EMAIL_LOG_SEARCH_SENDER","value":"sender"},{"name":"EMAIL_LOG_SEARCH_SMTP_RECIPIENT_IP","value":"1.1.1.1"},{"name":"EMAIL_LOG_SEARCH_SMTP_SENDER_IP","value":"1.1.1.1"},{"name":"EMAIL_LOG_SEARCH_START_DATE","value":"2002-10-02T10:00:00Z"}]}}
+{"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"admin","customerId":"1"},"actor":{"callerType":"USER","email":"[email protected]","profileId":1},"ownerDomain":"elastic.com","ipAddress":"98.235.162.24","events":{"type":"EMAIL_SETTINGS","name":"EMAIL_LOG_SEARCH","parameters":[{"name":"EMAIL_LOG_SEARCH_END_DATE","value":"2020/07/28 04:59:59 UTC"},{"name":"EMAIL_LOG_SEARCH_MSG_ID","value":"id"},{"name":"EMAIL_LOG_SEARCH_RECIPIENT","value":"recipient"},{"name":"EMAIL_LOG_SEARCH_SENDER","value":"sender"},{"name":"EMAIL_LOG_SEARCH_SMTP_RECIPIENT_IP","value":"1.1.1.1"},{"name":"EMAIL_LOG_SEARCH_SMTP_SENDER_IP","value":"1.1.1.1"},{"name":"EMAIL_LOG_SEARCH_START_DATE","value":"2002-10-02T10:00:00Z"}]}}
 {"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"admin","customerId":"1"},"actor":{"callerType":"USER","email":"[email protected]","profileId":1},"ownerDomain":"elastic.com","ipAddress":"98.235.162.24","events":{"type":"EMAIL_SETTINGS","name":"EMAIL_UNDELETE","parameters":[{"name":"END_DATE","value":"2002-10-02T12:00:00Z"},{"name":"USER_EMAIL","value":"[email protected]"},{"name":"START_DATE","value":"2002-10-02T10:00:00Z"}]}}
 {"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"admin","customerId":"1"},"actor":{"callerType":"USER","email":"[email protected]","profileId":1},"ownerDomain":"elastic.com","ipAddress":"98.235.162.24","events":{"type":"EMAIL_SETTINGS","name":"CHANGE_EMAIL_SETTING","parameters":[{"name":"DOMAIN_NAME","value":"example.com"},{"name":"GROUP_EMAIL","value":"[email protected]"},{"name":"NEW_VALUE","value":"new"},{"name":"OLD_VALUE","value":"old"},{"name":"ORG_UNIT_NAME","value":"org"},{"name":"SETTING_NAME","value":"setting"}]}}
 {"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"admin","customerId":"1"},"actor":{"callerType":"USER","email":"[email protected]","profileId":1},"ownerDomain":"elastic.com","ipAddress":"98.235.162.24","events":{"type":"EMAIL_SETTINGS","name":"CHANGE_GMAIL_SETTING","parameters":[{"name":"ORG_UNIT_NAME","value":"org"},{"name":"SETTING_DESCRIPTION","value":"setting description"},{"name":"SETTING_NAME","value":"setting"},{"name":"USER_DEFINED_SETTING_NAME","value":"setting name"}]}}

diff --git a/x-pack/filebeat/module/gsuite/admin/test/gsuite-admin-gmail-test.json.log-expected.json b/x-pack/filebeat/module/gsuite/admin/test/gsuite-admin-gmail-test.json.log-expected.json
@@ -57,20 +57,20 @@
         "event.dataset": "gsuite.admin",
         "event.id": "1",
         "event.module": "gsuite",
-        "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"admin\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"[email protected]\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"EMAIL_SETTINGS\",\"name\":\"EMAIL_LOG_SEARCH\",\"parameters\":[{\"name\":\"EMAIL_LOG_SEARCH_END_DATE\",\"value\":\"2002-10-02T12:00:00Z\"},{\"name\":\"EMAIL_LOG_SEARCH_MSG_ID\",\"value\":\"id\"},{\"name\":\"EMAIL_LOG_SEARCH_RECIPIENT\",\"value\":\"recipient\"},{\"name\":\"EMAIL_LOG_SEARCH_SENDER\",\"value\":\"sender\"},{\"name\":\"EMAIL_LOG_SEARCH_SMTP_RECIPIENT_IP\",\"value\":\"1.1.1.1\"},{\"name\":\"EMAIL_LOG_SEARCH_SMTP_SENDER_IP\",\"value\":\"1.1.1.1\"},{\"name\":\"EMAIL_LOG_SEARCH_START_DATE\",\"value\":\"2002-10-02T10:00:00Z\"}]}}",
+        "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"admin\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"[email protected]\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"EMAIL_SETTINGS\",\"name\":\"EMAIL_LOG_SEARCH\",\"parameters\":[{\"name\":\"EMAIL_LOG_SEARCH_END_DATE\",\"value\":\"2020/07/28 04:59:59 UTC\"},{\"name\":\"EMAIL_LOG_SEARCH_MSG_ID\",\"value\":\"id\"},{\"name\":\"EMAIL_LOG_SEARCH_RECIPIENT\",\"value\":\"recipient\"},{\"name\":\"EMAIL_LOG_SEARCH_SENDER\",\"value\":\"sender\"},{\"name\":\"EMAIL_LOG_SEARCH_SMTP_RECIPIENT_IP\",\"value\":\"1.1.1.1\"},{\"name\":\"EMAIL_LOG_SEARCH_SMTP_SENDER_IP\",\"value\":\"1.1.1.1\"},{\"name\":\"EMAIL_LOG_SEARCH_START_DATE\",\"value\":\"2002-10-02T10:00:00Z\"}]}}",
         "event.provider": "admin",
         "event.type": [
             "info"
         ],
         "fileset.name": "admin",
         "gsuite.actor.type": "USER",
-        "gsuite.admin.email.log_search_filter.end_date": "2002-10-02T12:00:00Z",
+        "gsuite.admin.email.log_search_filter.end_date": "2020-07-28T04:59:59.000Z",
         "gsuite.admin.email.log_search_filter.message_id": "id",
         "gsuite.admin.email.log_search_filter.recipient.ip": "1.1.1.1",
         "gsuite.admin.email.log_search_filter.recipient.value": "recipient",
         "gsuite.admin.email.log_search_filter.sender.ip": "1.1.1.1",
         "gsuite.admin.email.log_search_filter.sender.value": "sender",
-        "gsuite.admin.email.log_search_filter.start_date": "2002-10-02T10:00:00Z",
+        "gsuite.admin.email.log_search_filter.start_date": "2002-10-02T10:00:00.000Z",
         "gsuite.event.type": "EMAIL_SETTINGS",
         "gsuite.kind": "admin#reports#activity",
         "gsuite.organization.domain": "elastic.com",
@@ -110,12 +110,12 @@
         ],
         "event.dataset": "gsuite.admin",
         "event.duration": 7200000000000,
-        "event.end": "2002-10-02T12:00:00Z",
+        "event.end": "2002-10-02T12:00:00.000Z",
         "event.id": "1",
         "event.module": "gsuite",
         "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"admin\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"[email protected]\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"EMAIL_SETTINGS\",\"name\":\"EMAIL_UNDELETE\",\"parameters\":[{\"name\":\"END_DATE\",\"value\":\"2002-10-02T12:00:00Z\"},{\"name\":\"USER_EMAIL\",\"value\":\"[email protected]\"},{\"name\":\"START_DATE\",\"value\":\"2002-10-02T10:00:00Z\"}]}}",
         "event.provider": "admin",
-        "event.start": "2002-10-02T10:00:00Z",
+        "event.start": "2002-10-02T10:00:00.000Z",
         "event.type": [
             "creation"
         ],
@@ -126,7 +126,7 @@
         "gsuite.kind": "admin#reports#activity",
         "gsuite.organization.domain": "elastic.com",
         "input.type": "log",
-        "log.offset": 1185,
+        "log.offset": 1188,
         "organization.id": "1",
         "related.ip": [
             "98.235.162.24"
@@ -182,7 +182,7 @@
         "gsuite.kind": "admin#reports#activity",
         "gsuite.organization.domain": "elastic.com",
         "input.type": "log",
-        "log.offset": 1668,
+        "log.offset": 1671,
         "organization.id": "1",
         "related.ip": [
             "98.235.162.24"
@@ -233,7 +233,7 @@
         "gsuite.kind": "admin#reports#activity",
         "gsuite.organization.domain": "elastic.com",
         "input.type": "log",
-        "log.offset": 2251,
+        "log.offset": 2254,
         "organization.id": "1",
         "related.ip": [
             "98.235.162.24"
@@ -284,7 +284,7 @@
         "gsuite.kind": "admin#reports#activity",
         "gsuite.organization.domain": "elastic.com",
         "input.type": "log",
-        "log.offset": 2789,
+        "log.offset": 2792,
         "organization.id": "1",
         "related.ip": [
             "98.235.162.24"
@@ -335,7 +335,7 @@
         "gsuite.kind": "admin#reports#activity",
         "gsuite.organization.domain": "elastic.com",
         "input.type": "log",
-        "log.offset": 3327,
+        "log.offset": 3330,
         "organization.id": "1",
         "related.ip": [
             "98.235.162.24"
@@ -384,7 +384,7 @@
         "gsuite.kind": "admin#reports#activity",
         "gsuite.organization.domain": "elastic.com",
         "input.type": "log",
-        "log.offset": 3865,
+        "log.offset": 3868,
         "organization.id": "1",
         "related.ip": [
             "98.235.162.24"
@@ -433,7 +433,7 @@
         "gsuite.kind": "admin#reports#activity",
         "gsuite.organization.domain": "elastic.com",
         "input.type": "log",
-        "log.offset": 4299,
+        "log.offset": 4302,
         "organization.id": "1",
         "related.ip": [
             "98.235.162.24"
@@ -460,4 +460,4 @@
             "forwarded"
         ]
     }
-]
+]
diff --git a/x-pack/filebeat/module/gsuite/admin/test/gsuite-admin-user-test.json.log-expected.json b/x-pack/filebeat/module/gsuite/admin/test/gsuite-admin-user-test.json.log-expected.json
@@ -1389,12 +1389,12 @@
         ],
         "event.dataset": "gsuite.admin",
         "event.duration": 3600000000000,
-        "event.end": "2002-10-02T16:00:00Z",
+        "event.end": "2002-10-02T16:00:00.000Z",
         "event.id": "1",
         "event.module": "gsuite",
         "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"admin\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"[email protected]\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"USER_SETTINGS\",\"name\":\"CREATE_EMAIL_MONITOR\",\"parameters\":[{\"name\":\"USER_EMAIL\",\"value\":\"[email protected]\"},{\"name\":\"BEGIN_DATE_TIME\",\"value\":\"2002-10-02T15:00:00Z\"},{\"name\":\"EMAIL_MONITOR_DEST_EMAIL\",\"value\":\"[email protected]\"},{\"name\":\"EMAIL_MONITOR_LEVEL_CHAT\",\"value\":\"info\"},{\"name\":\"EMAIL_MONITOR_LEVEL_DRAFT_EMAIL\",\"value\":\"info\"},{\"name\":\"EMAIL_MONITOR_LEVEL_INCOMING_EMAIL\",\"value\":\"info\"},{\"name\":\"EMAIL_MONITOR_LEVEL_OUTGOING_EMAIL\",\"value\":\"info\"},{\"name\":\"END_DATE_TIME\",\"value\":\"2002-10-02T16:00:00Z\"}]}}",
         "event.provider": "admin",
-        "event.start": "2002-10-02T15:00:00Z",
+        "event.start": "2002-10-02T15:00:00.000Z",
         "event.type": [
             "user",
             "creation"
@@ -2361,12 +2361,12 @@
         ],
         "event.dataset": "gsuite.admin",
         "event.duration": 3600000000000,
-        "event.end": "2002-10-02T16:00:00Z",
+        "event.end": "2002-10-02T16:00:00.000Z",
         "event.id": "1",
         "event.module": "gsuite",
         "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"admin\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"[email protected]\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"USER_SETTINGS\",\"name\":\"REQUEST_MAILBOX_DUMP\",\"parameters\":[{\"name\":\"USER_EMAIL\",\"value\":\"[email protected]\"},{\"name\":\"BEGIN_DATE_TIME\",\"value\":\"2002-10-02T15:00:00Z\"},{\"name\":\"EMAIL_EXPORT_INCLUDE_DELETED\",\"value\":\"true\"},{\"name\":\"EMAIL_EXPORT_PACKAGE_CONTENT\",\"value\":\"contents\"},{\"name\":\"SEARCH_QUERY_FOR_DUMP\",\"value\":\"foo bar\"},{\"name\":\"END_DATE_TIME\",\"value\":\"2002-10-02T16:00:00Z\"}]}}",
         "event.provider": "admin",
-        "event.start": "2002-10-02T15:00:00Z",
+        "event.start": "2002-10-02T15:00:00.000Z",
         "event.type": [
             "user",
             "info"
@@ -2929,7 +2929,7 @@
         ],
         "fileset.name": "admin",
         "gsuite.actor.type": "USER",
-        "gsuite.admin.user.birthdate": "2002-10-02T15:00:00Z",
+        "gsuite.admin.user.birthdate": "2002-10-02T15:00:00.000Z",
         "gsuite.admin.user.email": "[email protected]",
         "gsuite.event.type": "USER_SETTINGS",
         "gsuite.kind": "admin#reports#activity",
@@ -3763,4 +3763,4 @@
             "forwarded"
         ]
     }
-]
+]