From 88a0a482fb613bf28a666710a5baa3ce262fdaf9 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 22 Mar 2022 11:23:28 -0400 Subject: [PATCH] google_workspace - Add mapping for event.created The package was missing a mapping for event.created which is a field added by the Beat input. Fixes #2861 --- packages/google_workspace/changelog.yml | 5 +++++ .../google_workspace/data_stream/admin/fields/ecs.yml | 2 ++ .../google_workspace/data_stream/drive/fields/ecs.yml | 2 ++ .../google_workspace/data_stream/groups/fields/ecs.yml | 2 ++ .../google_workspace/data_stream/login/fields/ecs.yml | 2 ++ packages/google_workspace/data_stream/saml/fields/ecs.yml | 2 ++ .../test/pipeline/test-user-accounts.log-expected.json | 8 -------- .../elasticsearch/ingest_pipeline/default.yml | 3 --- .../data_stream/user_accounts/fields/ecs.yml | 2 ++ packages/google_workspace/docs/README.md | 6 ++++++ packages/google_workspace/manifest.yml | 2 +- 11 files changed, 24 insertions(+), 12 deletions(-) diff --git a/packages/google_workspace/changelog.yml b/packages/google_workspace/changelog.yml index 1567d449dfd..4554b5c90a2 100644 --- a/packages/google_workspace/changelog.yml +++ b/packages/google_workspace/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.3.2" + changes: + - description: Add mapping for ECS event.created. + type: bugfix + link: https://github.com/elastic/integrations/pull/ - version: "1.3.1" changes: - description: Remove redundant `event.ingested` from pipelines. diff --git a/packages/google_workspace/data_stream/admin/fields/ecs.yml b/packages/google_workspace/data_stream/admin/fields/ecs.yml index a61d2b6344d..4ea90a14e84 100644 --- a/packages/google_workspace/data_stream/admin/fields/ecs.yml +++ b/packages/google_workspace/data_stream/admin/fields/ecs.yml @@ -8,6 +8,8 @@ name: event.action - external: ecs name: event.category +- external: ecs + name: event.created - external: ecs name: event.duration - external: ecs diff --git a/packages/google_workspace/data_stream/drive/fields/ecs.yml b/packages/google_workspace/data_stream/drive/fields/ecs.yml index 0400f55c4e6..11a8398fbab 100644 --- a/packages/google_workspace/data_stream/drive/fields/ecs.yml +++ b/packages/google_workspace/data_stream/drive/fields/ecs.yml @@ -8,6 +8,8 @@ name: event.action - external: ecs name: event.category +- external: ecs + name: event.created - external: ecs name: event.duration - external: ecs diff --git a/packages/google_workspace/data_stream/groups/fields/ecs.yml b/packages/google_workspace/data_stream/groups/fields/ecs.yml index b18c18f20ab..bbc5a2dd20c 100644 --- a/packages/google_workspace/data_stream/groups/fields/ecs.yml +++ b/packages/google_workspace/data_stream/groups/fields/ecs.yml @@ -8,6 +8,8 @@ name: event.action - external: ecs name: event.category +- external: ecs + name: event.created - external: ecs name: event.duration - external: ecs diff --git a/packages/google_workspace/data_stream/login/fields/ecs.yml b/packages/google_workspace/data_stream/login/fields/ecs.yml index b18c18f20ab..bbc5a2dd20c 100644 --- a/packages/google_workspace/data_stream/login/fields/ecs.yml +++ b/packages/google_workspace/data_stream/login/fields/ecs.yml @@ -8,6 +8,8 @@ name: event.action - external: ecs name: event.category +- external: ecs + name: event.created - external: ecs name: event.duration - external: ecs diff --git a/packages/google_workspace/data_stream/saml/fields/ecs.yml b/packages/google_workspace/data_stream/saml/fields/ecs.yml index b18c18f20ab..bbc5a2dd20c 100644 --- a/packages/google_workspace/data_stream/saml/fields/ecs.yml +++ b/packages/google_workspace/data_stream/saml/fields/ecs.yml @@ -8,6 +8,8 @@ name: event.action - external: ecs name: event.category +- external: ecs + name: event.created - external: ecs name: event.duration - external: ecs diff --git a/packages/google_workspace/data_stream/user_accounts/_dev/test/pipeline/test-user-accounts.log-expected.json b/packages/google_workspace/data_stream/user_accounts/_dev/test/pipeline/test-user-accounts.log-expected.json index 81c30c44223..a8ff332f235 100644 --- a/packages/google_workspace/data_stream/user_accounts/_dev/test/pipeline/test-user-accounts.log-expected.json +++ b/packages/google_workspace/data_stream/user_accounts/_dev/test/pipeline/test-user-accounts.log-expected.json @@ -11,7 +11,6 @@ "iam" ], "id": "1", - "ingested": "2022-02-03T12:23:04.590365923Z", "original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"user_accounts\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"67.43.156.13\",\"events\":{\"type\":\"2sv_change\",\"name\":\"2sv_disable\"}}", "provider": "user_accounts", "type": [ @@ -83,7 +82,6 @@ "iam" ], "id": "1", - "ingested": "2022-02-03T12:23:04.590369934Z", "original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"user_accounts\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"67.43.156.13\",\"events\":{\"type\":\"2sv_change\",\"name\":\"2sv_enroll\"}}", "provider": "user_accounts", "type": [ @@ -155,7 +153,6 @@ "iam" ], "id": "1", - "ingested": "2022-02-03T12:23:04.590371119Z", "original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"user_accounts\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"67.43.156.13\",\"events\":{\"type\":\"password_change\",\"name\":\"password_edit\"}}", "provider": "user_accounts", "type": [ @@ -227,7 +224,6 @@ "iam" ], "id": "1", - "ingested": "2022-02-03T12:23:04.590372028Z", "original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"user_accounts\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"67.43.156.13\",\"events\":{\"type\":\"recovery_info_change\",\"name\":\"recovery_email_edit\"}}", "provider": "user_accounts", "type": [ @@ -299,7 +295,6 @@ "iam" ], "id": "1", - "ingested": "2022-02-03T12:23:04.590372901Z", "original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"user_accounts\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"67.43.156.13\",\"events\":{\"type\":\"recovery_info_change\",\"name\":\"recovery_phone_edit\"}}", "provider": "user_accounts", "type": [ @@ -371,7 +366,6 @@ "iam" ], "id": "1", - "ingested": "2022-02-03T12:23:04.590373738Z", "original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"user_accounts\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"67.43.156.13\",\"events\":{\"type\":\"recovery_info_change\",\"name\":\"recovery_secret_qa_edit\"}}", "provider": "user_accounts", "type": [ @@ -443,7 +437,6 @@ "iam" ], "id": "1", - "ingested": "2022-02-03T12:23:04.590374612Z", "original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"user_accounts\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"67.43.156.13\",\"events\":{\"type\":\"titanium_change\",\"name\":\"titanium_enroll\"}}", "provider": "user_accounts", "type": [ @@ -515,7 +508,6 @@ "iam" ], "id": "1", - "ingested": "2022-02-03T12:23:04.590375455Z", "original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"user_accounts\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"67.43.156.13\",\"events\":{\"type\":\"titanium_change\",\"name\":\"titanium_unenroll\"}}", "provider": "user_accounts", "type": [ diff --git a/packages/google_workspace/data_stream/user_accounts/elasticsearch/ingest_pipeline/default.yml b/packages/google_workspace/data_stream/user_accounts/elasticsearch/ingest_pipeline/default.yml index 3edd2b327e5..65c556197e2 100644 --- a/packages/google_workspace/data_stream/user_accounts/elasticsearch/ingest_pipeline/default.yml +++ b/packages/google_workspace/data_stream/user_accounts/elasticsearch/ingest_pipeline/default.yml @@ -141,9 +141,6 @@ processors: - remove: field: json ignore_missing: true - - set: - field: event.ingested - value: "{{ _ingest.timestamp }}" - remove: field: event.original if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" diff --git a/packages/google_workspace/data_stream/user_accounts/fields/ecs.yml b/packages/google_workspace/data_stream/user_accounts/fields/ecs.yml index b18c18f20ab..bbc5a2dd20c 100644 --- a/packages/google_workspace/data_stream/user_accounts/fields/ecs.yml +++ b/packages/google_workspace/data_stream/user_accounts/fields/ecs.yml @@ -8,6 +8,8 @@ name: event.action - external: ecs name: event.category +- external: ecs + name: event.created - external: ecs name: event.duration - external: ecs diff --git a/packages/google_workspace/docs/README.md b/packages/google_workspace/docs/README.md index 2b72e780986..2a6ab4eea1a 100644 --- a/packages/google_workspace/docs/README.md +++ b/packages/google_workspace/docs/README.md @@ -179,6 +179,7 @@ An example event for `saml` looks as following: | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | | event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | +| event.created | event.created contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, @timestamp should be used. | date | | event.dataset | Event dataset | constant_keyword | | event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long | | event.end | event.end contains the date when the event ended or when the activity was last observed. | date | @@ -381,6 +382,7 @@ An example event for `user_accounts` looks as following: | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | | event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | +| event.created | event.created contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, @timestamp should be used. | date | | event.dataset | Event dataset | constant_keyword | | event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long | | event.end | event.end contains the date when the event ended or when the activity was last observed. | date | @@ -585,6 +587,7 @@ An example event for `login` looks as following: | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | | event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | +| event.created | event.created contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, @timestamp should be used. | date | | event.dataset | Event dataset | constant_keyword | | event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long | | event.end | event.end contains the date when the event ended or when the activity was last observed. | date | @@ -816,6 +819,7 @@ An example event for `admin` looks as following: | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | | event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | +| event.created | event.created contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, @timestamp should be used. | date | | event.dataset | Event dataset | constant_keyword | | event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long | | event.end | event.end contains the date when the event ended or when the activity was last observed. | date | @@ -1130,6 +1134,7 @@ An example event for `drive` looks as following: | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | | event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | +| event.created | event.created contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, @timestamp should be used. | date | | event.dataset | Event dataset | constant_keyword | | event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long | | event.end | event.end contains the date when the event ended or when the activity was last observed. | date | @@ -1376,6 +1381,7 @@ An example event for `groups` looks as following: | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | | event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | +| event.created | event.created contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, @timestamp should be used. | date | | event.dataset | Event dataset | constant_keyword | | event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long | | event.end | event.end contains the date when the event ended or when the activity was last observed. | date | diff --git a/packages/google_workspace/manifest.yml b/packages/google_workspace/manifest.yml index 5513a467009..6ca9d261642 100644 --- a/packages/google_workspace/manifest.yml +++ b/packages/google_workspace/manifest.yml @@ -1,6 +1,6 @@ name: google_workspace title: Google Workspace Audit Reports -version: 1.3.1 +version: 1.3.2 release: ga description: Collect audit reports from Google Workspaces with Elastic Agent. type: integration