[Filebeat] Change field type in aws cloudtrail fileset to flattened

CHANGELOG.next.asciidoc

@@ -42,6 +42,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 * iptables {pull}18756[18756]
 * Checkpoint {pull}18754[18754]
 - Preserve case of http.request.method.  ECS prior to 1.6 specified normalizing to lowercase, which lost information. Affects filesets: apache/access, elasticsearch/audit, iis/access, iis/error, nginx/access, nginx/ingress_controller, aws/elb, suricata/eve, zeek/http. {issue}18154[18154] {pull}18359[18359]
+- In aws cloudtrail fileset change type of field from keyword to flattened for request_parameters, response_elements, additional_eventdata & service_event_details. {issue}18866[18866] {pull}19020[19020]
 
 *Heartbeat*
 

filebeat/docs/fields.asciidoc

@@ -1223,7 +1223,7 @@ type: keyword
 --
 The parameters, if any, that were sent with the request.
 
-type: keyword
+type: flattened
 
 --
 
@@ -1232,7 +1232,7 @@ type: keyword
 --
 The response element for actions that make changes (create, update, or delete actions).
 
-type: keyword
+type: flattened
 
 --
 
@@ -1241,7 +1241,7 @@ type: keyword
 --
 Additional data about the event that was not part of the request or response.
 
-type: keyword
+type: flattened
 
 --
 
@@ -1337,7 +1337,7 @@ type: keyword
 --
 Identifies the service event, including what triggered the event and the result.
 
-type: keyword
+type: flattened
 
 --
 

libbeat/mapping/field.go

@@ -138,7 +138,7 @@ func (f *Field) validateType() error {
 		if f.Format != "" {
 			return fmt.Errorf("no format expected for field %s, found: %s", f.Name, f.Format)
 		}
-	case "object", "group", "nested":
+	case "object", "group", "nested", "flattened":
 		// No check for them yet
 	case "":
 		// Module keys, not used as fields

libbeat/tests/system/beat/beat.py

@@ -570,7 +570,7 @@ def extract_fields(doc_list, name):
                     aliases.extend(subaliases)
                 else:
                     fields.append(newName)
-                    if field.get("type") in ["object", "geo_point"]:
+                    if field.get("type") in ["object", "geo_point", "flattened"]:
                         dictfields.append(newName)
 
                 if field.get("type") == "object" and field.get("object_type") == "histogram":

x-pack/filebeat/module/aws/cloudtrail/_meta/fields.yml

@@ -87,16 +87,16 @@
       description: >-
         If the request returns an error, the description of the error.
     - name: request_parameters
-      type: keyword
+      type: flattened
       description: >-
         The parameters, if any, that were sent with the request.
     - name: response_elements
-      type: keyword
+      type: flattened
       description: >-
         The response element for actions that make changes (create,
         update, or delete actions).
     - name: additional_eventdata
-      type: keyword
+      type: flattened
       description: >-
         Additional data about the event that was not part of the
         request or response.
@@ -145,7 +145,7 @@
       description: >-
         Represents the account ID that received this event.
     - name: service_event_details
-      type: keyword
+      type: flattened
       description: >-
         Identifies the service event, including what triggered the
         event and the result.

x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml

@@ -118,26 +118,20 @@ processors:
       field: "json.errorMessage"
       target_field: "aws.cloudtrail.error_message"
       ignore_failure: true
-  - script:
-      lang: painless
-      source: |
-        if (ctx.json.requestParameters != null) {
-          ctx.aws.cloudtrail.request_parameters = ctx.json.requestParameters.toString();
-        }
+  - rename:
+      field: "json.requestParameters"
+      target_field: "aws.cloudtrail.request_parameters"
+      if: "ctx.json.requestParameters != null"
       ignore_failure: true
-  - script:
-      lang: painless
-      source: |
-        if (ctx.json.responseElements != null) {
-          ctx.aws.cloudtrail.response_elements = ctx.json.responseElements.toString();
-        }
+  - rename:
+      field: "json.responseElements"
+      target_field: "aws.cloudtrail.response_elements"
+      if: "ctx.json.responseElements != null"
       ignore_failure: true
-  - script:
-      lang: painless
-      source: |
-        if (ctx.json.additionalEventData != null) {
-          ctx.aws.cloudtrail.additional_eventdata = ctx.json.additionalEventData.toString();
-        }
+  - rename:
+      field: "json.additionalEventData"
+      target_field: "aws.cloudtrail.additional_eventdata"
+      if: "ctx.json.additionalEventData != null"
       ignore_failure: true
   - rename:
       field: "json.requestId"
@@ -179,12 +173,10 @@ processors:
       field: "json.recipientAccountId"
       target_field: "aws.cloudtrail.recipient_account_id"
       ignore_failure: true
-  - script:
-      lang: painless
-      source: |
-        if (ctx.json.serviceEventDetails != null) {
-          ctx.aws.cloudtrail.service_event_details = ctx.json.serviceEventDetails.toString();
-        }
+  - rename:
+      field: "json.serviceEventDetails"
+      target_field: "aws.cloudtrail.service_event_details"
+      if: "ctx.json.serviceEventDetails != null"
       ignore_failure: true
   - rename:
       field: "json.sharedEventId"

x-pack/filebeat/module/aws/cloudtrail/test/add-user-to-group-json.log-expected.json

@@ -2,7 +2,8 @@
     {
         "@timestamp": "2014-03-25T21:08:14.000Z",
         "aws.cloudtrail.event_version": "1.0",
-        "aws.cloudtrail.request_parameters": "{groupName=admin, userName=Bob}",
+        "aws.cloudtrail.request_parameters.groupName": "admin",
+        "aws.cloudtrail.request_parameters.userName": "Bob",
         "aws.cloudtrail.user_identity.access_key_id": "EXAMPLE_KEY_ID",
         "aws.cloudtrail.user_identity.arn": "arn:aws:iam::123456789012:user/Alice",
         "aws.cloudtrail.user_identity.session_context.creation_date": "2014-03-25T18:45:11.000Z",
@@ -21,9 +22,6 @@
         "fileset.name": "cloudtrail",
         "input.type": "log",
         "log.offset": 0,
-        "related.user": [
-            "Bob"
-        ],
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",

x-pack/filebeat/module/aws/cloudtrail/test/assume-role-json.log-expected.json

@@ -4,8 +4,29 @@
         "aws.cloudtrail.event_type": "AwsApiCall",
         "aws.cloudtrail.event_version": "1.05",
         "aws.cloudtrail.recipient_account_id": "111111111111",
-        "aws.cloudtrail.request_parameters": "{incomingTransitiveTags={Department=Engineering}, transitiveTagKeys=[Email, CostCenter], durationSeconds=3600, roleArn=arn:aws:iam::111111111111:role/JohnRole2, roleSessionName=Role2WithTags, tags=[{[email protected], key=Email}, {value=12345, key=CostCenter}]}",
-        "aws.cloudtrail.response_elements": "{assumedRoleUser={assumedRoleId=AROAIFR7WHDTSOYQYHFUE:Role2WithTags, arn=arn:aws:sts::111111111111:assumed-role/test-role/Role2WithTags}, credentials={accessKeyId=ASIAWHOJDLGPOEXAMPLE, sessionToken=AgoJb3JpZ2luX2VjEB4aCXVzLXdlc3QtMSJHMEXAMPLETOKEN+//rJb8Lo30mFc5MlhFCEbubZvEj0wHB/mDMwIgSEe9gk/Zjr09tZV7F1HDTMhmEXAMPLETOKEN/iEJ/rkqngII9///////////ARABGgw0MjgzMDc4NjM5NjYiDLZjZFKwP4qxQG5sFCryASO4UPz5qE97wPPH1eLMvs7CgSDBSWfonmRTCfokm2FN1+hWUdQQH6adjbbrVLFL8c3jSsBhQ383AvxpwK5YRuDE1AI/+C+WKFZb701eiv9J5La2EXAMPLETOKEN/c7S5Iro1WUJ0q3Cxuo/8HUoSxVhQHM7zF7mWWLhXLEQ52ivL+F6q5dpXu4aTFedpMfnJa8JtkWwG9x1Axj0Ypy2ok8v5unpQGWych1vwdvj6ez1Dm8Xg1+qIzXILiEXAMPLETOKEN/vQGqu8H+nxp3kabcrtOvTFTvxX6vsc8OGwUfHhzAfYGEXAMPLETOKEN/L6v1yMM3B1OwFOrQBno1HEjf1oNI8RnQiMNFdUOtwYj7HUZIOCZmjfN8PPHq77N7GJl9lzvIZKQA0Owcjg+mc78zHCj8y0siY8C96paEXAMPLETOKEN/E3cpksxWdgs91HRzJWScjN2+r2LTGjYhyPqcmFzzo2mCE7mBNEXAMPLETOKEN/oJy+2o83YNW5tOiDmczgDzJZ4UKR84yGYOMfSnF4XcEJrDgAJ3OJFwmTcTQICAlSwLEXAMPLETOKEN, expiration=Oct 2, 2019 11:12:29 PM}}",
+        "aws.cloudtrail.request_parameters.durationSeconds": 3600,
+        "aws.cloudtrail.request_parameters.incomingTransitiveTags.Department": "Engineering",
+        "aws.cloudtrail.request_parameters.roleArn": "arn:aws:iam::111111111111:role/JohnRole2",
+        "aws.cloudtrail.request_parameters.roleSessionName": "Role2WithTags",
+        "aws.cloudtrail.request_parameters.tags": [
+            {
+                "key": "Email",
+                "value": "[email protected]"
+            },
+            {
+                "key": "CostCenter",
+                "value": "12345"
+            }
+        ],
+        "aws.cloudtrail.request_parameters.transitiveTagKeys": [
+            "Email",
+            "CostCenter"
+        ],
+        "aws.cloudtrail.response_elements.assumedRoleUser.arn": "arn:aws:sts::111111111111:assumed-role/test-role/Role2WithTags",
+        "aws.cloudtrail.response_elements.assumedRoleUser.assumedRoleId": "AROAIFR7WHDTSOYQYHFUE:Role2WithTags",
+        "aws.cloudtrail.response_elements.credentials.accessKeyId": "ASIAWHOJDLGPOEXAMPLE",
+        "aws.cloudtrail.response_elements.credentials.expiration": "Oct 2, 2019 11:12:29 PM",
+        "aws.cloudtrail.response_elements.credentials.sessionToken": "AgoJb3JpZ2luX2VjEB4aCXVzLXdlc3QtMSJHMEXAMPLETOKEN+//rJb8Lo30mFc5MlhFCEbubZvEj0wHB/mDMwIgSEe9gk/Zjr09tZV7F1HDTMhmEXAMPLETOKEN/iEJ/rkqngII9///////////ARABGgw0MjgzMDc4NjM5NjYiDLZjZFKwP4qxQG5sFCryASO4UPz5qE97wPPH1eLMvs7CgSDBSWfonmRTCfokm2FN1+hWUdQQH6adjbbrVLFL8c3jSsBhQ383AvxpwK5YRuDE1AI/+C+WKFZb701eiv9J5La2EXAMPLETOKEN/c7S5Iro1WUJ0q3Cxuo/8HUoSxVhQHM7zF7mWWLhXLEQ52ivL+F6q5dpXu4aTFedpMfnJa8JtkWwG9x1Axj0Ypy2ok8v5unpQGWych1vwdvj6ez1Dm8Xg1+qIzXILiEXAMPLETOKEN/vQGqu8H+nxp3kabcrtOvTFTvxX6vsc8OGwUfHhzAfYGEXAMPLETOKEN/L6v1yMM3B1OwFOrQBno1HEjf1oNI8RnQiMNFdUOtwYj7HUZIOCZmjfN8PPHq77N7GJl9lzvIZKQA0Owcjg+mc78zHCj8y0siY8C96paEXAMPLETOKEN/E3cpksxWdgs91HRzJWScjN2+r2LTGjYhyPqcmFzzo2mCE7mBNEXAMPLETOKEN/oJy+2o83YNW5tOiDmczgDzJZ4UKR84yGYOMfSnF4XcEJrDgAJ3OJFwmTcTQICAlSwLEXAMPLETOKEN",
         "aws.cloudtrail.user_identity.access_key_id": "AKIAI44QH8DHBEXAMPLE",
         "aws.cloudtrail.user_identity.arn": "arn:aws:sts::111111111111:assumed-role/JohnDoe/JohnRole1",
         "aws.cloudtrail.user_identity.session_context.creation_date": "2019-10-02T21:50:54.000Z",

x-pack/filebeat/module/aws/cloudtrail/test/console-login-json.log-expected.json

@@ -1,12 +1,11 @@
 [
     {
         "@timestamp": "2014-07-16T15:49:27.000Z",
-        "aws.cloudtrail.additional_eventdata": "{LoginTo=https://console.aws.amazon.com/s3/, MobileVersion=No, MFAUsed=No}",
-        "aws.cloudtrail.console_login.additional_eventdata.login_to": "https://console.aws.amazon.com/s3/",
-        "aws.cloudtrail.console_login.additional_eventdata.mfa_used": false,
-        "aws.cloudtrail.console_login.additional_eventdata.mobile_version": false,
+        "aws.cloudtrail.additional_eventdata.LoginTo": "https://console.aws.amazon.com/s3/",
+        "aws.cloudtrail.additional_eventdata.MFAUsed": "No",
+        "aws.cloudtrail.additional_eventdata.MobileVersion": "No",
         "aws.cloudtrail.event_version": "1.05",
-        "aws.cloudtrail.response_elements": "{ConsoleLogin=Success}",
+        "aws.cloudtrail.response_elements.ConsoleLogin": "Success",
         "aws.cloudtrail.user_identity.arn": "arn:aws:iam::111122223333:user/JohnDoe",
         "aws.cloudtrail.user_identity.type": "IAMUser",
         "cloud.account.id": "111122223333",
@@ -42,13 +41,12 @@
     },
     {
         "@timestamp": "2014-07-08T17:35:27.000Z",
-        "aws.cloudtrail.additional_eventdata": "{LoginTo=https://console.aws.amazon.com/sns, MobileVersion=No, MFAUsed=No}",
-        "aws.cloudtrail.console_login.additional_eventdata.login_to": "https://console.aws.amazon.com/sns",
-        "aws.cloudtrail.console_login.additional_eventdata.mfa_used": false,
-        "aws.cloudtrail.console_login.additional_eventdata.mobile_version": false,
+        "aws.cloudtrail.additional_eventdata.LoginTo": "https://console.aws.amazon.com/sns",
+        "aws.cloudtrail.additional_eventdata.MFAUsed": "No",
+        "aws.cloudtrail.additional_eventdata.MobileVersion": "No",
         "aws.cloudtrail.error_message": "Failed authentication",
         "aws.cloudtrail.event_version": "1.05",
-        "aws.cloudtrail.response_elements": "{ConsoleLogin=Failure}",
+        "aws.cloudtrail.response_elements.ConsoleLogin": "Failure",
         "aws.cloudtrail.user_identity.arn": "arn:aws:iam::111122223333:user/JaneDoe",
         "aws.cloudtrail.user_identity.type": "IAMUser",
         "cloud.account.id": "111122223333",
@@ -84,13 +82,12 @@
     },
     {
         "@timestamp": "2014-07-08T17:35:27.000Z",
-        "aws.cloudtrail.additional_eventdata": "{LoginTo=https://console.aws.amazon.com/sns, MobileVersion=No, MFAUsed=No}",
-        "aws.cloudtrail.console_login.additional_eventdata.login_to": "https://console.aws.amazon.com/sns",
-        "aws.cloudtrail.console_login.additional_eventdata.mfa_used": false,
-        "aws.cloudtrail.console_login.additional_eventdata.mobile_version": false,
+        "aws.cloudtrail.additional_eventdata.LoginTo": "https://console.aws.amazon.com/sns",
+        "aws.cloudtrail.additional_eventdata.MFAUsed": "No",
+        "aws.cloudtrail.additional_eventdata.MobileVersion": "No",
         "aws.cloudtrail.error_message": "Failed authentication",
         "aws.cloudtrail.event_version": "1.05",
-        "aws.cloudtrail.response_elements": "{ConsoleLogin=Failure}",
+        "aws.cloudtrail.response_elements.ConsoleLogin": "Failure",
         "aws.cloudtrail.user_identity.access_key_id": "AKIAIOSFODNN7EXAMPLE",
         "aws.cloudtrail.user_identity.arn": "arn:aws:sts::123456789012:assumed-role/RoleToBeAssumed/MySessionName",
         "aws.cloudtrail.user_identity.session_context.mfa_authenticated": "false",

x-pack/filebeat/module/aws/cloudtrail/test/create-access-key-json.log-expected.json

@@ -4,8 +4,11 @@
         "aws.cloudtrail.event_type": "AwsApiCall",
         "aws.cloudtrail.event_version": "1.05",
         "aws.cloudtrail.recipient_account_id": "0123456789012",
-        "aws.cloudtrail.request_parameters": "{userName=Bob}",
-        "aws.cloudtrail.response_elements": "{accessKey={accessKeyId=EXAMPLE_KEY_ID, userName=Bob, status=Active, createDate=Jan 8, 2020 8:43:06 PM}}",
+        "aws.cloudtrail.request_parameters.userName": "Bob",
+        "aws.cloudtrail.response_elements.accessKey.accessKeyId": "EXAMPLE_KEY_ID",
+        "aws.cloudtrail.response_elements.accessKey.createDate": "Jan 8, 2020 8:43:06 PM",
+        "aws.cloudtrail.response_elements.accessKey.status": "Active",
+        "aws.cloudtrail.response_elements.accessKey.userName": "Bob",
         "aws.cloudtrail.user_identity.access_key_id": "EXAMPLE_KEY",
         "aws.cloudtrail.user_identity.arn": "arn:aws:iam::0123456789012:user/Alice",
         "aws.cloudtrail.user_identity.invoked_by": "signin.amazonaws.com",
@@ -26,9 +29,6 @@
         "fileset.name": "cloudtrail",
         "input.type": "log",
         "log.offset": 0,
-        "related.user": [
-            "Bob"
-        ],
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",

x-pack/filebeat/module/aws/cloudtrail/test/create-group-json.log-expected.json

@@ -4,8 +4,12 @@
         "aws.cloudtrail.event_type": "AwsApiCall",
         "aws.cloudtrail.event_version": "1.05",
         "aws.cloudtrail.recipient_account_id": "0123456789012",
-        "aws.cloudtrail.request_parameters": "{groupName=TEST-GROUP}",
-        "aws.cloudtrail.response_elements": "{group={path=/, groupName=TEST-GROUP, groupId=EXAMPLE_ID, arn=arn:aws:iam::0123456789012:group/TEST-GROUP, createDate=Jan 9, 2020 1:48:44 AM}}",
+        "aws.cloudtrail.request_parameters.groupName": "TEST-GROUP",
+        "aws.cloudtrail.response_elements.group.arn": "arn:aws:iam::0123456789012:group/TEST-GROUP",
+        "aws.cloudtrail.response_elements.group.createDate": "Jan 9, 2020 1:48:44 AM",
+        "aws.cloudtrail.response_elements.group.groupId": "EXAMPLE_ID",
+        "aws.cloudtrail.response_elements.group.groupName": "TEST-GROUP",
+        "aws.cloudtrail.response_elements.group.path": "/",
         "aws.cloudtrail.user_identity.access_key_id": "EXAMPLE_KEY",
         "aws.cloudtrail.user_identity.arn": "arn:aws:iam::0123456789012:user/Alice",
         "aws.cloudtrail.user_identity.invoked_by": "signin.amazonaws.com",
@@ -45,7 +49,7 @@
         "aws.cloudtrail.event_type": "AwsApiCall",
         "aws.cloudtrail.event_version": "1.05",
         "aws.cloudtrail.recipient_account_id": "0123456789012",
-        "aws.cloudtrail.request_parameters": "{groupName=TEST-GROUP}",
+        "aws.cloudtrail.request_parameters.groupName": "TEST-GROUP",
         "aws.cloudtrail.user_identity.access_key_id": "EXAMPLE_KEY",
         "aws.cloudtrail.user_identity.arn": "arn:aws:iam::0123456789012:user/Alice",
         "aws.cloudtrail.user_identity.type": "IAMUser",

x-pack/filebeat/module/aws/cloudtrail/test/create-key-pair-json.log-expected.json

@@ -2,8 +2,10 @@
     {
         "@timestamp": "2014-03-06T17:10:34.000Z",
         "aws.cloudtrail.event_version": "1.0",
-        "aws.cloudtrail.request_parameters": "{keyName=mykeypair}",
-        "aws.cloudtrail.response_elements": "{keyMaterial=<sensitiveDataRemoved>, keyFingerprint=30:1d:46:d0:5b:ad:7e:1b:b6:70:62:8b:ff:38:b5:e9:ab:5d:b8:21, keyName=mykeypair}",
+        "aws.cloudtrail.request_parameters.keyName": "mykeypair",
+        "aws.cloudtrail.response_elements.keyFingerprint": "30:1d:46:d0:5b:ad:7e:1b:b6:70:62:8b:ff:38:b5:e9:ab:5d:b8:21",
+        "aws.cloudtrail.response_elements.keyMaterial": "<sensitiveDataRemoved>",
+        "aws.cloudtrail.response_elements.keyName": "mykeypair",
         "aws.cloudtrail.user_identity.access_key_id": "EXAMPLE_KEY_ID",
         "aws.cloudtrail.user_identity.arn": "arn:aws:iam::123456789012:user/Alice",
         "aws.cloudtrail.user_identity.session_context.creation_date": "2014-03-06T15:15:06.000Z",

x-pack/filebeat/module/aws/cloudtrail/test/create-trail-json.log-expected.json

@@ -5,8 +5,20 @@
         "aws.cloudtrail.event_version": "1.05",
         "aws.cloudtrail.read_only": false,
         "aws.cloudtrail.recipient_account_id": "0123456789012",
-        "aws.cloudtrail.request_parameters": "{isMultiRegionTrail=true, s3BucketName=TEST-cloudtrail-bucket, name=TEST-trail, enableLogFileValidation=true, kmsKeyId=, isOrganizationTrail=false, includeGlobalServiceEvents=true}",
-        "aws.cloudtrail.response_elements": "{logFileValidationEnabled=true, isMultiRegionTrail=true, s3BucketName=TEST-cloudtrail-bucket, name=TEST-trail, trailARN=arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail, isOrganizationTrail=false, includeGlobalServiceEvents=true}",
+        "aws.cloudtrail.request_parameters.enableLogFileValidation": true,
+        "aws.cloudtrail.request_parameters.includeGlobalServiceEvents": true,
+        "aws.cloudtrail.request_parameters.isMultiRegionTrail": true,
+        "aws.cloudtrail.request_parameters.isOrganizationTrail": false,
+        "aws.cloudtrail.request_parameters.kmsKeyId": "",
+        "aws.cloudtrail.request_parameters.name": "TEST-trail",
+        "aws.cloudtrail.request_parameters.s3BucketName": "TEST-cloudtrail-bucket",
+        "aws.cloudtrail.response_elements.includeGlobalServiceEvents": true,
+        "aws.cloudtrail.response_elements.isMultiRegionTrail": true,
+        "aws.cloudtrail.response_elements.isOrganizationTrail": false,
+        "aws.cloudtrail.response_elements.logFileValidationEnabled": true,
+        "aws.cloudtrail.response_elements.name": "TEST-trail",
+        "aws.cloudtrail.response_elements.s3BucketName": "TEST-cloudtrail-bucket",
+        "aws.cloudtrail.response_elements.trailARN": "arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail",
         "aws.cloudtrail.user_identity.access_key_id": "EXAMPLE_KEY",
         "aws.cloudtrail.user_identity.arn": "arn:aws:iam::0123456789012:user/Alice",
         "aws.cloudtrail.user_identity.invoked_by": "signin.amazonaws.com",