elastic · marc-gr · May 27, 2021 · May 19, 2021
diff --git a/packages/okta/_dev/build/docs/README.md b/packages/okta/_dev/build/docs/README.md
@@ -8,4 +8,6 @@ The Okta integration collects events from the Okta API, specifically reading fro
 
 The Okta System Log records system events related to your organization in order to provide an audit trail that can be used to understand platform activity and to diagnose problems. This module is implemented using the httpjson input and is configured to paginate through the logs while honoring any rate-limiting headers sent by Okta.
 
-{{fields "system"}}
+{{event "system"}}
+
+{{fields "system"}}
diff --git a/packages/okta/changelog.yml b/packages/okta/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.5.1"
+  changes:
+    - description: Make event.original optional
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1009
 - version: "0.5.0"
   changes:
     - description: change okta.target to flattened type

diff --git a/packages/okta/data_stream/system/_dev/test/pipeline/test-okta-system-events.json-config.yml b/packages/okta/data_stream/system/_dev/test/pipeline/test-okta-system-events.json-config.yml
@@ -1,6 +1,6 @@
 fields:
   "@timestamp": "2020-04-28T11:07:58.223Z"
-  "_conf":
-    "keep_original_message": true
+  tags:
+    - preserve_original_event
 dynamic_fields:
   event.ingested: "^.*$"
diff --git a/...ges/okta/data_stream/system/_dev/test/pipeline/test-okta-system-events.json-expected.json b/...ges/okta/data_stream/system/_dev/test/pipeline/test-okta-system-events.json-expected.json
diff --git a/packages/okta/data_stream/system/agent/stream/httpjson.yml.hbs b/packages/okta/data_stream/system/agent/stream/httpjson.yml.hbs
@@ -38,11 +38,9 @@ tags:
 {{#each tags as |tag i|}}
  - {{tag}}
 {{/each}}
+{{#if preserve_original_event}}
+ - preserve_original_event
+{{/if}}
 {{#contains tags "forwarded"}}
 publisher_pipeline.disable_host: true
 {{/contains}}
-
-fields:
-  _conf:
-    keep_original_message: {{keep_original_message}}
-fields_under_root: true
diff --git a/packages/okta/data_stream/system/agent/stream/log.yml.hbs b/packages/okta/data_stream/system/agent/stream/log.yml.hbs
@@ -7,11 +7,9 @@ tags:
 {{#each tags as |tag i|}}
  - {{tag}}
 {{/each}}
+{{#if preserve_original_event}}
+ - preserve_original_event
+{{/if}}
 {{#contains tags "forwarded"}}
 publisher_pipeline.disable_host: true
 {{/contains}}
-
-fields:
-  _conf:
-    keep_original_message: {{keep_original_message}}
-fields_under_root: true
diff --git a/packages/okta/data_stream/system/elasticsearch/ingest_pipeline/default.yml b/packages/okta/data_stream/system/elasticsearch/ingest_pipeline/default.yml
@@ -7,15 +7,11 @@ processors:
   - set:
       field: ecs.version
       value: "1.9.0"
-  - set:
-      field: event.original
-      copy_from: message
-      if: "ctx?._conf?.keep_original_message == true"
-  - remove:
-      field: _conf
-      ignore_missing: true
-  - json:
+  - rename:
       field: message
+      target_field: event.original
+  - json:
+      field: event.original
       target_field: json
   - script:
       description: Drops null/empty values recursively
@@ -34,9 +30,6 @@ processors:
           return false;
         }
         drop(ctx);
-  - remove:
-      field: message
-      ignore_missing: true
   - convert:
       field: json.uuid
       target_field: _id
@@ -557,7 +550,11 @@ processors:
       field: destination.as.organization_name
       target_field: destination.as.organization.name
       ignore_missing: true
-
+  - remove:
+      field: event.original
+      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
+      ignore_failure: true
+      ignore_missing: true
 on_failure:
   - set:
       field: error.message

diff --git a/packages/okta/data_stream/system/manifest.yml b/packages/okta/data_stream/system/manifest.yml
@@ -4,13 +4,14 @@ release: experimental
 streams:
   - input: httpjson
     vars:
-      - name: keep_original_message
-        type: bool
-        title: Keep Original Message
-        multi: false
+      - name: preserve_original_event
         required: true
         show_user: true
-        default: true
+        title: Preserve original event
+        description: Preserves a raw copy of the original event, added to the field `event.original`
+        type: bool
+        multi: false
+        default: false
       - name: tags
         type: text
         title: Tags
@@ -25,13 +26,14 @@ streams:
   - input: logfile
     enabled: false
     vars:
-      - name: keep_original_message
-        type: bool
-        title: Keep Original Message
-        multi: false
+      - name: preserve_original_event
         required: true
         show_user: true
-        default: true
+        title: Preserve original event
+        description: Preserves a raw copy of the original event, added to the field `event.original`
+        type: bool
+        multi: false
+        default: false
       - name: tags
         type: text
         title: Tags

diff --git a/packages/okta/data_stream/system/sample_event.json b/packages/okta/data_stream/system/sample_event.json
@@ -0,0 +1,161 @@
+{
+    "@timestamp": "2020-02-14T20:18:57.718Z",
+    "agent": {
+        "ephemeral_id": "4927e29f-62b9-48b5-80d6-3a6472c7ec79",
+        "hostname": "docker-fleet-agent",
+        "id": "031c86fe-6c1b-47c6-8606-a84941974792",
+        "name": "docker-fleet-agent",
+        "type": "filebeat",
+        "version": "7.13.0"
+    },
+    "client": {
+        "geo": {
+            "city_name": "Dublin",
+            "country_name": "United States",
+            "location": {
+                "lat": 37.7201,
+                "lon": -121.919
+            },
+            "region_name": "California"
+        },
+        "ip": "108.255.197.247",
+        "user": {
+            "full_name": "xxxxxx",
+            "id": "00u1abvz4pYqdM8ms4x6"
+        }
+    },
+    "data_stream": {
+        "dataset": "okta.system",
+        "namespace": "ep",
+        "type": "logs"
+    },
+    "ecs": {
+        "version": "1.9.0"
+    },
+    "elastic_agent": {
+        "id": "edf17a53-264b-42ce-b1b0-12c4dd4fc099",
+        "snapshot": true,
+        "version": "7.13.0"
+    },
+    "event": {
+        "action": "user.session.start",
+        "category": [
+            "authentication",
+            "session"
+        ],
+        "dataset": "okta.system",
+        "id": "3aeede38-4f67-11ea-abd3-1f5d113f2546",
+        "ingested": "2021-05-19T12:23:00.173930400Z",
+        "kind": "event",
+        "outcome": "success",
+        "type": [
+            "start",
+            "user"
+        ]
+    },
+    "host": {
+        "name": "docker-fleet-agent"
+    },
+    "input": {
+        "type": "log"
+    },
+    "log": {
+        "file": {
+            "path": "/tmp/service_logs/okta-system.json.log"
+        },
+        "offset": 1665
+    },
+    "okta": {
+        "actor": {
+            "alternate_id": "[email protected]",
+            "display_name": "xxxxxx",
+            "id": "00u1abvz4pYqdM8ms4x6",
+            "type": "User"
+        },
+        "authentication_context": {
+            "authentication_step": 0,
+            "external_session_id": "102bZDNFfWaQSyEZQuDgWt-uQ"
+        },
+        "client": {
+            "device": "Computer",
+            "ip": "108.255.197.247",
+            "user_agent": {
+                "browser": "FIREFOX",
+                "os": "Mac OS X",
+                "raw_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:72.0) Gecko/20100101 Firefox/72.0"
+            },
+            "zone": "null"
+        },
+        "debug_context": {
+            "debug_data": {
+                "device_fingerprint": "541daf91d15bef64a7e08c946fd9a9d0",
+                "request_id": "XkcAsWb8WjwDP76xh@1v8wAABp0",
+                "request_uri": "/api/v1/authn",
+                "threat_suspected": "false",
+                "url": "/api/v1/authn?"
+            }
+        },
+        "display_message": "User login to Okta",
+        "event_type": "user.session.start",
+        "outcome": {
+            "result": "SUCCESS"
+        },
+        "transaction": {
+            "id": "XkcAsWb8WjwDP76xh@1v8wAABp0",
+            "type": "WEB"
+        },
+        "uuid": "3aeede38-4f67-11ea-abd3-1f5d113f2546"
+    },
+    "related": {
+        "ip": [
+            "108.255.197.247"
+        ],
+        "user": [
+            "xxxxxx"
+        ]
+    },
+    "source": {
+        "as": {
+            "number": 7018,
+            "organization": {
+                "name": "AT\u0026T Services, Inc."
+            }
+        },
+        "geo": {
+            "city_name": "Dublin",
+            "continent_name": "North America",
+            "country_iso_code": "US",
+            "country_name": "United States",
+            "location": {
+                "lat": 37.7201,
+                "lon": -121.919
+            },
+            "region_iso_code": "US-CA",
+            "region_name": "California"
+        },
+        "ip": "108.255.197.247",
+        "user": {
+            "full_name": "xxxxxx",
+            "id": "00u1abvz4pYqdM8ms4x6"
+        }
+    },
+    "tags": [
+        "forwarded"
+    ],
+    "user": {
+        "full_name": "xxxxxx"
+    },
+    "user_agent": {
+        "device": {
+            "name": "Mac"
+        },
+        "name": "Firefox",
+        "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:72.0) Gecko/20100101 Firefox/72.0",
+        "os": {
+            "full": "Mac OS X 10.15",
+            "name": "Mac OS X",
+            "version": "10.15"
+        },
+        "version": "72.0."
+    }
+}