[Imperva] update imperva ECS version and add event.original options

packages/imperva/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.0.0"
+  changes:
+    - description: update to ECS 1.10.0 and adding event.original options
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1055
 - version: "0.1.4"
   changes:
     - description: update to ECS 1.9.0

packages/imperva/data_stream/securesphere/_dev/test/pipeline/test-common-config.yml

@@ -0,0 +1,5 @@
+dynamic_fields:
+  event.ingested: ".*"
+fields:
+  tags:
+    - preserve_original_event

packages/imperva/data_stream/securesphere/agent/stream/stream.yml.hbs

@@ -4,8 +4,11 @@ paths:
 {{/each}}
 exclude_files: [".gz$"]
 tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
 {{#each tags as |tag i|}}
- - {{tag}}
+  - {{tag}}
 {{/each}}
 fields_under_root: true
 fields:
@@ -18,6 +21,9 @@ publisher_pipeline.disable_host: true
 {{/contains}}
 
 processors:
+{{#if processors}}
+{{processors}}
+{{/if}}
 - script:
     lang: javascript
     params:
@@ -2902,8 +2908,4 @@ processors:
     target_field: url.registered_domain
     target_subdomain_field: url.subdomain
     target_etld_field: url.top_level_domain
-- add_locale: ~
-- add_fields:
-    target: ''
-    fields:
-        ecs.version: 1.9.0
+- add_locale: ~

packages/imperva/data_stream/securesphere/agent/stream/tcp.yml.hbs

@@ -1,8 +1,11 @@
 tcp:
 host: "{{tcp_host}}:{{tcp_port}}"
 tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
 {{#each tags as |tag i|}}
- - {{tag}}
+  - {{tag}}
 {{/each}}
 fields_under_root: true
 fields:
@@ -15,6 +18,9 @@ publisher_pipeline.disable_host: true
 {{/contains}}
 
 processors:
+{{#if processors}}
+{{processors}}
+{{/if}}
 - script:
     lang: javascript
     params:
@@ -2899,8 +2905,4 @@ processors:
     target_field: url.registered_domain
     target_subdomain_field: url.subdomain
     target_etld_field: url.top_level_domain
-- add_locale: ~
-- add_fields:
-    target: ''
-    fields:
-        ecs.version: 1.9.0
+- add_locale: ~

packages/imperva/data_stream/securesphere/agent/stream/udp.yml.hbs

@@ -1,8 +1,11 @@
 udp:
 host: "{{udp_host}}:{{udp_port}}"
 tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
 {{#each tags as |tag i|}}
- - {{tag}}
+  - {{tag}}
 {{/each}}
 fields_under_root: true
 fields:
@@ -15,6 +18,9 @@ publisher_pipeline.disable_host: true
 {{/contains}}
 
 processors:
+{{#if processors}}
+{{processors}}
+{{/if}}
 - script:
     lang: javascript
     params:
@@ -2899,8 +2905,4 @@ processors:
     target_field: url.registered_domain
     target_subdomain_field: url.subdomain
     target_etld_field: url.top_level_domain
-- add_locale: ~
-- add_fields:
-    target: ''
-    fields:
-        ecs.version: 1.9.0
+- add_locale: ~

packages/imperva/data_stream/securesphere/elasticsearch/ingest_pipeline/default.yml

@@ -4,60 +4,68 @@ description: Pipeline for Imperva SecureSphere
 processors:
   # ECS event.ingested
   - set:
-        field: event.ingested
-        value: '{{_ingest.timestamp}}'
+      field: event.ingested
+      value: "{{_ingest.timestamp}}"
+  - set:
+      field: ecs.version
+      value: "1.10.0"
   # User agent
   - user_agent:
-        field: user_agent.original
-        ignore_missing: true
+      field: user_agent.original
+      ignore_missing: true
   # IP Geolocation Lookup
   - geoip:
-        field: source.ip
-        target_field: source.geo
-        ignore_missing: true
+      field: source.ip
+      target_field: source.geo
+      ignore_missing: true
   - geoip:
-        field: destination.ip
-        target_field: destination.geo
-        ignore_missing: true
+      field: destination.ip
+      target_field: destination.geo
+      ignore_missing: true
 
   # IP Autonomous System (AS) Lookup
   - geoip:
-        database_file: GeoLite2-ASN.mmdb
-        field: source.ip
-        target_field: source.as
-        properties:
-            - asn
-            - organization_name
-        ignore_missing: true
+      database_file: GeoLite2-ASN.mmdb
+      field: source.ip
+      target_field: source.as
+      properties:
+          - asn
+          - organization_name
+      ignore_missing: true
   - geoip:
-        database_file: GeoLite2-ASN.mmdb
-        field: destination.ip
-        target_field: destination.as
-        properties:
-            - asn
-            - organization_name
-        ignore_missing: true
+      database_file: GeoLite2-ASN.mmdb
+      field: destination.ip
+      target_field: destination.as
+      properties:
+          - asn
+          - organization_name
+      ignore_missing: true
   - rename:
-        field: source.as.asn
-        target_field: source.as.number
-        ignore_missing: true
+      field: source.as.asn
+      target_field: source.as.number
+      ignore_missing: true
   - rename:
-        field: source.as.organization_name
-        target_field: source.as.organization.name
-        ignore_missing: true
+      field: source.as.organization_name
+      target_field: source.as.organization.name
+      ignore_missing: true
   - rename:
-        field: destination.as.asn
-        target_field: destination.as.number
-        ignore_missing: true
+      field: destination.as.asn
+      target_field: destination.as.number
+      ignore_missing: true
   - rename:
-        field: destination.as.organization_name
-        target_field: destination.as.organization.name
-        ignore_missing: true
+      field: destination.as.organization_name
+      target_field: destination.as.organization.name
+      ignore_missing: true
   - append:
-        field: related.hosts
-        value: '{{host.name}}'
-        allow_duplicates: false
-        if: ctx.host?.name != null && ctx.host?.name != ''
+      field: related.hosts
+      value: '{{host.name}}'
+      allow_duplicates: false
+      if: ctx.host?.name != null && ctx.host?.name != ''
+  - remove:
+      field: event.original
+      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
+      ignore_failure: true
+      ignore_missing: true
 on_failure:
   - append:
         field: error.message

packages/imperva/data_stream/securesphere/manifest.yml

@@ -54,6 +54,23 @@ streams:
         required: false
         show_user: false
         default: false
+      - name: preserve_original_event
+        required: true
+        show_user: true
+        title: Preserve original event
+        description: Preserves a raw copy of the original event, added to the field `event.original`
+        type: bool
+        multi: false
+        default: false
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
+
   - input: tcp
     title: Imperva SecureSphere logs
     description: Collect Imperva SecureSphere logs
@@ -106,6 +123,23 @@ streams:
         required: false
         show_user: false
         default: false
+      - name: preserve_original_event
+        required: true
+        show_user: true
+        title: Preserve original event
+        description: Preserves a raw copy of the original event, added to the field `event.original`
+        type: bool
+        multi: false
+        default: false
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
+
   - input: logfile
     enabled: false
     title: Imperva SecureSphere logs
@@ -152,3 +186,19 @@ streams:
         required: false
         show_user: false
         default: false
+      - name: preserve_original_event
+        required: true
+        show_user: true
+        title: Preserve original event
+        description: Preserves a raw copy of the original event, added to the field `event.original`
+        type: bool
+        multi: false
+        default: false
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >-
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.

packages/imperva/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: imperva
 title: Imperva SecureSphere
-version: 0.1.4
+version: 1.0.0
 description: Imperva SecureSphere Integration
 categories: ["network", "security"]
 release: experimental