[Cyberark] update cyberark package ECS version and add event.original…

… option (elastic#1037) * update cyberark package ECS version and add event.original option * update changelog and manifest * linting * linting processors * update version and linting
build-security · Jun 10, 2021 · d53f2a9 · d53f2a9
1 parent 37ff332
commit d53f2a9
Show file tree

Hide file tree

Showing 10 changed files with 1,438 additions and 59 deletions.
diff --git a/packages/cyberark/changelog.yml b/packages/cyberark/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.2.0"
+  changes:
+    - description: update to ECS 1.10.0 and add event.original options
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1037
 - version: "0.1.4"
   changes:
     - description: update to ECS 1.9.0

diff --git a/packages/cyberark/data_stream/corepas/_dev/test/pipeline/test-generated.log b/packages/cyberark/data_stream/corepas/_dev/test/pipeline/test-generated.log
diff --git a/packages/cyberark/data_stream/corepas/_dev/test/pipeline/test-generated.log-config.yml b/packages/cyberark/data_stream/corepas/_dev/test/pipeline/test-generated.log-config.yml
@@ -0,0 +1,5 @@
+dynamic_fields:
+  event.ingested: ".*"
+fields:
+  tags:
+    - preserve_original_event
diff --git a/packages/cyberark/data_stream/corepas/_dev/test/pipeline/test-generated.log-expected.json b/packages/cyberark/data_stream/corepas/_dev/test/pipeline/test-generated.log-expected.json
diff --git a/packages/cyberark/data_stream/corepas/agent/stream/stream.yml.hbs b/packages/cyberark/data_stream/corepas/agent/stream/stream.yml.hbs
@@ -4,8 +4,11 @@ paths:
 {{/each}}
 exclude_files: [".gz$"]
 tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
 {{#each tags as |tag i|}}
- - {{tag}}
+  - {{tag}}
 {{/each}}
 fields_under_root: true
 fields:
@@ -18,6 +21,9 @@ publisher_pipeline.disable_host: true
 {{/contains}}
 
 processors:
+{{#if processors}}
+{{processors}}
+{{/if}}
 - script:
     lang: javascript
     params:
@@ -8825,8 +8831,4 @@ processors:
     target_field: url.registered_domain
     target_subdomain_field: url.subdomain
     target_etld_field: url.top_level_domain
-- add_locale: ~
-- add_fields:
-    target: ''
-    fields:
-        ecs.version: 1.9.0
+- add_locale: ~
diff --git a/packages/cyberark/data_stream/corepas/agent/stream/tcp.yml.hbs b/packages/cyberark/data_stream/corepas/agent/stream/tcp.yml.hbs
@@ -1,8 +1,11 @@
 tcp:
 host: "{{tcp_host}}:{{tcp_port}}"
 tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
 {{#each tags as |tag i|}}
- - {{tag}}
+  - {{tag}}
 {{/each}}
 fields_under_root: true
 fields:
@@ -15,6 +18,9 @@ publisher_pipeline.disable_host: true
 {{/contains}}
 
 processors:
+{{#if processors}}
+{{processors}}
+{{/if}}
 - script:
     lang: javascript
     params:
@@ -8823,7 +8829,3 @@ processors:
     target_subdomain_field: url.subdomain
     target_etld_field: url.top_level_domain
 - add_locale: ~
-- add_fields:
-    target: ''
-    fields:
-        ecs.version: 1.9.0
diff --git a/packages/cyberark/data_stream/corepas/agent/stream/udp.yml.hbs b/packages/cyberark/data_stream/corepas/agent/stream/udp.yml.hbs
@@ -1,8 +1,11 @@
 udp:
 host: "{{udp_host}}:{{udp_port}}"
 tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
 {{#each tags as |tag i|}}
- - {{tag}}
+  - {{tag}}
 {{/each}}
 fields_under_root: true
 fields:
@@ -15,6 +18,9 @@ publisher_pipeline.disable_host: true
 {{/contains}}
 
 processors:
+{{#if processors}}
+{{processors}}
+{{/if}}
 - script:
     lang: javascript
     params:
@@ -8823,7 +8829,3 @@ processors:
     target_subdomain_field: url.subdomain
     target_etld_field: url.top_level_domain
 - add_locale: ~
-- add_fields:
-    target: ''
-    fields:
-        ecs.version: 1.9.0
diff --git a/packages/cyberark/data_stream/corepas/elasticsearch/ingest_pipeline/default.yml b/packages/cyberark/data_stream/corepas/elasticsearch/ingest_pipeline/default.yml
@@ -4,61 +4,69 @@ description: Pipeline for Cyber-Ark
 processors:
   # ECS event.ingested
   - set:
-        field: event.ingested
-        value: '{{_ingest.timestamp}}'
+      field: event.ingested
+      value: '{{_ingest.timestamp}}'
+  - set:
+      field: ecs.version
+      value: "1.10.0"
   # User agent
   - user_agent:
-        field: user_agent.original
-        ignore_missing: true
+      field: user_agent.original
+      ignore_missing: true
   # IP Geolocation Lookup
   - geoip:
-        field: source.ip
-        target_field: source.geo
-        ignore_missing: true
+      field: source.ip
+      target_field: source.geo
+      ignore_missing: true
   - geoip:
-        field: destination.ip
-        target_field: destination.geo
-        ignore_missing: true
+      field: destination.ip
+      target_field: destination.geo
+      ignore_missing: true
 
   # IP Autonomous System (AS) Lookup
   - geoip:
-        database_file: GeoLite2-ASN.mmdb
-        field: source.ip
-        target_field: source.as
-        properties:
-            - asn
-            - organization_name
-        ignore_missing: true
+      database_file: GeoLite2-ASN.mmdb
+      field: source.ip
+      target_field: source.as
+      properties:
+      - asn
+      - organization_name
+      ignore_missing: true
   - geoip:
-        database_file: GeoLite2-ASN.mmdb
-        field: destination.ip
-        target_field: destination.as
-        properties:
-            - asn
-            - organization_name
-        ignore_missing: true
+      database_file: GeoLite2-ASN.mmdb
+      field: destination.ip
+      target_field: destination.as
+      properties:
+      - asn
+      - organization_name
+      ignore_missing: true
   - rename:
-        field: source.as.asn
-        target_field: source.as.number
-        ignore_missing: true
+      field: source.as.asn
+      target_field: source.as.number
+      ignore_missing: true
   - rename:
-        field: source.as.organization_name
-        target_field: source.as.organization.name
-        ignore_missing: true
+      field: source.as.organization_name
+      target_field: source.as.organization.name
+      ignore_missing: true
   - rename:
-        field: destination.as.asn
-        target_field: destination.as.number
-        ignore_missing: true
+      field: destination.as.asn
+      target_field: destination.as.number
+      ignore_missing: true
   - rename:
-        field: destination.as.organization_name
-        target_field: destination.as.organization.name
-        ignore_missing: true
+      field: destination.as.organization_name
+      target_field: destination.as.organization.name
+      ignore_missing: true
   - append:
-        field: related.hosts
-        value: '{{host.name}}'
-        allow_duplicates: false
-        if: ctx.host?.name != null && ctx.host?.name != ''
+      field: related.hosts
+      value: '{{host.name}}'
+      allow_duplicates: false
+      if: ctx.host?.name != null && ctx.host?.name != ''
+  - remove:
+      field: event.original
+      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
+      ignore_failure: true
+      ignore_missing: true
 on_failure:
   - append:
-        field: error.message
-        value: "{{ _ingest.on_failure_message }}"
+      field: error.message
+      value: "{{ _ingest.on_failure_message }}"
diff --git a/packages/cyberark/data_stream/corepas/manifest.yml b/packages/cyberark/data_stream/corepas/manifest.yml
@@ -54,6 +54,23 @@ streams:
         required: false
         show_user: false
         default: false
+      - name: preserve_original_event
+        required: true
+        show_user: true
+        title: Preserve original event
+        description: Preserves a raw copy of the original event, added to the field `event.original`
+        type: bool
+        multi: false
+        default: false
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
+
   - input: tcp
     title: CyberArk logs
     description: Collect CyberArk logs
@@ -106,6 +123,23 @@ streams:
         required: false
         show_user: false
         default: false
+      - name: preserve_original_event
+        required: true
+        show_user: true
+        title: Preserve original event
+        description: Preserves a raw copy of the original event, added to the field `event.original`
+        type: bool
+        multi: false
+        default: false
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
+
   - input: logfile
     enabled: false
     title: CyberArk logs
@@ -152,3 +186,20 @@ streams:
         required: false
         show_user: false
         default: false
+      - name: preserve_original_event
+        required: true
+        show_user: true
+        title: Preserve original event
+        description: Preserves a raw copy of the original event, added to the field `event.original`
+        type: bool
+        multi: false
+        default: false
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
+
diff --git a/packages/cyberark/manifest.yml b/packages/cyberark/manifest.yml
@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: cyberark
 title: Cyber-Ark
-version: 0.1.4
+version: 0.2.0
 description: Cyber-Ark Integration
 categories: ["security"]
 release: experimental