[crowdstrike] Add support to parse the custom notification format provided by Crowdstrike

packages/crowdstrike/_dev/build/docs/README.md

@@ -22,7 +22,17 @@ Contains endpoint data and CrowdStrike Falcon platform audit data forwarded from
 
 ### FDR
 
-The Falcon Data Replicator replicates log data from your CrowdStrike environment to a stand-alone target. This target can be a location on the file system, or an S3 bucket.
+The Falcon Data Replicator replicates log data from your CrowdStrike environment to a stand-alone target. 
+This target can be configured in different ways:
+
+- Use directly the AWS SQS queue provided by Crowdstrike. This is the default.
+- You can use the [FDR tool](https://github.com/CrowdStrike/FDR) (or any other similar) and read from a 
+location, or from a different SQS queue managed by you. 
+
+If the intention is to read from local files, is important to note that they can't be in `gzip` format
+and they will need to be extracted first. 
+When using an AWS SQS queue that is not the one managed by
+Crowdstrike, is important to disable the `Is FDR Queue` option in order to parse the notifications properly.
 
 
 #### Configuration for the S3 input

packages/crowdstrike/changelog.yml

@@ -1,4 +1,12 @@
 # newer versions go on top
+- version: "1.0.4"
+  changes:
+    - description: Add ability to read from both FDR provided and user owned SQS queues for FDR.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/2198
+    - description: Pipeline fixes for FDR
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/2198
 - version: "1.0.3"
   changes:
     - description: Uniform with guidelines

packages/crowdstrike/data_stream/fdr/_dev/test/pipeline/test-fdr.log

@@ -122,3 +122,4 @@
 {"AuthenticationId":"703298","ConfigBuild":"1007.3.0011603.1","ConfigStateHash":"2642284486","ContextProcessId":"1161025471861","ContextThreadId":"34929528116709","ContextTimeStamp":"1604851030.593","DiskParentDeviceInstanceId":"USB\\VID_1058\u0026PID_2621\\57583431453939315A4C5255","EffectiveTransmissionClass":"3","Entitlements":"15","FileEcpBitmask":"0","FileIdentifier":"262fbc677256cf4c8d6c6a227285a072c06830873b000000","FileObject":"18446664963104449168","IrpFlags":"1028","IsOnNetwork":"0","IsOnRemovableDisk":"1","MajorFunction":"18","MinorFunction":"0","OperationFlags":"0","Size":"517029","TargetFileName":"\\Device\\HarddiskVolume5\\01.png.tmp$$","TokenType":"1","UserName":"user9","aid":"ffffffff16bf4c7bb5ad755a4722025c","aip":"208.216.134.196","cid":"ffffffff30a3407dae27d0503611022d","event_platform":"Win","event_simpleName":"GenericFileWritten","id":"ffffffff-1111-11eb-800a-06cecfd73923","name":"GenericFileWrittenV11","timestamp":"1604851031298"}
 {"ConfigBuild":"1007.3.0011603.1","ConfigStateHash":"666346415","ContextProcessId":"1717987648455","ContextThreadId":"55064470042288","ContextTimeStamp":"1604850899.164","EffectiveTransmissionClass":"3","Entitlements":"15","VolumeName":"\\Device\\HarddiskVolume27","aid":"ffffffff896b43725b83c79aa79959da","aip":"208.216.150.196","cid":"ffffffff30a3407dae27d0503611022d","event_platform":"Win","event_simpleName":"FsVolumeUnmounted","id":"ffffffff-1111-11eb-9f70-0634389d9ea9","name":"FsVolumeUnmountedV2","timestamp":"1604850899812"}
 {"ConfigBuild":"1007.4.0009906.1","ConfigStateHash":"3429017943","ContextProcessId":"66426035996442255","ContextTimeStamp":"1604851098.548","Entitlements":"15","aid":"ffffffff899541b94b9adff8922aa70a","aip":"208.193.200.164","cid":"ffffffff30a3407dae27d0503611022d","event_platform":"Mac","event_simpleName":"FirewallDisabled","id":"ffffffff-1111-11eb-9d4c-02f402df8c1f","name":"FirewallDisabledMacV1","timestamp":"1604851040625"}
+{"AgentLoadFlags":"0","AgentLocalTime":"1636436839.9529998","AgentTimeOffset":"125.319","AgentVersion":"6.31.14404.0","BiosManufacturer":"Apple Inc.","BiosVersion":"1554.140.20.0.0 (iBridge: 18.16.14759.0.1,0)","ChassisType":"Laptop","City":"San Francisco","ComputerName":"mac1","ConfigBuild":"1007.4.0014404.1","ConfigIDBuild":"14404","Continent":"North America","Country":"United States","FalconGroupingTags":"-","FirstSeen":"1625682391.0","HostHiddenStatus":"Visible","MachineDomain":"none","OU":"none","PointerSize":"none","ProductType":"1","SensorGroupingTags":"-","ServicePackMajor":"none","SiteName":"none","SystemManufacturer":"Apple Inc.","SystemProductName":"MacBookPro16,2","Time":"1636448427.3539999","Timezone":"America/Los_Angeles","Version":"Big Sur (11.0)","aid":"fffffffffffaaaaaaaaabbbbbbbb","aip":"208.30.227.225","cid":"ffffffff30a3407dae27d0503611022ff","event_platform":"Mac"}

packages/crowdstrike/data_stream/fdr/agent/stream/aws-s3.yml.hbs

@@ -32,6 +32,9 @@ fips_enabled: {{fips_enabled}}
 {{#if proxy_url }}
 proxy_url: {{proxy_url}}
 {{/if}}
+{{#if fdr_queue}}
+sqs.notification_parse_script: {{fdr_parsing_script}}
+{{/if}}
 {{#if tags.length}}
 tags:
 {{else}}

packages/crowdstrike/data_stream/fdr/elasticsearch/ingest_pipeline/default.yml

@@ -3,20 +3,24 @@ description: Pipeline for processing sample logs
 processors:
   ## Message decoding.
   - rename:
+      tag: message-to-original
       field: message
       target_field: event.original
   - json:
+      tag: json-decoding
       description: Decodes original JSON into `crowdstrike` field.
       field: event.original
       target_field: crowdstrike
   - date:
+      tag: date-timestamp
       description: Parse timestamp from event.
       field: crowdstrike.timestamp
       target_field: event.created
       formats:
         - UNIX_MS
       ignore_failure: true
   - date:
+      tag: date-event-created
       description: Parse timestamp from event.
       field: crowdstrike.CreationTimeStamp
       target_field: event.created
@@ -25,15 +29,18 @@ processors:
       ignore_failure: true
       if: ctx?.event?.created == null
   - set:
+      tag: set-timestamp
       field: "@timestamp"
       copy_from: event.created
       if: ctx?.event?.created != null && (ctx?.crowdstrike?.ContextTimeStamp == null || ctx?.crowdstrike?.ContextTimeStamp == "")
   - date:
+      tag: date-context-timestamp
       field: crowdstrike.ContextTimeStamp
       formats:
         - UNIX
       ignore_failure: true
   - rename:
+      tag: rename-message
       field: crowdstrike.message
       target_field: message
       ignore_missing: true
@@ -45,6 +52,7 @@ processors:
 
   ## Categorization.
   - script:
+      tag: script-categorize-events
       description: Categorize events.
       lang: painless
       source: |-
@@ -306,7 +314,10 @@ processors:
         for (entry in ctx.crowdstrike.entrySet()) {
           def key = entry.getKey().toString();
           if (key.contains("Count") || key.contains("Port")) {
-            ctx.crowdstrike[key] = Long.parseLong(entry.getValue().toString());
+            try {
+              ctx.crowdstrike[key] = Long.parseLong(entry.getValue().toString());
+            } catch (Exception e) {
+            }
           }
         }
   - script:
@@ -369,6 +380,11 @@ processors:
       field: observer.address
       copy_from: observer.ip
       ignore_empty_value: true
+  - rename:
+      field: crowdstrike.AgentVersion
+      target_field: observer.version
+      ignore_missing: true
+      ignore_failure: true
   - rename:
       field: crowdstrike.ConfigBuild
       target_field: observer.version
@@ -377,6 +393,9 @@ processors:
   - set:
       field: observer.vendor
       value: crowdstrike
+  - set:
+      field: observer.type
+      value: agent
   - append:
       field: related.ip
       value: "{{observer.ip}}"
@@ -404,6 +423,31 @@ processors:
       value: "{{host.name}}"
       allow_duplicates: false
       if: ctx.host?.name != null
+  - rename:
+      field: crowdstrike.City
+      target_field: host.geo.city_name
+      ignore_missing: true
+      ignore_failure: true
+  - rename:
+      field: crowdstrike.Continent
+      target_field: host.geo.continent_name
+      ignore_missing: true
+      ignore_failure: true
+  - rename:
+      field: crowdstrike.Country
+      target_field: host.geo.country_name
+      ignore_missing: true
+      ignore_failure: true
+  - rename:
+      field: crowdstrike.Timezone
+      target_field: host.geo.timezone
+      ignore_missing: true
+      ignore_failure: true
+  - rename:
+      field: crowdstrike.MachineDomain
+      target_field: host.domain
+      ignore_missing: true
+      ignore_failure: true
 
   ## IP Geolocation Lookup
   - geoip:
@@ -466,6 +510,12 @@ processors:
       field: crowdstrike.OSVersionString
       target_field: os.version
       ignore_missing: true
+      ignore_failure: true
+  - rename:
+      field: crowdstrike.Version
+      target_field: os.version
+      ignore_missing: true
+      ignore_failure: true
 
   ## Process fields.
   - rename:
@@ -1037,6 +1087,16 @@ processors:
       if: ctx?.event?.action != null && (ctx.event.action.contains("File") || ctx.event.action.contains("Directory") || ctx.event.action.contains("Executable")) && ctx?._temp?.hashes != null && ctx?._temp?.hashes.size() > 0
 
   ## Crowdstrike fields.
+  - split:
+      field: crowdstrike.FalconGroupingTags
+      separator: ",\\s?"
+      ignore_missing: true
+      ignore_failure: true
+  - split:
+      field: crowdstrike.SensorGroupingTags
+      separator: ",\\s?"
+      ignore_missing: true
+      ignore_failure: true
   - split:
       field: crowdstrike.Tags
       separator: ",\\s?"
@@ -1102,6 +1162,23 @@ processors:
       formats:
         - UNIX
       if: ctx?.crowdstrike?.AgentLocalTime != null && ctx?.crowdstrike?.AgentLocalTime != ""
+  - date:
+      field: crowdstrike.FirstSeen
+      target_field: crowdstrike.FirstSeen
+      formats:
+        - UNIX
+      if: ctx?.crowdstrike?.FirstSeen != null && ctx?.crowdstrike?.FirstSeen != ""
+  - date:
+      field: crowdstrike.Time
+      target_field: crowdstrike.Time
+      formats:
+        - UNIX
+      if: ctx?.crowdstrike?.Time != null && ctx?.crowdstrike?.Time != ""
+  - convert:
+      field: crowdstrike.AgentTimeOffset
+      target_field: crowdstrike.AgentTimeOffset
+      type: float
+      ignore_missing: true
   - convert:
       field: crowdstrike.Timeout
       type: long
@@ -1181,7 +1258,7 @@ processors:
                 handleList(x);
             }
           }
-          map.values().removeIf(v -> v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0));
+          map.values().removeIf(v -> v == null || v == '' || v == '-' || v == 'none' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0));
         }
         void handleList(List list) {
           for (def x : list) {
@@ -1191,10 +1268,10 @@ processors:
                   handleList(x);
               }
           }
-          list.removeIf(v -> v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0));
+          list.removeIf(v -> v == null || v == '' || v == '-' || v == 'none' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0));
         }
         handleMap(ctx);
 on_failure:
   - set:
       field: error.message
-      value: "{{ _ingest.on_failure_message }}"
+      value: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message {{ _ingest.on_failure_message }}"

packages/crowdstrike/data_stream/fdr/fields/ecs.yml

@@ -90,6 +90,8 @@
   name: observer.vendor
 - external: ecs
   name: observer.version
+- external: ecs
+  name: observer.type
 - external: ecs
   name: os.type
 - external: ecs

packages/crowdstrike/data_stream/fdr/fields/fields.yml

@@ -3,6 +3,8 @@
 - name: crowdstrike
   type: group
   fields:
+    - name: AgentTimeOffset
+      type: float
     - name: AllocateVirtualMemoryCount
       type: long
     - name: ApiReturnValue
@@ -31,6 +33,8 @@
       type: keyword
     - name: CompletionEventId
       type: keyword
+    - name: ConfigBuild
+      type: keyword
     - name: ConHostId
       type: keyword
     - name: ConHostProcessId
@@ -75,6 +79,8 @@
       type: long
     - name: ExecutableDeletedCount
       type: long
+    - name: FalconGroupingTags
+      type: keyword
     - name: FileAttributes
       type: keyword
     - name: FileDeletedCount
@@ -83,12 +89,16 @@
       type: keyword
     - name: FileObject
       type: keyword
+    - name: FirstSeen
+      type: date
     - name: Flags
       type: keyword
     - name: GenericFileWrittenCount
       type: long
     - name: GrandParentBaseFileName
       type: keyword
+    - name: HostHiddenStatus
+      type: keyword
     - name: ImageSubsystem
       type: keyword
     - name: InContext
@@ -165,12 +175,16 @@
       type: keyword
     - name: Options
       type: keyword
+    - name: OU
+      type: keyword
     - name: ParentAuthenticationId
       type: keyword
     - name: PasswordLastSet
       type: keyword
     - name: PhysicalAddressLength
       type: long
+    - name: PointerSize
+      type: keyword
     - name: PrivilegedProcessHandleCount
       type: long
     - name: PrivilegesBitmask
@@ -181,6 +195,8 @@
       type: keyword
     - name: ProcessSxsFlags
       type: keyword
+    - name: ProductType
+      type: keyword
     - name: ProtectVirtualMemoryCount
       type: long
     - name: QueueApcCount
@@ -215,10 +231,14 @@
       type: long
     - name: ScriptEngineInvocationCount
       type: long
+    - name: SensorGroupingTags
+      type: keyword
     - name: ServiceDisplayName
       type: keyword
     - name: ServiceEventCount
       type: long
+    - name: ServicePackMajor
+      type: keyword
     - name: SessionId
       type: keyword
     - name: SessionProcessId
@@ -229,6 +249,8 @@
       type: keyword
     - name: Size
       type: long
+    - name: SiteName
+      type: keyword
     - name: SnapshotFileOpenCount
       type: long
     - name: SourceFileName
@@ -255,6 +277,8 @@
       type: keyword
     - name: TargetThreadId
       type: keyword
+    - name: Time
+      type: date
     - name: Timeout
       type: long
     - name: TokenType