mimecast: add cloud integrated logs data stream (elastic#11910)

Tested against a real endpoint. Pipeline test cases obtained from a test instance. Up to 10 examples of each available type are included. Not all types are represented.
eclecticiq · Dec 12, 2024 · 092dc86 · 092dc86
1 parent 6ee9c1a
commit 092dc86
Show file tree

Hide file tree

Showing 20 changed files with 2,308 additions and 1 deletion.
diff --git a/packages/mimecast/_dev/build/docs/README.md b/packages/mimecast/_dev/build/docs/README.md
@@ -49,6 +49,17 @@ https://integrations.mimecast.com/documentation/endpoint-reference/logs-and-stat
 
 {{fields "audit_events"}}
 
+### Cloud Integrated Logs
+
+This is the `mimecast.cloud_integrated_logs` dataset. These logs contain Mimecast
+threats and security events with the following details: entities, mail flows and URL
+protected events. More information about [these logs](
+https://developer.services.mimecast.com/docs/threatssecurityeventsanddataforci/1/routes/siem/v1/batch/events/ci/get).
+
+{{event "cloud_integrated_logs"}}
+
+{{fields "cloud_integrated_logs"}}
+
 ### DLP Logs
 
 This is the `mimecast.dlp_logs` dataset. These logs contain information about

diff --git a/packages/mimecast/_dev/deploy/docker/files/config.yml b/packages/mimecast/_dev/deploy/docker/files/config.yml
@@ -510,6 +510,139 @@ rules:
               "isCaughtUp": true
           }
 
+  - path: /siem/v1/batch/events/ci
+    methods: ["GET"]
+    query_params:
+      type: "entities"
+      nextPage: null
+    request_headers:
+      authorization: ["Bearer topsecretaccesstokenthatshouldnotbeleakedforabit"]
+    responses:
+      - status_code: 200
+        headers:
+          Content-Type:
+            - "application/octet-stream"
+        body: |
+          {
+              "value": [
+                  {
+                      "url": "http://svc-mimecast:8080/siemblob/ent",
+                      "expiry": "2024-11-19T02:14:04.839Z",
+                      "size": 629
+                  }
+              ],
+              "@nextPage": "nexttoken",
+              "isCaughtUp": false
+          }
+  - path: /siemblob/ent
+    methods: ["GET"]
+    responses:
+      - status_code: 200
+        headers:
+          Content-Type:
+            - "application/octet-stream"
+        body: '{{file "/files/ent.gz"}}'
+  - path: /siem/v1/batch/events/ci
+    methods: ["GET"]
+    query_params:
+      type: "entities"
+      nextPage: "nexttoken"
+    request_headers:
+      authorization: ["Bearer topsecretaccesstokenthatshouldnotbeleakedforabit"]
+    responses:
+      - status_code: 200
+        headers:
+          Content-Type:
+            - "application/octet-stream"
+        body: |
+          {
+              "value": [],
+              "@nextPage": "String",
+              "isCaughtUp": true
+          }
+  - path: /siem/v1/batch/events/ci
+    methods: ["GET"]
+    query_params:
+      type: "mailflow"
+      nextPage: null
+    request_headers:
+      authorization: ["Bearer topsecretaccesstokenthatshouldnotbeleakedforabit"]
+    responses:
+      - status_code: 200
+        headers:
+          Content-Type:
+            - "application/octet-stream"
+        body: |
+          {
+              "value": [
+                  {
+                      "url": "http://svc-mimecast:8080/siemblob/mail0",
+                      "expiry": "2024-11-19T02:14:04.839Z",
+                      "size": 361
+                  }
+              ],
+              "@nextPage": "nexttoken",
+              "isCaughtUp": false
+          }
+  - path: /siemblob/mail0
+    methods: ["GET"]
+    responses:
+      - status_code: 200
+        headers:
+          Content-Type:
+            - "application/octet-stream"
+        body: '{{file "/files/mail0.gz"}}'
+  - path: /siem/v1/batch/events/ci
+    methods: ["GET"]
+    query_params:
+      type: "mailflow"
+      nextPage: "nexttoken"
+    request_headers:
+      authorization: ["Bearer topsecretaccesstokenthatshouldnotbeleakedforabit"]
+    responses:
+      - status_code: 200
+        headers:
+          Content-Type:
+            - "application/octet-stream"
+        body: |
+          {
+              "value": [
+                  {
+                      "url": "http://svc-mimecast:8080/siemblob/mail1",
+                      "expiry": "2024-11-19T02:14:04.839Z",
+                      "size": 492
+                  }
+              ],
+              "@nextPage": "lasttoken",
+              "isCaughtUp": false
+          }
+  - path: /siemblob/mail1
+    methods: ["GET"]
+    responses:
+      - status_code: 200
+        headers:
+          Content-Type:
+            - "application/octet-stream"
+        body: '{{file "/files/mail1.gz"}}'
+  - path: /siem/v1/batch/events/ci
+    methods: ["GET"]
+    query_params:
+      type: "mailflow"
+      nextPage: "lasttoken"
+    request_headers:
+      authorization: ["Bearer topsecretaccesstokenthatshouldnotbeleakedforabit"]
+    responses:
+      - status_code: 200
+        headers:
+          Content-Type:
+            - "application/octet-stream"
+        body: |
+          {
+              "value": [],
+              "@nextPage": "String",
+              "isCaughtUp": true
+          }
+
   - path: /api/ttp/threat-intel/get-feed
     methods: ["POST"]
     request_body: /"feedType":"malware_customer","fileType":"stix","start":/

diff --git a/packages/mimecast/_dev/deploy/docker/files/ent.gz b/packages/mimecast/_dev/deploy/docker/files/ent.gz
diff --git a/packages/mimecast/_dev/deploy/docker/files/mail0.gz b/packages/mimecast/_dev/deploy/docker/files/mail0.gz
diff --git a/packages/mimecast/_dev/deploy/docker/files/mail1.gz b/packages/mimecast/_dev/deploy/docker/files/mail1.gz
diff --git a/packages/mimecast/changelog.yml b/packages/mimecast/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.3.0"
+  changes:
+    - description: Add `cloud_integrated_logs` data stream.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/11910
 - version: "2.2.0"
   changes:
     - description: Add "preserve_original_event" tag to documents with `event.kind` set to "pipeline_error".

diff --git a/...ecast/data_stream/cloud_integrated_logs/_dev/test/pipeline/test-cloud-integrated-logs.log b/...ecast/data_stream/cloud_integrated_logs/_dev/test/pipeline/test-cloud-integrated-logs.log
@@ -0,0 +1,20 @@
+{"_offset":1825747,"_partition":53,"accountId":"AUS2474","aggregateId":"4Xz4RX59R6z84lm-ip3sazqogbn7c5haarz3ydiaeg_1732724425","attachments":["1-MB-Test_PCI-PII.xls"],"direction":"INBOUND","historicalMail":false,"messageId":"<38989264-3456-4301-b2f9-22b8c78289db@SJ0PR18MB4090.namprd18.prod.outlook.com>","policiesApplied":[{"action":"DO_NOTHING","mode":"ACTIVE","name":"Default O365 Mail policy"}],"processingId":"127b2e8d1334f481d9dda0fe16530c1de6aede980e5313957516c6adddefddbc_1732724425","recipients":["[email protected]"],"senderEnvelope":"","senderHeader":"[email protected]","senderIp":"81.2.69.144","source":"OFFICE_365_MAIL","subject":"Undeliverable: Book Request","subtype":"NO_DETECTIONS","tags":null,"threatState":"DELIVERY_IN_PROGRESS","threatType":"NO_DETECTIONS","timestamp":1732724425453,"type":"entities"}
+{"_offset":1803495,"_partition":53,"accountId":"AUS2474","aggregateId":"4XvP1x6xnGzFB8Y-djq8atkief95yfdwmtw8thpgtr_1732206838","attachments":[],"direction":"INBOUND","historicalMail":false,"messageId":"<2470e325-20f1-40fb-ae9a-adf6b3efaddb@BY1PR18MB5971.namprd18.prod.outlook.com>","policiesApplied":[{"action":"DO_NOTHING","mode":"ACTIVE","name":"Default O365 Mail policy"}],"processingId":"e6d23f36b8e86965d62f6d9d94115b5ddbee437f2f5e184a253e57f0b6ed8441_1732206838","recipients":["[email protected]"],"senderEnvelope":"","senderHeader":"[email protected]","senderIp":"81.2.69.144","source":"OFFICE_365_MAIL","subject":"Undeliverable: Hoecker letter","subtype":"NO_DETECTIONS","tags":null,"threatState":"DELIVERY_IN_PROGRESS","threatType":"NO_DETECTIONS","timestamp":1732206838263,"type":"entities"}
+{"_offset":1807856,"_partition":53,"accountId":"AUS2474","aggregateId":"4Xw1kF1y6dzFyJB-hduwwwo67hmfwau3khzkdbfzo8_1732295253","attachments":[],"direction":"INBOUND","historicalMail":false,"messageId":"<[email protected]>","policiesApplied":[{"action":"DO_NOTHING","mode":"ACTIVE","name":"Default O365 Mail policy"}],"processingId":"e4caf5045cf8c4477d6a685eb9a6a2028ab84dc2f11b0b9ed175b6d4e4de124e_1732295253","recipients":["[email protected]"],"senderEnvelope":"[email protected]","senderHeader":"[email protected]","senderIp":"81.2.69.144","source":"OFFICE_365_MAIL","subject":"CONGRATULATIONS","subtype":"NO_DETECTIONS","tags":null,"threatState":"DELIVERY_IN_PROGRESS","threatType":"NO_DETECTIONS","timestamp":1732295253577,"type":"entities"}
+{"_offset":1807567,"_partition":53,"accountId":"AUS2474","aggregateId":"4Xvzwn3R6ZzFB8T-1p4nzntpnrcageetaz5mfk9bm8_1732290393","attachments":["Mimecast Termination.docx"],"direction":"INBOUND","historicalMail":false,"messageId":"<[email protected]>","policiesApplied":[{"action":"DO_NOTHING","mode":"ACTIVE","name":"Default O365 Mail policy"}],"processingId":"793eee52c80fb40134eed640afa87eaaadba4ac194c6cbd2d63fa3e6b75decfc_1732290393","recipients":["[email protected]"],"senderEnvelope":"[email protected]","senderHeader":"[email protected]","senderIp":"81.2.69.144","source":"OFFICE_365_MAIL","subject":"FW: San Juan Volumes","subtype":"NO_DETECTIONS","tags":null,"threatState":"DELIVERY_IN_PROGRESS","threatType":"NO_DETECTIONS","timestamp":1732290393848,"type":"entities"}
+{"_offset":1790506,"_partition":53,"accountId":"AUS2474","aggregateId":"4XsWdG63WDzFy33-kckoq8rx19ibc7iccjsjx6hsxd_1731943475","attachments":["Sandbox Test.xlsx"],"direction":"INBOUND","historicalMail":false,"messageId":"<[email protected]>","policiesApplied":[{"action":"BLOCK","mode":"ACTIVE","name":"Default O365 Mail policy"}],"processingId":"66e03b099e150698fc62f354796f2baa94c2f625a34ac92a0c7e8eb4a2afb11c_1731943475","recipients":["[email protected]"],"senderEnvelope":"[email protected]","senderHeader":"","senderIp":"81.2.69.144","source":"OFFICE_365_MAIL","subject":"Message from Node-RED","subtype":"MALWARE","tags":["MALWARE"],"threatState":"BLOCKED","threatType":"MALWARE","timestamp":1731943475250,"type":"entities"}
+{"_offset":1808181,"_partition":53,"accountId":"AUS2474","aggregateId":"4Xw3cr3g5BzFyJM-hw7nutbmioptjwcyjaxx6xiwi8_1732300380","attachments":[],"direction":"INBOUND","historicalMail":false,"messageId":"<[email protected]>","policiesApplied":[{"action":"DO_NOTHING","mode":"ACTIVE","name":"Default O365 Mail policy"}],"processingId":"4632969e69e264e6387667fe5b16b227e500e3f139bad62803ef05dc3db6f7f0_1732300380","recipients":["[email protected]"],"senderEnvelope":"[email protected]","senderHeader":"[email protected]","senderIp":"81.2.69.144","source":"OFFICE_365_MAIL","subject":"Mimecast quarantined a message detected as phishing from [email protected]","subtype":"NO_DETECTIONS","tags":null,"threatState":"DELIVERY_IN_PROGRESS","threatType":"NO_DETECTIONS","timestamp":1732300380868,"type":"entities"}
+{"_offset":1826382,"_partition":53,"accountId":"AUS2474","aggregateId":"4Xz8Fg2f0Vz84mw-gii93d63mdqw55hyxwhokyohhg_1732734727","attachments":[],"direction":"INBOUND","historicalMail":false,"messageId":"<315ecb5a-c9da-4920-beb4-3f9bd929629d@PH0PR18MB3990.namprd18.prod.outlook.com>","policiesApplied":[{"action":"DO_NOTHING","mode":"ACTIVE","name":"Default O365 Mail policy"}],"processingId":"d5e62983f7eb0b7b5eda24df668ef5b77d20adf2801b376dc799d1e44fc2ac52_1732734727","recipients":["[email protected]"],"senderEnvelope":"","senderHeader":"[email protected]","senderIp":"81.2.69.144","source":"OFFICE_365_MAIL","subject":"Undeliverable: Message from Node-RED","subtype":"NO_DETECTIONS","tags":null,"threatState":"DELIVERY_IN_PROGRESS","threatType":"NO_DETECTIONS","timestamp":1732734727836,"type":"entities"}
+{"_offset":1815844,"_partition":53,"accountId":"AUS2474","aggregateId":"4Xxhd273wrzFB8d-thdyqe6dgj83fsus95nwuzydwg_1732530231","attachments":["guess-what-day-it-is-hump-day-whoop-whoop-5af49483875db9003696b1cc.jpg"],"direction":"INBOUND","historicalMail":false,"messageId":"<cffa93ae-a44c-4d08-90f5-5c476afd0e20@CO6PR18MB3972.namprd18.prod.outlook.com>","policiesApplied":[{"action":"DO_NOTHING","mode":"ACTIVE","name":"Default O365 Mail policy"}],"processingId":"1b9fa7007467b03bcb45824cb283cc3dbaedeca258dd76737e42e40c33fff573_1732530231","recipients":["[email protected]"],"senderEnvelope":"","senderHeader":"[email protected]","senderIp":"81.2.69.144","source":"OFFICE_365_MAIL","subject":"Undeliverable: FW: CMS Opportunity","subtype":"NO_DETECTIONS","tags":null,"threatState":"DELIVERY_IN_PROGRESS","threatType":"NO_DETECTIONS","timestamp":1732530231824,"type":"entities"}
+{"_offset":1826720,"_partition":53,"accountId":"AUS2474","aggregateId":"4XzBMQ1fTlz84nK-5p1346dgpwaugm3gkr13k56agb_1732740434","attachments":[],"direction":"INBOUND","historicalMail":false,"messageId":"<889fdc9f-6e0c-4714-956d-9b20a6e63ab3@PH0PR18MB4734.namprd18.prod.outlook.com>","policiesApplied":[{"action":"DO_NOTHING","mode":"ACTIVE","name":"Default O365 Mail policy"}],"processingId":"b52070e40b85ddc3d74991bb3b9b80de3aa7df0133d927a275a2ca70a9eb1f84_1732740434","recipients":["[email protected]"],"senderEnvelope":"","senderHeader":"[email protected]","senderIp":"81.2.69.144","source":"OFFICE_365_MAIL","subject":"Undeliverable: P&D Pilots","subtype":"NO_DETECTIONS","tags":null,"threatState":"DELIVERY_IN_PROGRESS","threatType":"NO_DETECTIONS","timestamp":1732740434855,"type":"entities"}
+{"_offset":1790982,"_partition":53,"accountId":"AUS2474","aggregateId":"4XsZB263RjzFy2g-xjt37dsesi7fddbre96gprgpor_1731950379","attachments":["Screen Shot 2019-05-02 at 8.45.29 AM.png"],"direction":"INBOUND","historicalMail":false,"messageId":"<[email protected]>","policiesApplied":[{"action":"DO_NOTHING","mode":"ACTIVE","name":"Default O365 Mail policy"}],"processingId":"77f234462461122563d195bec0decddca5e6556bcdfc7c2797e62bda44979f63_1731950379","recipients":["[email protected]","[email protected]","[email protected]","[email protected]"],"senderEnvelope":"[email protected]","senderHeader":"[email protected]","senderIp":"81.2.69.144","source":"OFFICE_365_MAIL","subject":"Panhandle lateral operations","subtype":"NO_DETECTIONS","tags":null,"threatState":"DELIVERY_IN_PROGRESS","threatType":"NO_DETECTIONS","timestamp":1731950379429,"type":"entities"}
+{"_offset":1822823,"_partition":53,"accountId":"AUS2474","aggregateId":"4Xyccc0QWKz84PS-m1xto1ropcrh1cnp4tb7rzjsgn_1732660000","authResults":[{"aligned":true,"result":"pass","type":"SPF"},{"aligned":false,"result":"pass","type":"DKIM"},{"aligned":null,"result":"pass","type":"DMARC"}],"messageId":"<[email protected]>","processingId":"a3783bd4c73d1d157a366b4d61451f72e559af9251115b3275d6f020526d34d5_1732660000","subtype":null,"timestamp":1732660000405,"type":"mailflow"}
+{"_offset":1826870,"_partition":53,"accountId":"AUS2474","aggregateId":"4XzCL50vjVz84QW-1aohygaqrhj4oifhu6wz4xh36b_1732743069","authResults":[{"aligned":true,"result":"pass","type":"SPF"},{"aligned":false,"result":"pass","type":"DKIM"},{"aligned":null,"result":"fail","type":"DMARC"}],"messageId":"<774faf78-0457-4ac1-9987-fad251c0f445@LV3PR18MB6327.namprd18.prod.outlook.com>","processingId":"88f54ff7a57a7161477ae54bb2d3530f98b93f97ab17d745816bed5ec2475121_1732743069","subtype":null,"timestamp":1732743069761,"type":"mailflow"}
+{"_offset":1790380,"_partition":53,"accountId":"AUS2474","aggregateId":"4XsVwP74NKzFyJL-enzpecqrdi9b61fxxktmow9rsd_1731941558","authResults":[{"aligned":false,"result":"pass","type":"SPF"},{"aligned":false,"result":"none","type":"DKIM"},{"aligned":null,"result":"pass","type":"DMARC"}],"messageId":"<[email protected]>","processingId":"b6b6de3fb92174026699081356f2d9af211c654a4e3841a1839bd24f7f8e90ad_1731941558","subtype":null,"timestamp":1731941558336,"type":"mailflow"}
+{"_offset":1820809,"_partition":53,"accountId":"AUS2474","aggregateId":"4XyPGk5t8bz84mw-uonit3w88iwq13dqpqtzozhfir_1732629326","authResults":[{"aligned":true,"result":"pass","type":"SPF"},{"aligned":false,"result":"none","type":"DKIM"},{"aligned":null,"result":"pass","type":"DMARC"}],"messageId":"<[email protected]>","processingId":"46d8b9d3d194a6f9fff30255df9421cf46c2416568d7adeec4baf16a8980e696_1732629327","subtype":null,"timestamp":1732629327226,"type":"mailflow"}
+{"_offset":1802955,"_partition":53,"accountId":"AUS2474","aggregateId":"4XvL0W0KRGzFB8k-fxoe7ics1f9uk7y6aiuba5ek8g_1732198651","authResults":[{"aligned":true,"result":"pass","type":"SPF"},{"aligned":false,"result":"none","type":"DKIM"},{"aligned":null,"result":"pass","type":"DMARC"}],"messageId":"<[email protected]>","processingId":"7d5a611de68817a1aa0eac68cdf0fc8dba7878decbd639526ae862010331eb66_1732198651","subtype":null,"timestamp":1732198651453,"type":"mailflow"}
+{"_offset":1808128,"_partition":53,"accountId":"AUS2474","aggregateId":"4Xw3KV0jVqzFB8f-z3wa1ajy1jn6cftuepsu47xdob_1732299582","authResults":[{"aligned":true,"result":"pass","type":"SPF"},{"aligned":false,"result":"pass","type":"DKIM"},{"aligned":null,"result":"fail","type":"DMARC"}],"messageId":"<01c295b3-0110-4882-98a9-d60190c8e57b@CO1PR18MB4571.namprd18.prod.outlook.com>","processingId":"3e98a0a56f48c397d164d90e7129200c2b2e2ee801ee15a2de8e3dc556b99710_1732299582","subtype":null,"timestamp":1732299582518,"type":"mailflow"}
+{"_offset":1826377,"_partition":53,"accountId":"AUS2474","aggregateId":"4Xz88p751Yz84nC-a8cr4y85yq3hh6jjczgnn5sgj8_1732734475","authResults":[{"aligned":true,"result":"pass","type":"SPF"},{"aligned":false,"result":"none","type":"DKIM"},{"aligned":null,"result":"pass","type":"DMARC"}],"messageId":"<[email protected]>","processingId":"6a3e4abe35879b7a35f42fb785ddd4e9f0564d3a2584e0902120976e21d4a9ba_1732734475","subtype":null,"timestamp":1732734475302,"type":"mailflow"}
+{"_offset":1815702,"_partition":53,"accountId":"AUS2474","aggregateId":"4Xxg1c71KpzFB8T-j9zj9y3f91byjes4ysqetgk19g_1732525893","authResults":[{"aligned":true,"result":"pass","type":"SPF"},{"aligned":false,"result":"none","type":"DKIM"},{"aligned":null,"result":"pass","type":"DMARC"}],"messageId":"<[email protected]>","processingId":"821aff47dea6b57c6cb6bd262738eeabd28e2659f2ac0cb3ee490828d3a143f4_1732525893","subtype":null,"timestamp":1732525893398,"type":"mailflow"}
+{"_offset":1820908,"_partition":53,"accountId":"AUS2474","aggregateId":"4XyPl23Kn4z84ly-hnj5fgu9nbwnghx86mj18qp44b_1732630590","authResults":[{"aligned":true,"result":"pass","type":"SPF"},{"aligned":false,"result":"none","type":"DKIM"},{"aligned":null,"result":"pass","type":"DMARC"}],"messageId":"<[email protected]>","processingId":"3125eb6ff78c055eb7449a47286a453dcb66e176d2c751a6236bba4232c6fe31_1732630590","subtype":null,"timestamp":1732630590705,"type":"mailflow"}
+{"_offset":1803841,"_partition":53,"accountId":"AUS2474","aggregateId":"4XvR1B4m7BzFB8L-qk59b4szrgayciaagczc977rzb_1732212206","authResults":[{"aligned":true,"result":"pass","type":"SPF"},{"aligned":false,"result":"none","type":"DKIM"},{"aligned":null,"result":"pass","type":"DMARC"}],"messageId":"<[email protected]>","processingId":"c40337e6860db0301575d8d09362bff214c0b010d6c4d41da9d770759ff54d10_1732212206","subtype":null,"timestamp":1732212206960,"type":"mailflow"}